Applying GIS, Data Vis. and Deep Learning to environmental science and nuclear emergency responses
Public
Edited
Nov 1
rawData.flatMap((d) => d.properties.value)
function matchNDValue(str) {
const match = str.match(/ND\(([0-9.]+)\)/);
return match ? parseFloat(match[1]) : null;
}
function isND(str) {
return str.includes("ND");
}
matchNDValue("ND(0.69)")
isND("0.69") ? 1 : 2
rawData.flatMap((d) =>
d.properties.nuclide.map((p, i) => {
const nearestStation = turf.nearestPoint(d.geometry, stations);
const props = d.properties;
const geom = d.geometry;
return {
begPeriod: new Date(props.date[i]),
nuclide: props.nuclide[i],
station: nearestStation.properties.station,
org: nearestStation.properties.org,
lon: geom.coordinates[0],
lat: geom.coordinates[1],
activity: isND(props.value[0][i]) ? null : parseFloat(props.value[0][i]),
dl: isND(props.value[0][i]) ? matchNDValue(props.value[0][i]) : null
};
})
)
// or more succintly
rawData.flatMap(({ properties: props, geometry: geom }) =>
props.nuclide.map((nuclide, i) => {
const nearestStation = turf.nearestPoint(geom, stations);
const [lon, lat] = geom.coordinates;
const value = props.value[0][i];
return {
begPeriod: new Date(props.date[i]),
nuclide,
station: nearestStation.properties.station,
org: nearestStation.properties.org,
lon,
lat,
activity: isND(value) ? null : parseFloat(value),
dl: isND(value) ? matchNDValue(value) : null
};
})
)
d3.extent(processedData.map((d) => d.begperiod))
SeawaterSamplingLocations = FileAttachment("Seawater_station_points.csv").csv()
stations = turf.featureCollection(
SeawaterSamplingLocations.map((item) => {
return turf.point([parseFloat(item.Longitude), parseFloat(item.Latitude)], {
org: item.Organization,
station: item.Station
});
})
)
turf.nearestPoint(rawData[0].geometry, stations).properties
// const targetPoint = turf.point([longitude, latitude]);
// const nearest = turf.nearestPoint(targetPoint, featureCollection);
function transformMonitoringData(inputData, SampleType) {
function parseDMSCoordinate(dmsStr) {
const regex = /(\d+)°(\d+)′([\d.]+)″([NSEWnsew])/;
const match = dmsStr.match(regex);
if (!match) return null;
const [_, degrees, minutes, seconds, direction] = match;
let decimal = parseFloat(degrees) + parseFloat(minutes)/60 + parseFloat(seconds)/3600;
if (direction.toUpperCase() === 'S' || direction.toUpperCase() === 'W') {
decimal = -decimal;
}
return decimal;
}
function parseValue(valueStr) {
if (valueStr === undefined || valueStr === null) {
return { value: null, nd: null };
}
const ndMatch = String(valueStr).match(/ND\(([0-9.]+)\)/);
if (ndMatch) {
return {
value: null,
nd: parseFloat(ndMatch[1])
};
}
return {
value: parseFloat(valueStr),
nd: null
};
}
function findClosestStation(lon, lat, organization) {
let closestStation = null;
let minDistance = Infinity;
const toRadians = (deg) => (deg * Math.PI) / 180;
SeawaterSamplingLocations.forEach(station => {
if (station.Organization !== organization) return;
const stationLon = parseFloat(station.Longitude);
const stationLat = parseFloat(station.Latitude);
const R = 6371;
const dLat = toRadians(stationLat - lat);
const dLon = toRadians(stationLon - lon);
const a = Math.sin(dLat / 2) * Math.sin(dLat / 2) +
Math.cos(toRadians(lat)) * Math.cos(toRadians(stationLat)) *
Math.sin(dLon / 2) * Math.sin(dLon / 2);
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
const distance = R * c;
if (distance < minDistance) {
minDistance = distance;
closestStation = station.Station;
}
});
return closestStation;
}
function findMatchingStation(lon, lat, organization) {
const targetLon = parseFloat(lon).toFixed(8);
const targetLat = parseFloat(lat).toFixed(8);
const match = SeawaterSamplingLocations.find(station => {
return (
station.Organization === organization &&
parseFloat(station.Longitude).toFixed(8) === targetLon &&
parseFloat(station.Latitude).toFixed(8) === targetLat
);
});
return match ? match.Station : null;
}
const transformedData = [];
inputData.forEach(feature => {
const props = feature.properties;
const coords = feature.geometry.coordinates;
const lat = parseDMSCoordinate(props.lat);
const lon = parseDMSCoordinate(props.lon);
if (props.item === "Fishes") {
if (props.sample && Array.isArray(props.sample)) {
props.sample.forEach(sample => {
const baseEntry = {
begperiod: new Date(props.sampling_date),
point: coords,
org: props.organization,
species: sample.sample_name,
lon,
lat,
};
if (sample.data && Array.isArray(sample.data)) {
const measurementsByRadionuclide = {};
sample.data.forEach(measurement => {
if (!measurementsByRadionuclide[measurement.radionuclide]) {
measurementsByRadionuclide[measurement.radionuclide] = [];
}
measurementsByRadionuclide[measurement.radionuclide].push(measurement);
});
Object.entries(measurementsByRadionuclide).forEach(([radionuclide, measurements]) => {
const measurement = measurements.find(m => {
if (radionuclide.includes('TFWT')) {
return m.unit === 'Bq/L';
} else if (radionuclide.includes('OBT')) {
return m.unit === 'Bq/kg-fresh';
}
return true;
}) || measurements[0];
const { value: parsedValue, nd } = parseValue(measurement.value);
baseEntry[radionuclide] = parsedValue;
baseEntry[`${radionuclide}_nd`] = nd;
baseEntry[`${radionuclide}_unit`] = measurement.unit;
});
}
transformedData.push(baseEntry);
});
}
} else if (props.item === "Seawater") {
props.depth.forEach((depth, depthIndex) => {
props.date.forEach((date, dateIndex) => {
const baseEntry = {
begperiod: new Date(date),
station: findMatchingStation(coords[0], coords[1], props.organization) ||
findClosestStation(coords[0], coords[1], props.organization),
point: coords,
org: props.organization,
depth: depth.toLowerCase().includes("surface") ? "Surface" :
depth.toLowerCase().includes("bottom") ? "Bottom" : depth,
unit: props.unit[depthIndex],
lon,
lat,
};
props.nuclide.forEach((nuclide, nuclideIndex) => {
const value = props.value[depthIndex]?.[nuclideIndex];
const { value: parsedValue, nd } = parseValue(value);
baseEntry[nuclide] = parsedValue;
baseEntry[`${nuclide}_nd`] = nd;
});
transformedData.push(baseEntry);
});
});
}
});
return transformedData;
}
rawData
function transformMonitoringData2(inputData, SampleType) {
function parseDMSCoordinate(dmsStr) {
const regex = /(\d+)°(\d+)′([\d.]+)″([NSEWnsew])/;
const match = dmsStr.match(regex);
if (!match) return null;
const [_, degrees, minutes, seconds, direction] = match;
let decimal =
parseFloat(degrees) +
parseFloat(minutes) / 60 +
parseFloat(seconds) / 3600;
if (direction.toUpperCase() === "S" || direction.toUpperCase() === "W") {
decimal = -decimal;
}
return decimal;
}
function parseValue(valueStr) {
if (valueStr === undefined || valueStr === null) {
return { value: null, nd: null };
}
const ndMatch = String(valueStr).match(/ND\(([0-9.]+)\)/);
if (ndMatch) {
return {
value: null,
nd: parseFloat(ndMatch[1])
};
}
return {
value: parseFloat(valueStr),
nd: null
};
}
function findClosestStation(lon, lat, organization) {
let closestStation = null;
let minDistance = Infinity;
const toRadians = (deg) => (deg * Math.PI) / 180;
SeawaterSamplingLocations.forEach((station) => {
if (station.Organization !== organization) return;
const stationLon = parseFloat(station.Longitude);
const stationLat = parseFloat(station.Latitude);
const R = 6371;
const dLat = toRadians(stationLat - lat);
const dLon = toRadians(stationLon - lon);
const a =
Math.sin(dLat / 2) * Math.sin(dLat / 2) +
Math.cos(toRadians(lat)) *
Math.cos(toRadians(stationLat)) *
Math.sin(dLon / 2) *
Math.sin(dLon / 2);
const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
const distance = R * c;
if (distance < minDistance) {
minDistance = distance;
closestStation = station.Station;
}
});
return closestStation;
}
function findMatchingStation(lon, lat, organization) {
const targetLon = parseFloat(lon).toFixed(8);
const targetLat = parseFloat(lat).toFixed(8);
const match = SeawaterSamplingLocations.find((station) => {
return (
station.Organization === organization &&
parseFloat(station.Longitude).toFixed(8) === targetLon &&
parseFloat(station.Latitude).toFixed(8) === targetLat
);
});
return match ? match.Station : null;
}
const transformedData = [];
inputData.forEach((feature) => {
const props = feature.properties;
const coords = feature.geometry.coordinates;
const lat = parseDMSCoordinate(props.lat);
const lon = parseDMSCoordinate(props.lon);
if (props.item === "Fishes") {
if (props.sample && Array.isArray(props.sample)) {
props.sample.forEach((sample) => {
const baseEntry = {
begperiod: new Date(props.sampling_date),
point: coords,
org: props.organization,
species: sample.sample_name,
lon,
lat
};
if (sample.data && Array.isArray(sample.data)) {
const measurementsByRadionuclide = {};
sample.data.forEach((measurement) => {
if (!measurementsByRadionuclide[measurement.radionuclide]) {
measurementsByRadionuclide[measurement.radionuclide] = [];
}
measurementsByRadionuclide[measurement.radionuclide].push(
measurement
);
});
Object.entries(measurementsByRadionuclide).forEach(
([radionuclide, measurements]) => {
const measurement =
measurements.find((m) => {
if (radionuclide.includes("TFWT")) {
return m.unit === "Bq/L";
} else if (radionuclide.includes("OBT")) {
return m.unit === "Bq/kg-fresh";
}
return true;
}) || measurements[0];
const { value: parsedValue, nd } = parseValue(
measurement.value
);
baseEntry[radionuclide] = parsedValue;
baseEntry[`${radionuclide}_nd`] = nd;
baseEntry[`${radionuclide}_unit`] = measurement.unit;
}
);
}
transformedData.push(baseEntry);
});
}
} else if (props.item === "Seawater") {
props.depth.forEach((depth, depthIndex) => {
props.date.forEach((date, dateIndex) => {
const baseEntry = {
begperiod: new Date(date),
station:
findMatchingStation(coords[0], coords[1], props.organization) ||
findClosestStation(coords[0], coords[1], props.organization),
point: coords,
org: props.organization,
depth: depth.toLowerCase().includes("surface")
? "Surface"
: depth.toLowerCase().includes("bottom")
? "Bottom"
: depth,
unit: props.unit[depthIndex],
lon,
lat
};
props.nuclide.forEach((nuclide, nuclideIndex) => {
const value = props.value[depthIndex]?.[nuclideIndex];
const { value: parsedValue, nd } = parseValue(value);
baseEntry[nuclide] = parsedValue;
baseEntry[`${nuclide}_nd`] = nd;
});
transformedData.push(baseEntry);
});
});
}
});
return transformedData;
}
getUrls = (Agency, SampleType) => {
const baseUrl = "https://www.monitororbs.jp/en/data";
const agencyMap = {
"Fukushima Prefecture": "fukushima",
"Ministry of the Environment": "moe",
"TEPCO": "tepco",
"Nuclear Regulation Authority": "nra",
"Fisheries Agency": "fa"
};
if (Agency === "All") {
if (SampleType === "Seawater") {
return ["fukushima", "moe", "tepco", "nra"].map(
(agency) => `${baseUrl}/${SampleType}/${agency}.json`
);
}
if (SampleType === "Fish") {
return ["fukushima", "moe", "tepco", "nra", "fa"].map(
(agency) => `${baseUrl}/${SampleType}/${agency}_fish.json`
);
}
}
const agency = agencyMap[Agency];
if (!agency) return null;
const suffix = SampleType === "Fish" ? "_fish" : "";
return `${baseUrl}/${SampleType}/${agency}${suffix}.json`;
}
// Example of use
getUrls(Agency, SampleType)
fetchUrl = async (url) => {
try {
// Calculate timestamp for 3 years ago
const threeYearsAgo = new Date();
threeYearsAgo.setFullYear(threeYearsAgo.getFullYear() - 3);
const timestamp = threeYearsAgo.getTime();
const urlWith3Years = url + `?date=${timestamp}`;
const response = await fetchp(urlWith3Years);
return await response.json();
} catch (error) {
console.error(`Error fetching ${url}:`, error);
return null;
}
}
start
fetchUrl("https://www.monitororbs.jp/en/data/Seawater/fukushima.json")
fetchUrl2("https://www.monitororbs.jp/en/data/Seawater/fukushima.json", start)
fetchUrl2 = async (url, start) => {
try {
const urlWithStartDate = url + `?date=${start.getTime()}`;
const response = await fetchp(urlWithStartDate);
return await response.json();
} catch (error) {
console.error(`Error fetching ${url}:`, error);
return null;
}
}
turf = require("@turf/turf@7")
import { fetchp } from "@tomlarkworthy/fetchp"
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
