Design, tech, engineering and science. Head of Design and User Research.
Public
Edited
Sep 24, 2024
listings
Type Table, then Shift-Enter. Ctrl-space for more options.
prices = _.map(listingData, 'price')
areaPrices = _.map(listingData, 'areaPrice')
areas = _.map(listingData, 'area')
averagePrice = avg(prices)
averageArea = avg(areas)
averageAreaPrice = avg(areaPrices)
listings = listingData.map((listing) => {
return {
...listing,
avgPriceDiff: (listing.price - averagePrice) / listing.price,
avgAreaPriceDiff: (listing.areaPrice - averageAreaPrice) / listing.areaPrice,
avgAreaDiff: (listing.area - averageArea) / listing.area
};
})
listingData = await Promise.all(
normalizeStrings(urls).map(async (urlInput) => {
// Page fetching happens here
const qi = urlInput.indexOf("?");
const url = qi < 0 ? urlInput : urlInput.substring(0, qi);
const html = await fetchPage(url);
const page = parseHtml(html);
// Basic details
const id = url.replace("https://www.etuovi.com/kohde/", "");
const title = getRawText(page.querySelector("h1"));
// Parsing
const basicDetailsNodes = page.querySelector(
"#infos > div > div > div > :nth-child(2)"
).childNodes[0].childNodes;
const details = samplePage
.querySelectorAll(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > div > div > div + div"
)
.map(getRawText);
const [city, buildingType, shareType, description, type, floorString] = [
details[1],
details[2],
details[3],
details[4],
details[5],
details[8]
];
const [floor, floorMax] = floorString
? floorString.split("/").map(parseFloat).map(round)
: [1, 1];
const basicDetails = basicDetailsNodes.map(({ childNodes }) => {
return getRawText(childNodes[1].childNodes[0]);
});
const [priceString, areaString, yearString] = basicDetails;
const price = parseCurrencyString(priceString);
const area = parseAreaString(areaString);
const year = parseInt(yearString);
const taxPercentage = 0.015;
// const taxPercentage = _.includes(["Kerrostalo", "Rivitalo"], buildingType)
// ? 0.02
// : 0.04;
const tax = price * taxPercentage;
const areaPrice = price / area;
return {
// html,
// page,
id,
url,
title,
price,
areaPrice,
tax,
taxPercentage,
area,
year,
city,
buildingType,
shareType,
description,
type,
floor,
floorMax
};
})
)
samplePage = parseHtml(await fetchPage('https://www.etuovi.com/kohde/u66694?haku=M2011592486'))
floor = samplePage.querySelectorAll(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > div > div > div + div"
).map(getRawText)[8]
floor.split('/').map(parseFloat)
samplePage.querySelector(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > :nth-child(2) > div > a > img"
)
getRawText(samplePage.querySelector(
"#showings > div:nth-child(3) > div:nth-child(2)"
))
getRawText(samplePage.querySelector(
"#showings > div:nth-child(4) > div:nth-child(2)"
))
function parseCurrencyString(string) {
return parseFloat(
string.replace(/\s/g, "").replaceAll(",", ".").replaceAll("€", "")
);
}
function normalizeStrings (...items) {
return _.flatten(_.flattenDeep(items).map(splitString))
}
function splitString (string) {
const list = string.split(',')
return _.map(list, (s) => {
return s.trim()
})
}
function corsifyUrl(url) {
return `https://corsproxy.io/?${encodeURIComponent(url)}`;
}
async function fetchPage(url, noCors) {
const response = await fetch(noCors ? url : corsifyUrl(url));
return response.text()
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
