Design, tech, engineering and science. Head of Design and User Research.
Public
Edited
Sep 24, 2024
listings
Type Table, then Shift-Enter. Ctrl-space for more options.
prices = _.map(listingData, 'price')
areaPrices = _.map(listingData, 'areaPrice')
areas = _.map(listingData, 'area')
averagePrice = avg(prices)
averageArea = avg(areas)
averageAreaPrice = avg(areaPrices)
listings = listingData.map((listing) => {
return {
...listing,
avgPriceDiff: (listing.price - averagePrice) / listing.price,
avgAreaPriceDiff: (listing.areaPrice - averageAreaPrice) / listing.areaPrice,
avgAreaDiff: (listing.area - averageArea) / listing.area
};
})
listingData = await Promise.all(
normalizeStrings(urls).map(async (urlInput) => {
// Page fetching happens here
const qi = urlInput.indexOf("?");
const url = qi < 0 ? urlInput : urlInput.substring(0, qi);
const html = await fetchPage(url);
const page = parseHtml(html);
// Basic details
const id = url.replace("https://www.etuovi.com/kohde/", "");
const title = getRawText(page.querySelector("h1"));
// Parsing
const basicDetailsNodes = page.querySelector(
"#infos > div > div > div > :nth-child(2)"
).childNodes[0].childNodes;
const details = samplePage
.querySelectorAll(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > div > div > div + div"
)
.map(getRawText);
const [city, buildingType, shareType, description, type, floorString] = [
details[1],
details[2],
details[3],
details[4],
details[5],
details[8]
];
const [floor, floorMax] = floorString
? floorString.split("/").map(parseFloat).map(round)
: [1, 1];
const basicDetails = basicDetailsNodes.map(({ childNodes }) => {
return getRawText(childNodes[1].childNodes[0]);
});
const [priceString, areaString, yearString] = basicDetails;
const price = parseCurrencyString(priceString);
const area = parseAreaString(areaString);
const year = parseInt(yearString);
const taxPercentage = 0.015;
// const taxPercentage = _.includes(["Kerrostalo", "Rivitalo"], buildingType)
// ? 0.02
// : 0.04;
const tax = price * taxPercentage;
const areaPrice = price / area;
return {
// html,
// page,
id,
url,
title,
price,
areaPrice,
tax,
taxPercentage,
area,
year,
city,
buildingType,
shareType,
description,
type,
floor,
floorMax
};
})
)
samplePage = parseHtml(await fetchPage('https://www.etuovi.com/kohde/u66694?haku=M2011592486'))
floor = samplePage.querySelectorAll(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > div > div > div + div"
).map(getRawText)[8]
floor.split('/').map(parseFloat)
samplePage.querySelector(
"#showings > div:nth-child(2) > div:nth-child(2) > div > div > div > div > :nth-child(2) > div > a > img"
)
getRawText(samplePage.querySelector(
"#showings > div:nth-child(3) > div:nth-child(2)"
))
getRawText(samplePage.querySelector(
"#showings > div:nth-child(4) > div:nth-child(2)"
))
function parseCurrencyString(string) {
return parseFloat(
string.replace(/\s/g, "").replaceAll(",", ".").replaceAll("€", "")
);
}
function normalizeStrings (...items) {
return _.flatten(_.flattenDeep(items).map(splitString))
}
function splitString (string) {
const list = string.split(',')
return _.map(list, (s) => {
return s.trim()
})
}
function corsifyUrl(url) {
return `https://corsproxy.io/?${encodeURIComponent(url)}`;
}
async function fetchPage(url, noCors) {
const response = await fetch(noCors ? url : corsifyUrl(url));
return response.text()
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.