chinaIpRecords = Object.entries(latestIpsGeoLite2Report.ipsGeoLite2).map(([ip, record]) => ({ ip, ...record })).filter(({ country }) => country === 'CN')
latestIpsBaiduReportUrl = `https://geoip.feeds.provider.quest/ips-baidu-latest.json`
latestIpsBaiduReport = (await fetch(latestIpsBaiduReportUrl)).json()
missingChinaIpRecords = chinaIpRecords.filter(({ ip: ipAddress }) => {
if (!ip.isV4Format(ipAddress)) {
return false
}
const baidu = latestIpsBaiduReport.ipsBaidu[ipAddress]
if (baidu) {
if (baidu.baidu && baidu.baidu.status !== 0) {
if (baidu.baidu.message.match(/loc failed/)) return false
return true
}
return false
}
return true
})
geoIpBaiduKey = 'jim' //Secret("geoIpBaiduKey")
geoIpBaiduSecret = 'baidu123' // Secret("geoIpBaiduSecret")
basicAuthToken = base64.encode(`${geoIpBaiduKey}:${geoIpBaiduSecret}`)
newIps = d3.shuffle(missingChinaIpRecords.map(({ ip }) => ip).filter(ipAddress => ip.isV4Format(ipAddress)))
async function* lookupIpsStream() {
const concurrency = 1
const callGeoLookupsStream = transform(concurrency, async ip => {
const baiduApiUrl = `${geoApiBaseUrl}/baidu/${ip}`
try {
const baidu = await (await fetch(
baiduApiUrl,
{ headers: new Headers({ "Authorization": `Basic ${basicAuthToken}` }) }
)).json()
const result = {
ip,
baidu
}
if (baidu.content) {
if (baidu.content.address_detail) {
const cityCode = baidu.content.address_detail.city_code
if (baiduCities[cityCode]) {
result.city = baiduCities[cityCode]
}
}
if (baidu.content.point) {
result.long = Number(baidu.content.point.x)
result.lat = Number(baidu.content.point.y)
}
}
return result
} catch (e) {
console.info('IP lookup error', ip, e.message)
return {
error: `${e.message}, ${baiduApiUrl}`
}
}
})
const startTime = new Date()
let counter = 0
let hits = 0
let errors = 0
let lastError = ''
for await (const geoLookup of callGeoLookupsStream([...newIps])) {
const now = new Date()
if (now - startTime > maxElapsed || counter >= maxLookups) {
yield {
done: true,
timeout: true,
counter,
hits,
errors,
lastError
}
return
}
if (geoLookup.ip) {
hits++
yield {
counter,
hits,
errors,
lastError,
...geoLookup
}
} else {
errors++
if (geoLookup.error) {
lastError = geoLookup.error
}
yield {
counter,
hits,
errors,
lastError
}
}
counter++
}
yield {
done: true,
counter,
hits,
errors,
lastError
}
}
// lookupIpsStream()
ipsBaidu = {
if (start === 0) {
yield {
state: 'paused'
}
return
}
yield {
state: 'starting'
}
let records = []
let totalErrors = 0
let lastCounter = 0
let lastError = ''
const startTime = new Date()
for await (const {counter, hits, errors, ...record} of lookupIpsStream()) {
totalErrors = errors
lastCounter = counter
lastError = record.lastError
if (record.ip) {
records.push(record)
}
yield {
state: "streaming",
elapsed: ((new Date()) - startTime) / 1000,
scannedIps: counter,
totalIps: ipsCount,
recordsLength: records.length,
errors,
lastError
}
}
const endTime = new Date()
yield {
state: "done",
elapsed: (endTime - startTime) / 1000,
scannedIps: lastCounter,
totalIps: ipsCount,
records: records.sort(),
startTime,
endTime,
errors: totalErrors,
lastError
}
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
