Public
Edited
Jan 13
viewof form = Inputs.form({
startDate: Inputs.text({type: "date", label: "Start Date"}),
endDate: Inputs.text({type: "date", label: "End Date"}),
})
viewof namespaceNameFilter = Inputs.text({label: "Name:"})
viewof namespaceInstitutionFilter = Inputs.text({label: "Institution:"})
viewof namespaceUserInstitutionFilter = Inputs.text({label: "User Institution:"})
filteredNamespaces = namespaces.filter((row) =>
(new RegExp(namespaceNameFilter)).test(row.Name) &&
(new RegExp(namespaceInstitutionFilter)).test(row.Institution) &&
(new RegExp(namespaceUserInstitutionFilter)).test(row.UserInstitutions)
)
viewof selectedNamespaces = Inputs.table(filteredNamespaces, {required: false, value: filteredNamespaces})
viewof nodeNameFilter = Inputs.text({label: "Name:"})
viewof nodeRegionFilter = Inputs.text({label: "Region:"})
viewof nodeZoneFilter = Inputs.text({label: "Zone:"})
viewof nodeGPUFilter = Inputs.text({label: "GPU:"})
viewof nodeFPGAFilter = Inputs.text({label: "FPGA:"})
viewof nodeTaintsFilter = Inputs.text({label: "Taints:"})
filteredNodes = nodes.filter((row) =>
(new RegExp(nodeNameFilter)).test(row.Name) &&
(new RegExp(nodeRegionFilter)).test(row.Region) &&
(new RegExp(nodeZoneFilter)).test(row.Zone) &&
(new RegExp(nodeGPUFilter)).test(row.GPUType) &&
(new RegExp(nodeFPGAFilter)).test(row.FPGAType) &&
(new RegExp(nodeTaintsFilter)).test(row.Taints)
)
viewof selectedNodes = Inputs.table(filteredNodes, {required: false, value: filteredNodes})
jayson = import("https://esm.sh/jayson")
dataStatus = {
return {
status: null,
error: ''
}
}
namespaces = new Promise((resolve, reject) => {
const client = jayson.client.https({
host: 'portal.nrp-nautilus.io',
port: 443,
path: '/rpc',
withCredentials: false,
})
client.request('admin.ListNsInfo', [], function(err, error, response) {
if(error) {
dataStatus.error = "Error retrieving namespace info"
reject(error)
} else if(err) {
dataStatus.error = "Error retrieving namespace info"
reject(err)
} else {
let namespaces = response['Namespaces']
let results = []
namespaces.forEach((namespace) => {
results.push({
Name: namespace['Name'],
PI: namespace['PI'],
Admins: Array(namespace['Admins']).join(),
Institution: namespace['Institution'],
UserInstitutions: Array(namespace['UserInstitutions']).join(),
Description: namespace['Description']
})
})
resolve(results)
}
})
})
nodes = new Promise((resolve, reject) => {
const client = jayson.client.https({
host: 'portal.nrp-nautilus.io',
port: 443,
path: '/rpc',
withCredentials: true,
})
client.request('guest.ListNodeInfo', [], function(err, error, response) {
if(error) {
dataStatus.error = "Error retrieving node info"
reject(error)
} else if(err) {
dataStatus.error = "Error retrieving node info"
reject(err)
} else {
let nodes = []
for (var node of response['Nodes']) {
nodes.push({
Name: node.Name,
Region: node.Region,
Zone: node.Zone,
Taints: node.Taints?.map(t => t.key + '=' + t.value)?.join(','),
GPUType: node.GPUType,
FPGAType: node.FPGAType
})
}
resolve(nodes)
}
})
})
filteredData = rawData.filter((data) => {
return selectedNamespaces.some((selectedNamespace) => selectedNamespace.Name === data.namespace)
}).filter((data) => {
return selectedNodes.some((selectedNode) => selectedNode.Name === data.node)
})
class NamespaceResource {
constructor(namespace, node, resource, value=0) {
this.namespace = namespace
this.node = node
this.resource = this.convertResource(resource)
this.value = this.convertValue(value)
}
convertResource (resource) {
if (resource?.includes("amd_com_xilinx")) return "fpga"
if (resource == "cpu") return "cpu"
if (resource?.includes("nvidia_com")) return "gpu"
if (resource == "memory") return "memory"
if (resource == "ephemeral_storage") return "storage"
return "other"
}
add(value) {
this.value += this.convertValue(value)
}
convertValue(value) {
return (this.resource == 'memory' || this.resource == 'storage') ? parseFloat(value) / 1000000000 : parseFloat(value)
}
}
async function queryPrometheus(query) {
var encodedUrl = "https://thanos.nrp-nautilus.io/api/v1/query?query=" + encodeURIComponent(query)
return new Promise((resolve, reject) => {
fetch(encodedUrl).then((response) => {
if (response.ok) {
response.json().then((json) => {
resolve(json.data.result)
})
} else {
var errorMessage = `Invalid response: Status ${response.status} ${response.statusText}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
}
}).catch((error) => {
var errorMessage = `Networking error: ${error}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
})
})
}
sortedData = rawData.sort((a,b) => a.namespace > b.namespace)
rawData = {
class NamespaceResource {
constructor(namespace, node, resource, value=0) {
this.namespace = namespace
this.node = node
this.resource = this.convertResource(resource)
this.value = this.convertValue(value)
}
convertResource (resource) {
if (resource?.includes("amd_com_xilinx")) return "fpga"
if (resource == "cpu") return "cpu"
if (resource?.includes("nvidia_com")) return "gpu"
if (resource == "memory") return "memory"
if (resource == "ephemeral_storage") return "storage"
return "other"
}
add(value) {
this.value += this.convertValue(value)
}
convertValue(value) {
return (this.resource == 'memory' || this.resource == 'storage') ? parseFloat(value) / 1000000000 : parseFloat(value)
}
}
async function queryPrometheus(query) {
var encodedUrl = "https://thanos.nrp-nautilus.io/api/v1/query?query=" + encodeURIComponent(query)
return new Promise((resolve, reject) => {
fetch(encodedUrl).then((response) => {
if (response.ok) {
response.json().then((json) => {
resolve(json.data.result)
})
} else {
var errorMessage = `Invalid response: Status ${response.status} ${response.statusText}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
}
}).catch((error) => {
var errorMessage = `Networking error: ${error}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
})
})
}
// Reset Status
dataStatus.status = null
dataStatus.error = ''
var output = [{None: null}]
// Input Validation
if (form.startDate == "") {
dataStatus.error = "Missing Input: Start Date"
return output
}
if (form.endDate == "") {
dataStatus.error = "Missing Input: End Date"
return output
}
var today = new Date()
var maxDate = new Date('09/15/23')
var startDate = new Date(form.startDate)
var endDate = new Date(form.endDate)
if (startDate > today || endDate > today || startDate >= endDate || startDate < maxDate) {
dataStatus.status = "failure"
dataStatus.error = "Invalid Date(s)"
return output
}
dataStatus.status = 'pending'
const moment = await require('moment')
var days = moment(endDate).diff(startDate, "days")
var dateRanges = []
for(var weeks = days == 7 ? 0 : Math.floor(days/7); weeks >= 0; weeks-- ) {
var searchDays = days - 7*weeks >= 7 || days - 7*weeks == 0 ? 7 : days - 7*weeks
var searchEnd = new Date()
searchEnd.setTime(endDate.getTime() - 1000*60*60*24*7*weeks)
searchEnd = weeks == Math.floor(days/7) ? moment(searchEnd).unix() : moment(searchEnd).unix() + 1 // Don't include overlapping timestamps
dateRanges.push([searchDays, searchEnd])
}
var allocatedQueries = []
for (const [searchDays, searchEnd] of dateRanges) {
allocatedQueries.push(queryPrometheus(`sum_over_time(namespace_allocated_resources[${searchDays}d:1h]@${searchEnd})`))
}
var data = []
var isMissingData = false
await Promise.allSettled(allocatedQueries).then((queries) => {
for (const query of queries) {
if (query.value.length == 0) {
isMissingData = true
} else {
for (const value of query.value) {
var namespace = value['metric']['namespace']
var node = value['metric']['node']
var resource = value['metric']['resource']
var allocatedValue = value['value'][1]
if(namespace && node && resource) {
var dataRow = data.find((row) => row.namespace == namespace && row.node == node && row.resource == row.convertResource(resource))
if (dataRow) {
dataRow.add(allocatedValue)
}
else {
var namespaceResource = new NamespaceResource(namespace, node, resource, allocatedValue)
data.push(namespaceResource)
}
}
}
}
}
})
if (isMissingData) {
dataStatus.status = "failure"
dataStatus.error = `Missing data, partial query failure`
} else {
dataStatus.status = "success"
}
return data
}
tick = {
while(true) {
yield new Promise(resolve => setTimeout(resolve, 500))
}
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.