The avatar for @nrp-nautilus
NRP Nautilus
Public
Edited
Jan 13
Insert cell
Insert cell
viewof form = Inputs.form({
startDate: Inputs.text({type: "date", label: "Start Date"}),
endDate: Inputs.text({type: "date", label: "End Date"}),
})
Insert cell
Insert cell
Insert cell
viewof namespaceNameFilter = Inputs.text({label: "Name:"})
Insert cell
viewof namespaceInstitutionFilter = Inputs.text({label: "Institution:"})
Insert cell
viewof namespaceUserInstitutionFilter = Inputs.text({label: "User Institution:"})
Insert cell
filteredNamespaces = namespaces.filter((row) =>
(new RegExp(namespaceNameFilter)).test(row.Name) &&
(new RegExp(namespaceInstitutionFilter)).test(row.Institution) &&
(new RegExp(namespaceUserInstitutionFilter)).test(row.UserInstitutions)
)
Insert cell
viewof selectedNamespaces = Inputs.table(filteredNamespaces, {required: false, value: filteredNamespaces})
Insert cell
Insert cell
viewof nodeNameFilter = Inputs.text({label: "Name:"})
Insert cell
viewof nodeRegionFilter = Inputs.text({label: "Region:"})
Insert cell
viewof nodeZoneFilter = Inputs.text({label: "Zone:"})
Insert cell
viewof nodeGPUFilter = Inputs.text({label: "GPU:"})
Insert cell
viewof nodeFPGAFilter = Inputs.text({label: "FPGA:"})
Insert cell
viewof nodeTaintsFilter = Inputs.text({label: "Taints:"})
Insert cell
filteredNodes = nodes.filter((row) =>
(new RegExp(nodeNameFilter)).test(row.Name) &&
(new RegExp(nodeRegionFilter)).test(row.Region) &&
(new RegExp(nodeZoneFilter)).test(row.Zone) &&
(new RegExp(nodeGPUFilter)).test(row.GPUType) &&
(new RegExp(nodeFPGAFilter)).test(row.FPGAType) &&
(new RegExp(nodeTaintsFilter)).test(row.Taints)
)
Insert cell
viewof selectedNodes = Inputs.table(filteredNodes, {required: false, value: filteredNodes})
Insert cell
jayson = import("https://esm.sh/jayson")
Insert cell
dataStatus = {
return {
status: null,
error: ''
}
}
Insert cell
namespaces = new Promise((resolve, reject) => {
const client = jayson.client.https({
host: 'portal.nrp-nautilus.io',
port: 443,
path: '/rpc',
withCredentials: false,
})
client.request('admin.ListNsInfo', [], function(err, error, response) {
if(error) {
dataStatus.error = "Error retrieving namespace info"
reject(error)
} else if(err) {
dataStatus.error = "Error retrieving namespace info"
reject(err)
} else {
let namespaces = response['Namespaces']
let results = []
namespaces.forEach((namespace) => {
results.push({
Name: namespace['Name'],
PI: namespace['PI'],
Admins: Array(namespace['Admins']).join(),
Institution: namespace['Institution'],
UserInstitutions: Array(namespace['UserInstitutions']).join(),
Description: namespace['Description']
})
})
resolve(results)
}
})
})
Insert cell
nodes = new Promise((resolve, reject) => {
const client = jayson.client.https({
host: 'portal.nrp-nautilus.io',
port: 443,
path: '/rpc',
withCredentials: true,
})
client.request('guest.ListNodeInfo', [], function(err, error, response) {
if(error) {
dataStatus.error = "Error retrieving node info"
reject(error)
} else if(err) {
dataStatus.error = "Error retrieving node info"
reject(err)
} else {
let nodes = []
for (var node of response['Nodes']) {
nodes.push({
Name: node.Name,
Region: node.Region,
Zone: node.Zone,
Taints: node.Taints?.map(t => t.key + '=' + t.value)?.join(','),
GPUType: node.GPUType,
FPGAType: node.FPGAType
})
}
resolve(nodes)
}
})
})
Insert cell
filteredData = rawData.filter((data) => {
return selectedNamespaces.some((selectedNamespace) => selectedNamespace.Name === data.namespace)
}).filter((data) => {
return selectedNodes.some((selectedNode) => selectedNode.Name === data.node)
})
Insert cell
class NamespaceResource {
constructor(namespace, node, resource, value=0) {
this.namespace = namespace
this.node = node
this.resource = this.convertResource(resource)
this.value = this.convertValue(value)
}
convertResource (resource) {
if (resource?.includes("amd_com_xilinx")) return "fpga"
if (resource == "cpu") return "cpu"
if (resource?.includes("nvidia_com")) return "gpu"
if (resource == "memory") return "memory"
if (resource == "ephemeral_storage") return "storage"
return "other"
}
add(value) {
this.value += this.convertValue(value)
}
convertValue(value) {
return (this.resource == 'memory' || this.resource == 'storage') ? parseFloat(value) / 1000000000 : parseFloat(value)
}
}
Insert cell
async function queryPrometheus(query) {
var encodedUrl = "https://thanos.nrp-nautilus.io/api/v1/query?query=" + encodeURIComponent(query)
return new Promise((resolve, reject) => {
fetch(encodedUrl).then((response) => {
if (response.ok) {
response.json().then((json) => {
resolve(json.data.result)
})
} else {
var errorMessage = `Invalid response: Status ${response.status} ${response.statusText}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
}
}).catch((error) => {
var errorMessage = `Networking error: ${error}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
})
})
}
Insert cell
sortedData = rawData.sort((a,b) => a.namespace > b.namespace)
Insert cell
rawData = {
class NamespaceResource {
constructor(namespace, node, resource, value=0) {
this.namespace = namespace
this.node = node
this.resource = this.convertResource(resource)
this.value = this.convertValue(value)
}
convertResource (resource) {
if (resource?.includes("amd_com_xilinx")) return "fpga"
if (resource == "cpu") return "cpu"
if (resource?.includes("nvidia_com")) return "gpu"
if (resource == "memory") return "memory"
if (resource == "ephemeral_storage") return "storage"
return "other"
}
add(value) {
this.value += this.convertValue(value)
}
convertValue(value) {
return (this.resource == 'memory' || this.resource == 'storage') ? parseFloat(value) / 1000000000 : parseFloat(value)
}
}
async function queryPrometheus(query) {
var encodedUrl = "https://thanos.nrp-nautilus.io/api/v1/query?query=" + encodeURIComponent(query)
return new Promise((resolve, reject) => {
fetch(encodedUrl).then((response) => {
if (response.ok) {
response.json().then((json) => {
resolve(json.data.result)
})
} else {
var errorMessage = `Invalid response: Status ${response.status} ${response.statusText}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
}
}).catch((error) => {
var errorMessage = `Networking error: ${error}`
dataStatus.status = "failure"
dataStatus.error = errorMessage
reject(errorMessage)
})
})
}
// Reset Status
dataStatus.status = null
dataStatus.error = ''
var output = [{None: null}]

// Input Validation
if (form.startDate == "") {
dataStatus.error = "Missing Input: Start Date"
return output
}
if (form.endDate == "") {
dataStatus.error = "Missing Input: End Date"
return output
}

var today = new Date()
var maxDate = new Date('09/15/23')
var startDate = new Date(form.startDate)
var endDate = new Date(form.endDate)
if (startDate > today || endDate > today || startDate >= endDate || startDate < maxDate) {
dataStatus.status = "failure"
dataStatus.error = "Invalid Date(s)"
return output
}

dataStatus.status = 'pending'
const moment = await require('moment')
var days = moment(endDate).diff(startDate, "days")
var dateRanges = []
for(var weeks = days == 7 ? 0 : Math.floor(days/7); weeks >= 0; weeks-- ) {
var searchDays = days - 7*weeks >= 7 || days - 7*weeks == 0 ? 7 : days - 7*weeks
var searchEnd = new Date()
searchEnd.setTime(endDate.getTime() - 1000*60*60*24*7*weeks)
searchEnd = weeks == Math.floor(days/7) ? moment(searchEnd).unix() : moment(searchEnd).unix() + 1 // Don't include overlapping timestamps
dateRanges.push([searchDays, searchEnd])
}

var allocatedQueries = []
for (const [searchDays, searchEnd] of dateRanges) {
allocatedQueries.push(queryPrometheus(`sum_over_time(namespace_allocated_resources[${searchDays}d:1h]@${searchEnd})`))
}
var data = []
var isMissingData = false
await Promise.allSettled(allocatedQueries).then((queries) => {
for (const query of queries) {
if (query.value.length == 0) {
isMissingData = true
} else {
for (const value of query.value) {
var namespace = value['metric']['namespace']
var node = value['metric']['node']
var resource = value['metric']['resource']
var allocatedValue = value['value'][1]
if(namespace && node && resource) {
var dataRow = data.find((row) => row.namespace == namespace && row.node == node && row.resource == row.convertResource(resource))
if (dataRow) {
dataRow.add(allocatedValue)
}
else {
var namespaceResource = new NamespaceResource(namespace, node, resource, allocatedValue)
data.push(namespaceResource)
}
}
}
}
}
})
if (isMissingData) {
dataStatus.status = "failure"
dataStatus.error = `Missing data, partial query failure`
} else {
dataStatus.status = "success"
}

return data
}
Insert cell
tick = {
while(true) {
yield new Promise(resolve => setTimeout(resolve, 500))
}
}
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more