broken account, please ignore - migrating
Public
Edited
Mar 17, 2023
1 fork
1 star
{
playKidsInAmerica();
}
axios = require("axios@0.18.0/dist/axios.js")
util = {
const API_ROOT = "https://api.datacommons.org";
const API_ENDPOINTS = {
query: "/query",
get_property_labels: "/node/property-labels",
get_property_values: "/node/property-values",
get_triples: "/node/triples",
get_places_in: "/node/places-in",
get_related_places: "/node/related-places",
get_stats: "/bulk/stats",
get_stat_value: "/stat/value",
get_stat_series: "/stat/series",
get_stat_all: "/stat/all"
};
const MAX_LIMIT = 100;
const QUERY_BATCH_SIZE = 500;
async function _post(path, data = {}) {
const url = API_ROOT + path;
console.log(
`_post POST to: ${path} with data: ${JSON.stringify(
data
)} url is: ${url}`,
data
);
const headers = { "Content-Type": "application/json" };
const apiKey = "AIzaSyCTI4Xz-UW_G2Q2RfknhcfdAnTHq5X5XuI"; // default public limited
if (apiKey) {
headers["x-api-key"] = apiKey;
}
try {
const resp = await axios({
url: url,
method: "post",
data: data,
headers: headers
});
console.log(`_post resp was: ${resp}`);
if (resp.status !== 200) {
throw new Error(
`${resp.status}: +++ ${resp.statusText}\n${resp.data.message}`
);
}
return resp.data;
} catch (error) {
if (error.response) {
throw new Error(
`${error.response.status}: --- ${error.response.statusText}\n${error.response.data.message}`
);
} else {
throw error;
}
}
}
async function sendRequest(url, data) {
console.log(`sr ${url}`);
console.log(`${JSON.stringify(data)}`);
const headers = { "Content-Type": "application/json" };
const apiKey = "AIzaSyCTI4Xz-UW_G2Q2RfknhcfdAnTHq5X5XuI"; // default public limited
if (apiKey) {
headers["x-api-key"] = apiKey;
}
try {
const resp = await axios({
url: url,
method: "post",
data: data,
headers: headers
});
if (resp.status !== 200) {
throw new Error(
`${resp.status}: +++ ${resp.statusText}\n${resp.data.message}`
);
}
return resp.data;
} catch (error) {
if (error.response) {
throw new Error(
`${error.response.status}: --- ${error.response.statusText}\n${error.response.data.message}`
);
} else {
throw error;
}
}
}
/* PYTHON WAS: def _send_request(req_url,
req_json={},
compress=False,
post=True,
use_payload=True):
""" Sends a POST/GET request to req_url with req_json, default to POST.
Returns:
The payload returned by sending the POST/GET request formatted as a dict.
"""
headers = {'Content-Type': 'application/json'} */
function _getDirection(isOut) {
return isOut ? "out" : "in";
}
async function properties(nodes, isOut = true) {
const resp = await _post(`/v1/bulk/properties/${_getDirection(isOut)}`, {
nodes
});
const result = {};
for (const item of resp.data || []) {
const node = item.node;
const properties = item.properties || [];
result[node] = properties;
}
return result;
}
async function propertyValues(nodes, property, isOut = true) {
const resp = await _post(
`/v1/bulk/property/values/${_getDirection(isOut)}`,
{ nodes, property }
);
const result = {};
for (const item of resp.data || []) {
const node = item.node;
const values = item.values || [];
result[node] = [];
for (const v of values) {
if ("dcid" in v) {
result[node].push(v.dcid);
} else {
result[node].push(v.value);
}
}
}
return result;
}
async function triples(nodes, isOut = true) {
const resp = await _post(`/v1/bulk/triples/${_getDirection(isOut)}`, {
nodes
});
const result = {};
for (const item of resp.data || []) {
const node = item.node;
const triples = item.triples || {};
result[node] = {};
for (const [property, otherNodes] of Object.entries(triples)) {
result[node][property] = otherNodes.nodes || [];
}
}
return result;
}
// part 2
async function getPropertyLabels(dcids, out = true) {
// Filter out NaN values and generate the request payload
dcids = dcids.filter((v) => v === v);
const url = API_ROOT + API_ENDPOINTS.get_property_labels;
const payload = await sendRequest(url, { dcids });
// Return the results based on the orientation
const results = {};
for (const dcid of dcids) {
if (out) {
results[dcid] = payload[dcid].outLabels;
} else {
results[dcid] = payload[dcid].inLabels;
}
}
return results;
}
async function getPropertyValues(
dcids,
prop,
out = true,
valueType = null,
limit = MAX_LIMIT
) {
// Filter out NaN values and prepare the request payload
dcids = dcids.filter((v) => v === v);
const direction = out ? "out" : "in";
const reqJson = {
dcids,
property: prop,
limit,
direction
};
if (valueType) {
reqJson.value_type = valueType;
}
// Send the request
const url = API_ROOT + API_ENDPOINTS.get_property_values;
const payload = await sendRequest(url, reqJson);
// Create the result format for when dcids is provided as a list
const uniqueResults = new Map();
for (const dcid of dcids) {
const nodes = out ? payload[dcid].out || [] : payload[dcid].in || [];
for (const node of nodes) {
if (!uniqueResults.has(dcid)) {
uniqueResults.set(dcid, new Set());
}
if ("dcid" in node) {
uniqueResults.get(dcid).add(node.dcid);
} else if ("value" in node) {
uniqueResults.get(dcid).add(node.value);
}
}
}
// Make sure each dcid is in the results map, and convert all sets to arrays
const results = {};
for (const dcid of dcids) {
results[dcid] = Array.from(uniqueResults.get(dcid) || []);
}
return results;
}
async function getTriples(dcids, limit = MAX_LIMIT) {
// Filter out NaN values and generate the request payload
dcids = dcids.filter((v) => v === v);
const url = API_ROOT + API_ENDPOINTS.get_triples;
const payload = await sendRequest(url, { dcids, limit });
// Create a map from dcid to list of triples
const results = new Map();
for (const dcid of dcids) {
results.set(dcid, []);
for (const t of payload[dcid]) {
let triple;
if ("objectId" in t) {
triple = [t.subjectId, t.predicate, t.objectId];
} else if ("objectValue" in t) {
triple = [t.subjectId, t.predicate, t.objectValue];
}
results.get(dcid).push(triple);
}
}
return Object.fromEntries(results);
}
/* module.exports = {
getPropertyLabels,
getPropertyValues,
getTriples,
};*/
return {
_post,
_getDirection,
properties,
propertyValues,
getTriples,
getPropertyLabels,
getPropertyValues
};
}
import { logTail } from "@sorig/console-log-without-leaving-your-notebook"
logTail(10)
tester = {
const dcids = ["geoId/06085", "dc/p/zsb968m3v1f97"];
// Print all incoming and outgoing properties from Santa Clara County.
// print('Property Labels for Santa Clara County')
const in_labels = await util.getPropertyLabels(dcids);
/* {
"code": 3,
"message": "Missing required arguments: dcids",
"details": [
{
"@type": "type.googleapis.com/google.rpc.DebugInfo",
"stackEntries": [],
"detail": "internal"
}
]
} */
const out_labels = await util.getPropertyLabels(dcids, { out: false });
console.log(`> Printing properties for ${dcids}`);
console.log(`> Incoming properties: ${in_labels}`);
console.log(`> Outgoing properties: ${out_labels}`);
//Print all property values for "containedInPlace" for Santa Clara County.
print('Property Values for "containedInPlace" of Santa Clara County');
const prop_vals = util.getPropertyValues(dcids, "containedInPlace", {
out: false,
value_type: "City"
});
console.log(
`prop_vals for containedInPlace Santa Clara County is: ${prop_vals}`
);
}
(
await fetch(
"https://api.datacommons.org/v1/property/values/out/geoId/sch3620580/name?key=AIzaSyCTI4Xz-UW_G2Q2RfknhcfdAnTHq5X5XuI"
)
).json()
await fetch(
"https://api.datacommons.org/v1/property/values/in/country/MDG/affectedPlace?key=AIzaSyCTI4Xz-UW_G2Q2RfknhcfdAnTHq5X5XuI"
).json()
d = {
const u =
"https://api.datacommons.org/v1/property/values/in/country/MDG/affectedPlace?key=AIzaSyCTI4Xz-UW_G2Q2RfknhcfdAnTHq5X5XuI";
const d = await fetch(u);
yield d;
}
d.json()
dot`
digraph G {
rankdir=TB;
node [shape=box, style=filled, fontsize=10];
// Input Layer
person1 [label="Person 1\nInput (24)"];
relationship [label="Relationship\nInput (12)"];
// Layer 2
person1_l2 [label="Dense (6)\nReLU"];
relationship_l2 [label="Dense (6)\nReLU"];
// Layer 3 (Central Layer)
central [label="Dense (12)\nReLU"];
// Layer 4 (Penultimate Layer)
penultimate [label="Dense (6)\nReLU"];
// Output Layer
output [label="Output\nDense (24)\nSoftmax"];
// Connections
person1 -> person1_l2;
relationship -> relationship_l2;
person1_l2 -> central [label="Concatenate"];
relationship_l2 -> central [label="Concatenate"];
central -> penultimate;
penultimate -> output;
}
`
tf = require("@tensorflow/tfjs@latest")
model = {
// Define the input layers
const person1Input = tf.input({ shape: [24], name: "person1_input" });
const relationshipInput = tf.input({
shape: [12],
name: "relationship_input"
});
// Define Layer 2
const person1Layer2 = tf.layers
.dense({ units: 6, activation: "relu" })
.apply(person1Input);
const relationshipLayer2 = tf.layers
.dense({ units: 6, activation: "relu" })
.apply(relationshipInput);
// Define Layer 3 (Central layer)
const concatenated = tf.layers
.concatenate()
.apply([person1Layer2, relationshipLayer2]);
const centralLayer = tf.layers
.dense({ units: 12, activation: "relu" })
.apply(concatenated);
// Define Layer 4 (Penultimate layer)
const penultimateLayer = tf.layers
.dense({ units: 6, activation: "relu" })
.apply(centralLayer);
// Define the output layer
const output = tf.layers
.dense({ units: 24, activation: "softmax" })
.apply(penultimateLayer);
// Create the model
const model = tf.model({
inputs: [person1Input, relationshipInput],
outputs: output
});
// Compile the model
model.compile({
optimizer: "adam",
loss: "categoricalCrossentropy",
metrics: ["accuracy"]
});
yield model;
/* Make sure to preprocess your data to match the input and output shapes, and replace the Keras-specific imports and functions with their TensorFlow.js equivalents. Train the model using your encoded input data for person 1 and relationship, and the corresponding person 2 output data.*/
}
data = {
// Here is a compact JavaScript representation of the 52 facts about the English and Italian families using the provided integer mappings:
const personMapping = {
0: "Christopher",
1: "Penelope",
2: "Andrew",
3: "Christine",
4: "Victoria",
5: "James",
6: "Colin",
7: "Charlotte",
8: "Margaret",
9: "Arthur",
10: "Jennifer",
11: "Charles",
12: "Roberto",
13: "Maria",
14: "Pierro",
15: "Francesca",
16: "Lucia",
17: "Marco",
18: "Sophia",
19: "Alfonso",
20: "Gina",
21: "Emilio",
22: "Angela",
23: "Tomaso"
};
const relationshipMapping = {
0: "father",
1: "mother",
2: "husband",
3: "wife",
4: "son",
5: "daughter",
6: "uncle",
7: "aunt",
8: "brother",
9: "sister",
10: "nephew",
11: "niece"
};
const facts = [
[0, 0, 4],
[0, 0, 8],
[1, 1, 4],
[1, 1, 8],
[2, 0, 5],
[2, 0, 10],
[3, 1, 5],
[3, 1, 10],
[4, 3, 5],
[5, 2, 4],
[8, 3, 9],
[9, 2, 8],
[10, 3, 11],
[11, 2, 10],
[5, 0, 6],
[5, 0, 7],
[4, 1, 6],
[4, 1, 7],
[6, 8, 7],
[7, 9, 6],
[8, 7, 6],
[8, 7, 7],
[9, 6, 6],
[9, 6, 7],
[10, 7, 6],
[10, 7, 7],
[11, 6, 6],
[11, 6, 7],
[12, 0, 16],
[12, 0, 20],
[13, 1, 16],
[13, 1, 20],
[14, 0, 18],
[14, 0, 22],
[15, 1, 18],
[15, 1, 22],
[16, 3, 18],
[18, 2, 16],
[20, 3, 21],
[21, 2, 20],
[22, 3, 23],
[23, 2, 22],
[18, 0, 17],
[18, 0, 19],
[16, 1, 17],
[16, 1, 19],
[17, 8, 19],
[19, 9, 17],
[20, 7, 17],
[20, 7, 19],
[21, 6, 17],
[21, 6, 19],
[22, 7, 17],
[22, 7, 19],
[23, 6, 17],
[23, 6, 19]
];
function getFactString(fact) {
const person1 = personMapping[fact[0]];
const relationship = relationshipMapping[fact[1]];
const person2 = personMapping[fact[2]];
return `${person1} is the ${relationship} of ${person2}.`;
}
yield { facts, getFactString };
}
data.facts.forEach((fact) => {
console.log(data.getFactString(fact));
})
logTail(52)
todo={
```javascript
import * as tf from '@tensorflow/tfjs';
// One-hot encode the data
function oneHotEncode(facts, numPersons, numRelationships) {
const encodedFacts = facts.map(([p1, r, p2]) => {
const p1Encoding = Array(numPersons).fill(0);
const rEncoding = Array(numRelationships).fill(0);
const p2Encoding = Array(numPersons).fill(0);
p1Encoding[p1] = 1;
rEncoding[r] = 1;
p2Encoding[p2] = 1;
return {
person1Input: p1Encoding,
relationshipInput: rEncoding,
output: p2Encoding,
};
});
return encodedFacts;
}
// Split the data into training and testing sets
function splitData(data, testSplit) {
const trainData = [];
const testData = [];
data.forEach(item => {
if (Math.random() < testSplit) {
testData.push(item);
} else {
trainData.push(item);
}
});
return { trainData, testData };
}
// Create the model
function createModel() {
const person1Input = tf.input({ shape: [24], name: 'person1_input' });
const relationshipInput = tf.input({ shape: [12], name: 'relationship_input' });
const person1Layer2 = tf.layers.dense({ units: 6, activation: 'relu' }).apply(person1Input);
const relationshipLayer2 = tf.layers.dense({ units: 6, activation: 'relu' }).apply(relationshipInput);
const concatenated = tf.layers.concatenate().apply([person1Layer2, relationshipLayer2]);
const centralLayer = tf.layers.dense({ units: 12, activation: 'relu' }).apply(concatenated);
const penultimateLayer = tf.layers.dense({ units: 6, activation: 'relu' }).apply(centralLayer);
const output = tf.layers.dense({ units: 24, activation: 'softmax' }).apply(penultimateLayer);
const model = tf.model({
inputs: [person1Input, relationshipInput],
outputs: output,
});
model.compile({
optimizer: 'adam',
loss: 'categoricalCrossentropy',
metrics: ['accuracy'],
});
return model;
}
// Train the model
async function trainModel(model, trainData, batchSize, epochs) {
const person1Inputs = tf.tensor(trainData.map(item => item.person1Input));
const relationshipInputs = tf.tensor(trainData.map(item => item.relationshipInput));
const targets = tf.tensor(trainData.map(item => item.output));
return model.fit([person1Inputs, relationshipInputs], targets, { batchSize, epochs });
}
// Evaluate the model
function evaluateModel(model, testData) {
const person1Inputs = tf.tensor(testData.map(item => item.person1Input));
const relationshipInputs = tf.tensor(testData.map(item => item.relationshipInput));
const targets = tf.tensor(testData.map(item => item.output));
const predictions = model.predict([person1Inputs, relationshipInputs]);
const accuracy = tf.metrics.categoricalAccuracy(targets, predictions).mean().arraySync();
return accuracy;
}
(async () => {
const numPersons = 24;
const numRelationships = 12;
const testSplit = 0.2;
const batchSize = 8;
const epochs = 100;
const encodedFacts = oneHotEncode(facts, numPersons, numRelationships);
const { trainData, testData } = splitData(encodedFacts, testSplit);
const model = createModel();
await trainModel(model, trainData, batchSize, epochs);
const accuracy = evaluateModel(model, testData);
console.log(`Model accuracy: ${accuracy * 100}%`);
})();
```
This code includes the `createModel` function with the given model architecture and inputs. The data preprocessing, training, and evaluation steps have been adjusted to accommodate the new model structure. The model now expects separate inputs for person1 and relationship, and it returns a single output tensor for person2.
}
trainModelEpochGenerator = {
// You can separate the setup and the training loop into different blocks of code. For monitoring the progress of training, you can create an async generator function that trains the model for one epoch at a time and yields the progress. Here's an example of how you can achieve this:
// Train the model for one epoch and yield progress
async function* trainModelEpochGenerator(
model,
trainData,
testData,
batchSize
) {
const person1Inputs = tf.tensor(trainData.map((item) => item.person1Input));
const relationshipInputs = tf.tensor(
trainData.map((item) => item.relationshipInput)
);
const targets = tf.tensor(trainData.map((item) => item.output));
let epoch = 0;
while (true) {
await model.fit([person1Inputs, relationshipInputs], targets, {
batchSize,
epochs: 1
});
epoch++;
const accuracy = console.log(model, testData);
yield { epoch, accuracy, trainModelEpochGenerator };
}
}
// Evaluate the model
// ... (same as before)
/*
(async () => {
const numPersons = 24;
const numRelationships = 12;
const testSplit = 0.2;
const batchSize = 8;
const maxEpochs = 100;
const encodedFacts = oneHotEncode(facts, numPersons, numRelationships);
const { trainData, testData } = splitData(encodedFacts, testSplit);
const model = createModel(); // Use the provided model
const trainGenerator = trainModelEpochGenerator(model, trainData, testData, batchSize);
for await (const { epoch, accuracy } of trainGenerator) {
console.log(`Epoch: ${epoch}, Model accuracy: ${accuracy * 100}%`);
if (epoch >= maxEpochs) break;
}
})();
*/
/* In this code, the trainModelEpochGenerator` function is an async generator that trains the model for one epoch at a time and yields the current epoch and the model's accuracy on the test data. The training loop now iterates over the generator, logging the progress for each epoch. The loop stops after a specified number of epochs (`maxEpochs`).
*/
}
onehot = {
function oneHotEncode(facts, numPersons, numRelationships) {
const encodedFacts = facts.map(([p1, r, p2]) => {
const p1Encoding = Array(numPersons).fill(0);
const rEncoding = Array(numRelationships).fill(0);
const p2Encoding = Array(numPersons).fill(0);
p1Encoding[p1] = 1;
rEncoding[r] = 1;
p2Encoding[p2] = 1;
return {
person1Input: p1Encoding,
relationshipInput: rEncoding,
output: p2Encoding
};
});
return encodedFacts;
}
return oneHotEncode;
}
// These two functions, `oneHotEncode` and `splitData`, are responsible for one-hot encoding the data and splitting it into training and testing sets, respectively.
splitData = {
function splitData(data, testSplit = 0.9) {
const trainData = [];
const testData = [];
data.facts.forEach((item) => {
if (Math.random() < testSplit) {
testData.push(item);
} else {
trainData.push(item);
}
});
return { trainData, testData };
}
yield splitData;
}
splitData(data, 0.599)
encode = {
const split = splitData(data, 0.9);
yield {
all: onehot(data.facts, 24, 12),
traindata: onehot(split.trainData, 24, 12),
testdata: onehot(split.testData, 24, 12)
};
}
encode
go = {
trainModelEpochGenerator.trainModelEpochGenerator(
model,
encode.traindata,
encode.testdata,
1
);
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
