28:null 29:["$","title",null,{"children":"Logs Manager | Endpoint Services | Observable"}] 2e:T631,ui = { if (subdomains.length === 0) return md` ## No subdomains certified You will need to [certify ownership of an _Observable_ subdomain](https://observablehq.com/@tomlarkworthy/subdomain-certification) before you can access subdomain logs. Follow the instructions [here first](https://observablehq.com/@tomlarkworthy/subdomain-certification). ` return html`

Logging service

${subdomains.map(subdomain => html``)}

Subdomain	Status	Users	Actions
${subdomain}	${subdomain_states[subdomain].state \|\| "NOT ENABLED"}	${(subdomain_members[subdomain] \|\| []).map(m => m.email).join(", ")}	${subdomain_states[subdomain].state === undefined ? html``: null } ${(subdomain_members[subdomain] \|\| []).map(m => m.email).indexOf(user.email) == -1 ? html`` : html` Goto logs ` }

` }2f:T95b,overview = md` ## Open Cloud Implementation The code in this notebook implements the logs service API endpoints through the use of [serverless cells](https://observablehq.com/@tomlarkworthy/serverless-cells). The code within this notebook **is** the code in production! This notebook aims to deliver a multi-tenancy production logging experience for serverless cell users *'on the cheap'*. To keep development costs down we reuse Google Cloud Platforms logging solution and exploit [Google Cloud Logging Views](https://cloud.google.com/logging/docs/logs-views) and [IAM Conditions](https://cloud.google.com/iam/docs/conditions-overview) to enforce isolation between users. Thus, we get a production logging solution with power features like a query language but without having to develop any of the backend or UI. Reuse rocks! ### Technical Approach The serverless cell runtime emits logs named *\${subdomain}* as a *'global'* monitored [resource_type](https://cloud.google.com/logging/docs/api/v2/resource-list#resource-types). All logs for *resource_type* 'global' are exported to a project name _'endpointserviceusers'_ into a [regional log bucket](https://cloud.google.com/logging/docs/buckets) _(europe-west1)_. This separates user logs from system logs, as system logs have more specific monitored resource types. When a user enables the logging service for a subdomain ([enable endpoint](https://observablehq.com/@endpointservices/logs#enableEndpoint)), a [log view](https://cloud.google.com/logging/docs/logs-views) is created which filters the logs with log id *\${subdomain}*. When a user is given read access to logs ([addMember endpoint](https://observablehq.com/@endpointservices/logs#addMemberEndpoint)). The user's email is added to the project IAM policy as a [log view accessor with conditional access](https://cloud.google.com/logging/docs/access-control) predicted on the log resource name. Thus, the user is granted access only to a single log view which is a filter to *\${subdomain}*. In both cases we record the changes in Firestore so the front end can update in realtime. Both operations are idempotent. The IAM policy for *'endpointserviceusers'* will get large, as it serves end-users. End-user data will also appear in the console, but by putting it in an isolated project user data will not be in view for normal operations. ### Serverside Endpoints `30:T809,enableEndpoint = deploy("enable", async (req, res, ctx) => { // Check signed in user req.auth = await verifyIdToken(firebase.app(), req.headers['idtoken']); const subdomain = JSON.parse(req.body).subdomain; // Login in with service account const service_account = JSON.parse( ctx.secrets["endpointservices_secretadmin_service_account_key"]); const access_token = await getAccessTokenFromServiceAccount(service_account); const credential = firebase.auth.GoogleAuthProvider.credential(null, access_token); await firebase.auth().signInWithCredential(credential); console.log(`Logged in with uid ${firebase.auth().currentUser.uid}`) // Check user is owner of subdomain const ownersQuery = await firebase.firestore().collection("/services/ownership/owners") .where("subdomain", '==', subdomain) .where("uid", '==', req.auth.uid).limit(1).get(); if (ownersQuery.empty) { res.status(403).send("Not owner of subdomain"); } // Create a logging view for the namespace console.log("Create GAPI client") const gapi = await createGapi({ "apiKey": "AIzaSyCclj9WTy8ZAPxOBQeLyt_JS8zVF93wVnI", "discoveryDocs": ["https://logging.googleapis.com/$discovery/rest?version=v2"], "access_token": access_token }); console.log("Create Google Cloud Logging View for ${subdomain}") const response = await gapi.logging.projects.locations.buckets.views.create({ parent: "projects/endpointserviceusers/locations/europe-west1/buckets/users_europe_west1", viewId: subdomain, resource: { "name": subdomain, "description": `Logs for ${subdomain}`, // https://cloud.google.com/logging/docs/reference/v2/rest/v2/billingAccounts.buckets.views#LogView "filter": `LOG_ID(${subdomain})` } }); console.log("Write to Firestore") await firebase.firestore().doc(`/services/logs/subdomains_state/${subdomain}`).set({ state: "ENABLED" }); res.status(response.status); res.json(response.result); }, { secrets: ['endpointservices_secretadmin_service_account_key'] });31:Tae0,addMemberEndpoint = deploy("addMember", async (req, res, ctx) => { // Check signed in user req.auth = await verifyIdToken(firebase.app(), req.headers['idtoken']); const subdomain = JSON.parse(req.body).subdomain; const viewer_email = JSON.parse(req.body).viewer_email; // Login in with service account const service_account = JSON.parse( ctx.secrets["endpointservices_secretadmin_service_account_key"]); const access_token = await getAccessTokenFromServiceAccount(service_account); const credential = firebase.auth.GoogleAuthProvider.credential(null, access_token); await firebase.auth().signInWithCredential(credential); console.log(`Logged in with uid ${firebase.auth().currentUser.uid}`) // Check user is owner of subdomain const ownersQuery = await firebase.firestore().collection("/services/ownership/owners") .where("subdomain", '==', subdomain) .where("uid", '==', req.auth.uid).limit(1).get(); if (ownersQuery.empty) { res.status(403).send("Not owner of subdomain"); } // Use Cloud Resource Manager API to adjust access to the logging view console.log("Create GAPI client") const gapi = await createGapi({ "apiKey": "AIzaSyCclj9WTy8ZAPxOBQeLyt_JS8zVF93wVnI", "discoveryDocs": ["https://cloudresourcemanager.googleapis.com/$discovery/rest?version=v1"], "access_token": access_token }); console.log("Read existing IAM policy") const readResponse = await gapi.cloudresourcemanager.projects.getIamPolicy({ resource: "endpointserviceusers", options: { "requestedPolicyVersion": 3 // We need 3 for CEL conditions to work } }); const existingPolicy = JSON.parse(readResponse.body); const newBinding = { "role": "roles/logging.viewAccessor", "members": [ `user:${viewer_email}` ], "condition": { "expression": `resource.name == "projects/endpointserviceusers/locations/europe-west1/buckets/users_europe_west1/views/${subdomain}"`, "title": `${subdomain}` } }; const updatedPolicy = { version: existingPolicy.version, etag: existingPolicy.etag, bindings: [ ...existingPolicy.bindings, newBinding ] } console.log("Write new policy") const writeResponse = await gapi.cloudresourcemanager.projects.setIamPolicy({ resource: "endpointserviceusers", policy: updatedPolicy }) // Record success if (writeResponse.status == 200) { console.log("Write to Firestore") await firebase.firestore().doc( `/services/logs/subdomains_state/${subdomain}/members/${viewer_email}` ).set({role: "VIEWER"}); } res.status(writeResponse.status); res.json({msg: `${viewer_email} added to ${subdomain}`}); }, { secrets: ['endpointservices_secretadmin_service_account_key'] });32:T43e,async function getAccessTokenFromServiceAccount(serviceAccountKey) { // First create a JWT from the credentials const tNow = Math.floor((new Date()).getTime() / 1000); const sHeader = JSON.stringify({alg: 'RS256', typ: 'JWT'}); const sPayload = JSON.stringify({ iss: serviceAccountKey.client_email, scope: "https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/cloud-platform", iat: tNow, exp: tNow + 600, aud: "https://oauth2.googleapis.com/token", }); const JWT = jsrsasign.KJUR.jws.JWS.sign("RS256", sHeader, sPayload, serviceAccountKey.private_key); // Swap JWT for access_token const tokenResponse = await fetch( 'https://oauth2.googleapis.com/token', { method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'}, body: `grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion=${JWT}`, } ); if (tokenResponse.status != 200) { throw new Error(await tokenResponse.text()) } return (await tokenResponse.json()).access_token; }33:[{"id":"80f385bd757c2b37181f93d4a6f341849a0a28ccbdb21b28535a6dba3a52fb889d3ac52918d2849edbd76ae90fb8b732fa23821baebed742cb58f515ef67316b","href":"https://static.observableusercontent.com/files/80f385bd757c2b37181f93d4a6f341849a0a28ccbdb21b28535a6dba3a52fb889d3ac52918d2849edbd76ae90fb8b732fa23821baebed742cb58f515ef67316b","name":"robert-larsson-UsET4S0ginw-unsplash@1.jpg","type":"image/jpeg","lastModified":1611229800193.372,"size":171035}] 2c:{"id":"140a8b8c4bcbfb43","slug":"logs","title":"Logs Manager","thumbnail":"a359f9046ee61c578a622e7f43a95f7c6688c96bd34374ed9adc344f52e30c0f","access_level":"public","create_time":"$D2021-01-11T14:51:50.072Z","update_time":"$D2022-02-04T21:18:27.712Z","view_time":null,"first_public_time":"$D2021-01-21T10:28:08.687Z","has_unpublished_edits":null,"fork":{"id":"20bc3731121bf9c5","slug":"secrets","title":"Endpoint Secret Manager","owner":{"id":"8d94490bed468d92","login":"endpointservices","name":"Endpoint Services"}},"stars":2,"starred":false,"owner":{"type":"team","id":"8d94490bed468d92","login":"endpointservices","name":"Endpoint Services","avatar_url":"https://avatars-next.observableusercontent.com/aa238f4bd0be02bb2364438ff6bfe17b74a7a58f5c04aa1a84d29dddc79bc15e","tier":"starter_2024"},"version":1920,"publish_version":1920,"role":null,"authored":false,"authors":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null}],"members":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"title":"Logs Manager","cells":[{"id":790,"value":"md`# Logs Manager\n\nHere you can enable logging for your [serverless cells](https://observablehq.com/@tomlarkworthy/serverless-cells). Logs are organized by Observable subdomain. Once enabled, you will be provided a link to your logs which are hosted on [Google Cloud Logging](https://console.cloud.google.com/logs/query?project=endpointserviceusers). You will need to [certify ownership](https://observablehq.com/@tomlarkworthy/subdomain-certification) of your observable subdomain with Endpoint Services before you can access logs.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1838,"value":"md`\n\n![](${await FileAttachment(\"robert-larsson-UsET4S0ginw-unsplash@1.jpg\").url()})\n\nPhoto by Robert Larsson on Unsplash\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1166,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1361,"value":"md`### Login\n\nA Google Identity (gmail or gsuite account) is required to access the logs service.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":55,"value":"viewof user","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1828,"value":"md `## Open Cloud\n\n\nLike other Open Cloud tooling, the source code and design of this integration is in this notebook and permissively licensed.\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1397,"value":"md`### Change log\n- 2021-01-11: Created\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1622,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1510,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1680,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1626,"value":"md`### Clientside SDK`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1575,"value":"enable = async (subdomain) => {\n if (subdomain_states_mask[subdomain] != \"ENABLING\") {\n subdomain_states_mask[subdomain] = \"ENABLING\"\n mutable subdomain_states_mask = subdomain_states_mask;\n }\n const response = await fetch(enableEndpoint.href, {\n method: \"POST\",\n headers: {\n idtoken: await user.getIdToken()\n },\n body: JSON.stringify({\n subdomain: subdomain\n })\n })\n \n subdomain_states_mask[subdomain] = undefined\n mutable subdomain_states_mask = subdomain_states_mask;\n return await response.json();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1710,"value":"addMember = async (subdomain, viewer_email) => {\n if (subdomain_states_mask[subdomain] != \"ADDING_MEMBER\") {\n subdomain_states_mask[subdomain] = \"ADDING_MEMBER\"\n mutable subdomain_states_mask = subdomain_states_mask;\n }\n const response = await fetch(addMemberEndpoint.href, {\n method: \"POST\",\n headers: {\n idtoken: await user.getIdToken()\n },\n body: JSON.stringify({\n subdomain: subdomain,\n viewer_email: viewer_email\n })\n })\n \n subdomain_states_mask[subdomain] = undefined\n mutable subdomain_states_mask = subdomain_states_mask;\n return await response.json();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1323,"value":"md`### State`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1143,"value":"subdomain_certificates = listen(firebase.firestore().collection(\"/services/ownership/owners\").where(\"uid\", \"==\", user.uid))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1150,"value":"subdomains = subdomain_certificates.reduce(\n (acc, cert) => {\n if (!acc.includes(cert.subdomain)) acc.push(cert.subdomain)\n return acc\n },\n []\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1434,"value":"subdomain_states = {\n // Fetch all the states for all subdomains\n const states = await Promise.all(subdomains.map(\n subdomain => listen(\n firebase.firestore().doc(\n `/services/logs/subdomains_state/${subdomain}`),\n {includeId: true}\n ).next()));\n // Now index state indexed by subdomain\n return states.reduce(\n (acc, record) => Object.defineProperty(acc, record.value._id, {\n value: {...record.value, state: subdomain_states_mask[record.value._id] || record.value.state},\n enumerable: true\n })\n ,{}\n )\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1516,"value":"mutable subdomain_states_mask = ({})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1740,"value":"subdomain_members = {\n const members = await Promise.all(subdomains.map(\n subdomain => listen(\n firebase.firestore().collection(\n `/services/logs/subdomains_state/${subdomain}/members`),\n {includeRef: true}\n ).next()));\n \n const flatMembers = members.flatMap(s => s.value);\n const subdomainFromRef = (ref) => ref.match(/^services\\/logs\\/subdomains_state\\/([^/]*)\\//)[1]\n const emailFromRef = (ref) => ref.match(/^services\\/logs\\/subdomains_state\\/[^/]*\\/members\\/([^/]*)/)[1]\n const extractedmembers = flatMembers.map(member => ({\n email: emailFromRef(member._ref),\n subdomain: subdomainFromRef(member._ref),\n role: member.role\n }));\n return groupBy(extractedmembers, 'subdomain')\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1774,"value":"groupBy = function(xs, key) {\n return xs.reduce(function(rv, x) {\n (rv[x[key]] = rv[x[key]] || []).push(x);\n return rv;\n }, {});\n};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":823,"value":"import {deploy, Router, handleWithExpress} from '@tomlarkworthy/api-hosting-with-express'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":885,"value":"import {verifyIdToken} from '@tomlarkworthy/verifyidtoken'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":799,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"async function signinWithGoogleServiceAccountKeyJson(serviceAccountKey) {\n const access_token = await getAccessTokenFromServiceAccount(serviceAccountKey)\n const credential = firebase.auth.GoogleAuthProvider.credential(null, access_token);\n return await firebase.auth().signInWithCredential(credential)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"import {viewof user, firebase, listen} with {firebaseConfig as firebaseConfig} from '@tomlarkworthy/firebaseui'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":58,"value":"firebaseConfig = ({\n apiKey: \"AIzaSyD882c8YEgeYpNkX01fhpUDfioWl_ETQyQ\",\n authDomain: \"endpointservice.firebaseapp.com\",\n databaseURL: \"https://endpointservice.firebaseio.com\",\n projectId: \"endpointservice\",\n storageBucket: \"endpointservice.appspot.com\",\n messagingSenderId: \"1986724398\",\n appId: \"1:1986724398:web:9b8bc33895b45dd2e095bf\",\n uiConfig: { // https://github.com/firebase/firebaseui-web#configuration\n signInOptions: [\"google.com\"],\n },\n});","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":250,"value":"jsrsasign = require('https://bundle.run/jsrsasign@10.1.4')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":475,"value":"import {html} from \"@observablehq/htl\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":860,"value":"import {viewof suite as testsuite, expect} from '@tomlarkworthy/testing'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1109,"value":"import {encode, decode} from '@mbostock/base64'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1615,"value":"import {createGapi} from '@tomlarkworthy/gapi'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1913,"value":"import { footer } from \"@endpointservices/footer-with-backups\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1917,"value":"footer","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 2d:"$2c:body"