Support dynamic particle and recipe updates in custom particle (#149)

https://screencast.googleplex.com/cast/NjMyNTAzNTI4NDI5OTc3NnxhM2U0MmU5ZC1iMg

core/ts/Runtime.ts

@@ -225,8 +225,8 @@ export class Runtime extends EventEmitter {
   protected async marshalParticleFactory(kind: string): Promise<ParticleFactory> {
     return particleFactoryCache[kind] ?? this.lateBindParticle(kind);
   }
-  protected lateBindParticle(kind: string): Promise<unknown> {
-    return Runtime.registerParticleFactory(kind, Runtime?.particleIndustry(kind, Runtime.particleOptions));
+  protected lateBindParticle(kind: string, code?: string): Promise<unknown> {
+    return Runtime.registerParticleFactory(kind, Runtime?.particleIndustry(kind, {...Runtime.particleOptions, code}));
   }
   protected static registerParticleFactory(kind, factoryPromise: Promise<unknown>) {
     return particleFactoryCache[kind] = factoryPromise;

pkg/Library/App/Worker/Arcs.js

@@ -126,6 +126,7 @@ arcs.addPaths         = (paths)                   => socket.sendVibration({kind:
 arcs.createArc        = (arc)                     => socket.sendVibration({kind: 'createArc', arc});
 arcs.createParticle   = (name, arc, meta, code)   => socket.sendVibration({kind: 'createParticle', name, arc, meta, code});
 arcs.destroyParticle  = (name, arc)               => socket.sendVibration({kind: 'destroyParticle', name, arc});
+arcs.updateParticle   = (particle, code, arc)     => socket.sendVibration({kind: 'updateParticle', particle, code, arc});
 arcs.setInputs        = (arc, particle, inputs)   => socket.sendVibration({kind: 'setInputs', arc, particle, inputs});
 arcs.addRecipe        = (recipe, arc)             => socket.sendVibration({kind: 'addRecipe', recipe, arc});
 arcs.addAssembly      = (recipes, arc)            => socket.sendVibration({kind: 'addAssembly', recipes, arc});

pkg/Library/App/Worker/ArcsWorker.js

@@ -12,6 +12,7 @@ import {RecipeService} from '../RecipeService.js';
 import {StoreService} from '../StoreService.js';
 import {ComposerService} from '../ComposerService.js';
 import {JSONataService} from '../../JSONata/JSONataService.js';
+import {deepCopy} from '../../Core/utils.min.js';
 
 // n.b. lives in Worker context
 
@@ -164,6 +165,20 @@ const handlers = {
     // connect arc to runtime
     return user.addArc(realArc);
   },
+
+  updateParticle: async ({particle, code, arc}) => {
+    await user.lateBindParticle(particle, code);
+    // Update corresponding hosts.
+    const realArc = getArc(arc);
+    const hosts = Object.values(realArc.hosts).filter(({meta}) => meta?.kind === particle);
+    for (const host of hosts) {
+      const inputsCopy = deepCopy(host.particle.internal.inputs);
+      await user.marshalParticle(host, host.meta);
+      host.particle.inputs = inputsCopy;
+    }
+    return true;
+  },
+
   createParticle: async ({name, arc, meta, code}) => {
     const realArc = getArc(arc);
     if (realArc) {

pkg/Library/Designer/Designer.js

@@ -54,11 +54,14 @@ async removeStores({$stores}, state, service) {
 async renewRecipes(recipes, state, service) {
   for (const recipe of recipes) {
     const runningRecipe = state.recipes[recipe.$meta.name];
-    if (runningRecipe) {
+    if (!runningRecipe || runningRecipe.$meta.custom) {
+      if (runningRecipe && !deepEqual(runningRecipe, recipe)) {
+        await this.stopRecipe(recipe, state, service);
+      }
+      await this.startRecipe(recipe, state, service);
+    } else {
       await this.updateConnections(recipe, runningRecipe, state, service);
       await this.updateContainers(recipe, runningRecipe, state, service);
-    } else {
-      await this.startRecipe(recipe, state, service);
     }
   }
 },

pkg/Library/NodeCatalog/CategoryCatalog.js

@@ -21,7 +21,7 @@ selectCategory(nodeTypes, category, search) {
 filter(nodeTypes, category, search) {
   const matchSearch = (name) => (!search || name.toLowerCase().includes(search.toLowerCase()));
   const selectedNodeTypes = {};
-  keys(nodeTypes)?.forEach(id => {
+  keys(nodeTypes).forEach(id => {
     const nodeType = nodeTypes[id];
     if (nodeType.$meta.category === category && matchSearch(nodeType.$meta.displayName || nodeType.$meta.id)) {
       selectedNodeTypes[id] = nodeType;

pkg/Library/NodeGraph/ConnectionUpdator.js

@@ -32,8 +32,9 @@ update(inputs, state) {
 },
 
 inputsChanged({pipeline, candidates}, state) {
+  // TODO(mariakleiner): for custom nodes, recompute connections, if nodeType changed.
   return pipeline &&
-      (this.pipelineChanged(pipeline, state.pipeline) || this.candidatesChanged(candidates, state.candidates))
+      (this.pipelineChanged(pipeline, state.pipeline) || this.candidatesChanged(candidates, state.candidates));
 },
 
 pipelineChanged(pipeline, oldPipeline) {
@@ -55,11 +56,14 @@ removeNodeOutdatedConnections(node, candidates) {
   let changed = false;
   keys(node.connections).forEach(key => {
     const conns = node.connections[key];
-    node.connections[key] = conns.filter(conn => this.hasMatchingCandidate(conn, candidates[key]));
-    if (node.connections[key].length === 0) {
-      delete node.connections[key];
+    const connCandidates = candidates[key];
+    if (connCandidates) {
+      node.connections[key] = conns.filter(conn => this.hasMatchingCandidate(conn, connCandidates));
+      if (node.connections[key].length === 0) {
+        delete node.connections[key];
+      }
+      changed = changed || (node.connections[key]?.length === values.length);
     }
-    changed = changed || (node.connections[key]?.length === values.length);
   });
   return changed;
 },

pkg/Library/NodeGraph/Editor.js

@@ -56,6 +56,8 @@ renderNode({node, categories, pipeline, hoveredNodeId, selectedNodeId, candidate
     name: node.displayName,
     displayName: node.displayName,
     position: layout?.[node.id] || {x: 0, y: 0},
+    // TODO(mariakleiner): node-graph-editor doesn't get updated, if nodeType (and hence color)
+    // for a customNode was loaded after the node was rendered
     color: this.colorByCategory(category, categories),
     bgColor: this.bgColorByCategory(category, categories),
     selected: node.id === selectedNodeId,

pkg/Library/NodeGraph/NodeTypesCombiner.js

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) 2022 Google LLC All rights reserved.
+ * Use of this source code is governed by a BSD-style
+ * license that can be found in the LICENSE file.
+ */
+({
+
+async update({builtinNodeTypes, selectedPipeline}, state, {service}) {
+  const results = {};
+  entries(builtinNodeTypes).forEach(([key, nodeType]) => results[key] = nodeType);
+  for (const [nodeId, {html, js, spec}] of entries(selectedPipeline?.custom)) {
+    if (spec?.$meta?.id && spec?.$meta?.category) {
+      results[nodeId] = spec;
+    }
+    await this.registerCustomParticle(nodeId, {html, js}, state, service);
+  }
+  return {results};
+},
+
+async registerCustomParticle(nodeId, {html, js}, state, service) {
+  if (html !== state[nodeId]?.html || js !== state[nodeId]?.js) {
+    assign(state, {[nodeId]: {html, js}});
+    await service({msg: 'updateParticle', data: {
+      kind: nodeId,
+      code: this.formatCode({html, js})
+    }});
+  }
+},
+
+formatCode({html, js}) {
+  return `({
+    ${js ? `${js},` : ''}
+    ${html ? `template: html\`${html}\`` : ''}
+  });`;
+}
+
+});

pkg/Library/NodeGraph/RecipeBuilder.js

@@ -13,7 +13,7 @@ idDelim: ':',
 defaultContainer: `main#runner`,
 
 async update(inputs, state) {
-  const {pipeline, layout} = inputs;
+  const {pipeline, nodeTypes, layout} = inputs;
   if (pipeline) {
     let changed = false;
     if (this.pipelineChanged(pipeline, state.pipeline)) {
@@ -24,6 +24,10 @@ async update(inputs, state) {
       assign(state, {layout});
       changed = true;
     }
+    if (this.nodeTypesChanged(nodeTypes, state.nodeTypes)) {
+      assign(state, {nodeTypes});
+      changed = true;
+    }
     if (changed) {
       return {recipes: this.recipesForPipeline(inputs, state)};
     }
@@ -32,11 +36,21 @@ async update(inputs, state) {
   }
 },
 
+nodeTypesChanged(nodeTypes, oldNodeTypes) {
+  return keys(nodeTypes).length !== keys(oldNodeTypes).length
+    || !entries(nodeTypes).every(([id, nodeType]) => !nodeType.$meta.custom || deepEqual(nodeType, oldNodeTypes[id]));
+
+},
+
 nodeTypeMap(nodeTypes, state) {
   if (!state.nodeTypeMap) {
     state.nodeTypeMap = {};
-    values(nodeTypes).forEach(t => state.nodeTypeMap[t.$meta.id] = this.flattenNodeType(t));
   }
+  values(nodeTypes).forEach(t => {
+    if (!state.nodeTypeMap[t.$meta.id] || t.$meta.custom) {
+      state.nodeTypeMap[t.$meta.id] = this.flattenNodeType(t);
+    }
+  });
   return state.nodeTypeMap;
 },
 
@@ -84,20 +98,26 @@ layoutChanged(pipeline, layout, oldLayout) {
 
 recipesForPipeline(inputs, state) {
   const {pipeline} = inputs;
-  return values(pipeline.nodes).map(node => this.recipeForNode(node, inputs, state));
+  return values(pipeline.nodes)
+    .map(node => this.recipeForNode(node, inputs, state))
+    .filter(recipe => recipe)
+    ;
 },
 
 recipeForNode(node, inputs, state) {
   const {pipeline, nodeTypes, layout} = inputs;
   const nodeTypeMap = this.nodeTypeMap(nodeTypes, state);
   const nodeType = nodeTypeMap[node.type];
-  const stores = this.buildStoreSpecs(node, nodeType, state);
-  const recipe = this.buildParticleSpecs(node, nodeType, layout, state);
-  recipe.$meta = {
-    name: this.encodeFullNodeId(node, pipeline, this.connectorDelim)
-  };
-  recipe.$stores = stores;
-  return recipe;
+  if (nodeType) {
+    const stores = this.buildStoreSpecs(node, nodeType, state);
+    const recipe = this.buildParticleSpecs(node, nodeType, layout, state);
+    recipe.$meta = {
+      name: this.encodeFullNodeId(node, pipeline, this.connectorDelim),
+      custom: nodeType.$meta.custom
+    };
+    recipe.$stores = stores;
+    return recipe;
+  }
 },
 
 buildParticleSpecs(node, nodeType, layout, {storeMap}) {
@@ -148,7 +168,7 @@ resolveBindings(particleSpec, storeMap) {
 buildStoreSpecs(node, nodeType, state) {
   const specs = {};
   state.storeMap = {};
-  entries(nodeType.$stores).forEach(([name, store]) => {
+  entries(nodeType?.$stores).forEach(([name, store]) => {
     state.storeMap[name] = [];
     if (store.connection) {
       const connections = node.connections?.[name];

pkg/nodegraph/Library/Librarian/CustomNodes.js

@@ -13,22 +13,27 @@ export const Librarian = {
     category: 'Custom'
   },
   $stores: {
-    particle: {
-      $type: '[Particle]',
-      // TODO(mariakleiner): if not inspected, changing in Designer/Runner doesn't affect the pipeline!
-      // noinspect: true
-      //
-    },
     nodeId: {
       $type: 'String',
       noinspect: true,
       nodisplay: true,
       value: 'node.id'
-    },  
+    },
+    selectedPipeline: {
+      $type: 'JSON',
+      connection: true,
+      noinspect: true,
+      nodisplay: true
+    },
   },
   customParticle: {
     $kind: '$app/Library/Librarian/CustomParticle',
-    $inputs: ['particle', 'nodeId'],
-    $outputs: ['particle']
+    $inputs: [
+      'nodeId',
+      {'pipeline': 'selectedPipeline'}
+    ],
+    $outputs: [
+      {'pipeline': 'selectedPipeline'}
+    ]
   }
 };