diff --git a/crates/pipeline-manager/demos/sql/01-medallion-architecture.sql b/crates/pipeline-manager/demos/sql/01-medallion-architecture.sql index d7596f02bfb..0b403723b70 100644 --- a/crates/pipeline-manager/demos/sql/01-medallion-architecture.sql +++ b/crates/pipeline-manager/demos/sql/01-medallion-architecture.sql @@ -87,6 +87,7 @@ CREATE TABLE bronze_clickstream_events ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_clickstream_events", "transport": { "name": "delta_table_input", "config": { @@ -115,6 +116,7 @@ CREATE TABLE bronze_orders ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_orders", "transport": { "name": "delta_table_input", "config": { @@ -139,6 +141,7 @@ CREATE TABLE bronze_order_items ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_order_items", "transport": { "name": "delta_table_input", "config": { @@ -167,6 +170,7 @@ CREATE TABLE bronze_products ( 'skip_unused_columns' = 'true', 'connectors' = ' [{ + "name": "bronze_products", "transport": { "name": "delta_table_input", "config": { @@ -191,6 +195,7 @@ CREATE TABLE bronze_inventory_events ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_inventory_events", "transport": { "name": "delta_table_input", "config": { @@ -215,6 +220,7 @@ CREATE TABLE bronze_customers ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_customers", "transport": { "name": "delta_table_input", "config": { @@ -237,6 +243,7 @@ CREATE TABLE bronze_suppliers ( ) WITH ( 'skip_unused_columns' = 'true', 'connectors' = '[{ + "name": "bronze_suppliers", "transport": { "name": "delta_table_input", "config": { diff --git a/crates/pipeline-manager/demos/sql/03-sec-ops.sql b/crates/pipeline-manager/demos/sql/03-sec-ops.sql index 444759b199e..e7295b6076c 100644 --- a/crates/pipeline-manager/demos/sql/03-sec-ops.sql +++ b/crates/pipeline-manager/demos/sql/03-sec-ops.sql @@ -84,7 +84,7 @@ CREATE TABLE pipeline_sources ( pipeline_id BIGINT NOT NULL FOREIGN KEY REFERENCES pipeline(pipeline_id) ) WITH ( 'connectors' = '[{ - "name": "pipelie_sources", + "name": "pipeline_sources", "transport": { "name": "datagen", "config": { diff --git a/docs.feldera.com/docs/operations/visualizing-profiles.md b/docs.feldera.com/docs/operations/visualizing-profiles.md index c446d662e19..25898bf08b2 100644 --- a/docs.feldera.com/docs/operations/visualizing-profiles.md +++ b/docs.feldera.com/docs/operations/visualizing-profiles.md @@ -221,11 +221,14 @@ selection. ![Choosing the workers to display](workers.png) -### Searching a node by name - -The search box allows searching a node by name. The display will be -centered around the node. (In the future we may allow searching by -attributes as well.) +### Searching for a node + +The search box finds a node by its ID, by the name of an input table +or an output view (an exact name match wins over a substring match), +or by a substring of its persistent ID. The +display is centered around the node found; if the node is hidden +inside collapsed regions, these regions are expanded first. (In the +future we may allow searching by attributes as well.) ![Searching for nodes](search.png) diff --git a/js-packages/profiler-layout/src/lib/components/ProfilerLayout.svelte b/js-packages/profiler-layout/src/lib/components/ProfilerLayout.svelte index f07bf6c3da6..2ba7e3eeeb3 100644 --- a/js-packages/profiler-layout/src/lib/components/ProfilerLayout.svelte +++ b/js-packages/profiler-layout/src/lib/components/ProfilerLayout.svelte @@ -232,8 +232,8 @@ e.key === 'Enter' && handleSearch()} class="input w-32 text-sm" /> diff --git a/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte b/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte index 93593bb56f2..e67d287c1c1 100644 --- a/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte +++ b/js-packages/profiler-layout/src/lib/components/SupportBundleViewerLayout.svelte @@ -336,7 +336,7 @@ bind:value={nodeSearchQuery} type="text" placeholder="Search node" - title="Search for a node by ID or persistent ID" + title="Search for a node by ID, by table or view name, or by a substring of a persistent ID" onkeydown={(e) => e.key === 'Enter' && handleSearch()} class="input h-6 w-36 text-sm" /> diff --git a/js-packages/profiler-lib/src/cytograph.ts b/js-packages/profiler-lib/src/cytograph.ts index bf63801d4ca..df551eaf909 100644 --- a/js-packages/profiler-lib/src/cytograph.ts +++ b/js-packages/profiler-lib/src/cytograph.ts @@ -3,7 +3,7 @@ import cytoscape, { type EdgeCollection, type EdgeDefinition, type ElementsDefinition, type EventObject, type NodeDefinition, type NodeSingular, type StylesheetJson } from 'cytoscape'; import dblclick from 'cytoscape-dblclick'; import { assert, Graph, OMap, Option, type EncodableAsString, NumericRange, Edge } from './util.js'; -import { CircuitProfile, NodeAndMetric, PropertyValue, type NodeId } from './profile.js'; +import { CircuitProfile, ComplexNode, NodeAndMetric, PropertyValue, type NodeId } from './profile.js'; import { CircuitSelection } from './selection.js'; import elk from 'cytoscape-elk'; import { Sources } from './dataflow.js'; @@ -320,7 +320,10 @@ export class Cytograph { visibleParents.add(p); } let src = sources.toString(node.sourcePositions); - let operation = node.operation; + let operation = node instanceof ComplexNode + // node is complex only when drawn collapsed; show the tables and views hidden inside + ? node.collapsedOperation() + : node.operation; if (operation === CircuitProfile.Z1_TRACE_OUTPUT) // These nodes were modified in the profile.fixZ1Nodes() function. operation = CircuitProfile.Z1_TRACE; @@ -565,6 +568,11 @@ export class CytographRendering { return this.metadataSelection.metric; } + /** Center the view on this node after the next layout completes. */ + centerOnNextLayout(node: Option) { + this.lastNode = node; + } + /** Search a node by ID, return 'true' if found. */ search(value: string): boolean { let el = this.cy.getElementById(value); diff --git a/js-packages/profiler-lib/src/profile.test.ts b/js-packages/profiler-lib/src/profile.test.ts index 762f7338393..5bc3ea0d21d 100644 --- a/js-packages/profiler-lib/src/profile.test.ts +++ b/js-packages/profiler-lib/src/profile.test.ts @@ -3,13 +3,16 @@ import { BooleanValue, BytesValue, CircuitProfile, + ComplexNode, CountValue, MissingValue, PercentValue, PropertyValue, + SimpleNode, StringValue, TimeValue } from './profile.js' +import type { Dataflow } from './dataflow.js' import { NumericRange } from './util.js' describe('CircuitProfile.isTop', () => { @@ -342,3 +345,71 @@ describe('NumericRange cross-node normalization', () => { expect(point.isPoint()).toBe(true) }) }) + +describe('CircuitProfile.byName', () => { + const mirNode = (persistent_id: string, table: string | null, view: string | null) => ({ + operation: 'op', table, view, inputs: [], calcite: {}, positions: [], persistent_id + }) + + const makeProfile = () => { + const profile = new CircuitProfile(1, 'n') + const source = new SimpleNode('n1', 'source', 1) + const sink = new SimpleNode('n2', 'sink', 1) + const port = new SimpleNode('n3', 'source', 1) + for (const [pid, node] of [['abc123', source], ['def456', sink], ['789fed', port]] as const) { + profile.simpleNodes.set(node.id, node) + profile.byPersistentId.set(pid, node) + } + const dataflow: Dataflow = { + calcite_plan: {}, + mir: { + s1: mirNode('abc123', 'CUSTOMERS', null), + s2: mirNode('def456', null, 'report'), + s3: mirNode('789fed', 'port', null) + } + } + profile.setDataflow(dataflow) + return { profile, source, sink, port } + } + + it('indexes input tables and output views by lowercase name', () => { + const { profile, source, sink } = makeProfile() + // Quoted uppercase table names are found by their lowercase key + expect(profile.byName.get('customers').unwrap()).toBe(source) + expect(profile.byName.get('report').unwrap()).toBe(sink) + expect(profile.byName.get('missing').isNone()).toBe(true) + }) + + it('findByName falls back to a substring match', () => { + const { profile, source, sink } = makeProfile() + expect(profile.findByName('CUSTOM').unwrap()).toBe(source) + expect(profile.findByName('epor').unwrap()).toBe(sink) + expect(profile.findByName('missing').isNone()).toBe(true) + }) + + it('findByName prefers an exact match over a substring match', () => { + const { profile, port } = makeProfile() + // 'port' is a substring of 'report', but the exact match wins + expect(profile.findByName('port').unwrap()).toBe(port) + }) + + it('propagates the name to ancestors for collapsed display', () => { + const profile = new CircuitProfile(1, 'n') + const outer = new ComplexNode('c1', 'region', 1) + const inner = new ComplexNode('c2', 'subregion', 1) + const source = new SimpleNode('n1', 'source', 1) + profile.complexNodes.set(outer.id, outer) + profile.complexNodes.set(inner.id, inner) + profile.simpleNodes.set(source.id, source) + profile.parents.set(inner.id, outer.id) + profile.parents.set(source.id, inner.id) + profile.byPersistentId.set('abc123', source) + + profile.setDataflow({ calcite_plan: {}, mir: { s1: mirNode('abc123', 'customers', null) } }) + + expect(outer.collapsedOperation()).toBe('region customers') + expect(inner.collapsedOperation()).toBe('subregion customers') + // The expanded label stays unchanged + expect(outer.operation).toBe('region') + }) +}) diff --git a/js-packages/profiler-lib/src/profile.ts b/js-packages/profiler-lib/src/profile.ts index f9388d9739c..f6040f7f917 100644 --- a/js-packages/profiler-lib/src/profile.ts +++ b/js-packages/profiler-lib/src/profile.ts @@ -1286,12 +1286,23 @@ export class SimpleNode implements JsonSimpleCircuitNode { export class ComplexNode extends SimpleNode { children: Array; depth: number = 0; + // Names of tables and views of descendant nodes. + readonly containedNames: Array = []; constructor(id: NodeId, label: string, worker_count: number) { super(id, label, worker_count); this.children = []; } + /** Label to display when the node is drawn collapsed: + * the operation plus the names of the tables and views hidden inside. */ + collapsedOperation(): string { + if (this.containedNames.length === 0) { + return this.operation; + } + return this.operation + " " + this.containedNames.join(", "); + } + addChild(node: SimpleNode) { this.children.push(node.id); } @@ -1332,6 +1343,8 @@ export class CircuitProfile { readonly dataRange: OMap = new OMap(); // Index nodes by their persistent IDs readonly byPersistentId: OMap = new OMap(); + // Index nodes by table or view name (lowercase); filled from the dataflow graph. + readonly byName: OMap = new OMap(); // Source information; only available if the dataflow information is provided. sources: Option = Option.none(); @@ -1379,16 +1392,41 @@ export class CircuitProfile { // This can happen for some Z nodes in recursive components if (profileNode.isNone()) return; let n = profileNode.unwrap(); - if (mir.table !== null) { - n.operation += " " + mir.table; - } else if (mir.view !== null) { - n.operation += " " + mir.view; + const name = mir.table !== null ? mir.table : mir.view; + if (name !== null) { + n.operation += " " + name; + this.byName.set(name.toLowerCase(), n); + // Ancestors display the contained names when collapsed + let parent = this.parents.get(n.id); + while (parent.isSome()) { + const complex = this.complexNodes.get(parent.unwrap()).unwrap(); + if (!complex.containedNames.includes(name)) { + complex.containedNames.push(name); + } + parent = this.parents.get(parent.unwrap()); + } } n.setSourcePositions(new SourcePositionRanges( mir.positions.map(p => new SourcePositionRange(p)))); } + /** Find the node of an input table or output view by name. + * The match is case-insensitive; an exact match wins over a substring match. */ + findByName(name: string): Option { + const key = name.toLowerCase(); + const exact = this.byName.get(key); + if (exact.isSome()) { + return exact; + } + for (const [n, node] of this.byName.entries()) { + if (n.includes(key)) { + return Option.some(node); + } + } + return Option.none(); + } + // Get the topmost parent of a node which is not the toplevel graph node. getTopParent(node: NodeId): NodeId { while (this.parents.has(node)) { diff --git a/js-packages/profiler-lib/src/profiler.ts b/js-packages/profiler-lib/src/profiler.ts index 1159bdf6e92..ddd1508325d 100644 --- a/js-packages/profiler-lib/src/profiler.ts +++ b/js-packages/profiler-lib/src/profiler.ts @@ -297,47 +297,55 @@ export class Visualizer { } /** - * Search for a node by ID or a substring of the persistent ID. + * Center the view on the node with the given ID. + * If the node is hidden inside collapsed clusters, expand them; + * the layout is recomputed asynchronously, and the rendering + * centers on the node when the layout completes. + * Return 'true' if the node was found. + */ + private reveal(id: string): boolean { + if (this.rendering?.search(id)) { + return true; + } + if (!this.rendering || !this.circuitSelector) { + return false; + } + this.rendering.centerOnNextLayout(Option.some(id)); + if (this.circuitSelector.expandAncestors(id)) { + return true; + } + // Nothing was expanded; do not center on the next unrelated layout. + this.rendering.centerOnNextLayout(Option.none()); + return false; + } + + /** + * Search for a node by ID, by input table or output view name, + * or by a substring of the persistent ID. */ search(query: string): void { - // First search by ID - let success = this.rendering?.search(query); - if (success) { + // First search by node ID + if (this.reveal(query)) { return; } if (!this.profile) { return; } - // Find ID of node with given persistent ID + // Find node of an input table or output view with the given name + let named = this.profile.findByName(query); + if (named.isSome() && this.reveal(named.unwrap().id)) { + return; + } + + // Find node with given persistent ID for (const [pid, node] of this.profile.byPersistentId) { if (pid.includes(query)) { - let success = this.rendering?.search(node.id); - if (success) { + if (this.reveal(node.id)) { return; } } } - - // Neither succeeded, check if we are searching for an unexpanded node - let nodeN = this.profile.simpleNodes.get(query); - if (nodeN.isSome()) { - let node = nodeN.unwrap(); - let current = node.id; - // Find the outermost parent - while (true) { - let parent = this.profile.parents.get(current); - if (parent.isNone()) { - break; - } - current = parent.unwrap(); - } - // Current is the outermost parent - // Expansion will happen asynchronously, - // so we are not searching again, hopefully that is good - // enough to locate the node. - this.circuitSelector?.toggleExpand(current); - } } /** diff --git a/js-packages/profiler-lib/src/selection.ts b/js-packages/profiler-lib/src/selection.ts index 1cdce5bb8c0..b3263ab74b5 100644 --- a/js-packages/profiler-lib/src/selection.ts +++ b/js-packages/profiler-lib/src/selection.ts @@ -39,6 +39,25 @@ export class CircuitSelector { this.onChange(); } + /** Expand all collapsed ancestors of a node so that it becomes visible. + * Return 'true' if anything changed. */ + expandAncestors(node: NodeId): boolean { + let changed = false; + let parent = this.circuit.parents.get(node); + while (parent.isSome()) { + const id = parent.unwrap(); + if (!this.regionsExpanded.has(id)) { + this.regionsExpanded.add(id); + changed = true; + } + parent = this.circuit.parents.get(id); + } + if (changed) { + this.onChange(); + } + return changed; + } + getFullSelection(): CircuitSelection { return new CircuitSelection(new CompleteSet(this.allNodeIds)) }