Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ CREATE TABLE bronze_clickstream_events (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_clickstream_events",
"transport": {
"name": "delta_table_input",
"config": {
Expand Down Expand Up @@ -115,6 +116,7 @@ CREATE TABLE bronze_orders (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_orders",
"transport": {
"name": "delta_table_input",
"config": {
Expand All @@ -139,6 +141,7 @@ CREATE TABLE bronze_order_items (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_order_items",
"transport": {
"name": "delta_table_input",
"config": {
Expand Down Expand Up @@ -167,6 +170,7 @@ CREATE TABLE bronze_products (
'skip_unused_columns' = 'true',
'connectors' = '
[{
"name": "bronze_products",
"transport": {
"name": "delta_table_input",
"config": {
Expand All @@ -191,6 +195,7 @@ CREATE TABLE bronze_inventory_events (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_inventory_events",
"transport": {
"name": "delta_table_input",
"config": {
Expand All @@ -215,6 +220,7 @@ CREATE TABLE bronze_customers (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_customers",
"transport": {
"name": "delta_table_input",
"config": {
Expand All @@ -237,6 +243,7 @@ CREATE TABLE bronze_suppliers (
) WITH (
'skip_unused_columns' = 'true',
'connectors' = '[{
"name": "bronze_suppliers",
"transport": {
"name": "delta_table_input",
"config": {
Expand Down
2 changes: 1 addition & 1 deletion crates/pipeline-manager/demos/sql/03-sec-ops.sql
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ CREATE TABLE pipeline_sources (
pipeline_id BIGINT NOT NULL FOREIGN KEY REFERENCES pipeline(pipeline_id)
) WITH (
'connectors' = '[{
"name": "pipelie_sources",
"name": "pipeline_sources",
"transport": {
"name": "datagen",
"config": {
Expand Down
13 changes: 8 additions & 5 deletions docs.feldera.com/docs/operations/visualizing-profiles.md
Original file line number Diff line number Diff line change
Expand Up @@ -221,11 +221,14 @@ selection.

![Choosing the workers to display](workers.png)

### Searching a node by name

The search box allows searching a node by name. The display will be
centered around the node. (In the future we may allow searching by
attributes as well.)
### Searching for a node

The search box finds a node by its ID, by the name of an input table
or an output view (an exact name match wins over a substring match),
or by a substring of its persistent ID. The
display is centered around the node found; if the node is hidden
inside collapsed regions, these regions are expanded first. (In the
future we may allow searching by attributes as well.)

![Searching for nodes](search.png)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,8 @@
<input
bind:value={searchQuery}
type="text"
placeholder="ID or persistent ID"
title="Search for node by ID or a substring of a persistent ID"
placeholder="ID, name, or persistent ID"
title="Search for node by ID, by table or view name, or by a substring of a persistent ID"
onkeydown={(e) => e.key === 'Enter' && handleSearch()}
class="input w-32 text-sm"
/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@
bind:value={nodeSearchQuery}
type="text"
placeholder="Search node"
title="Search for a node by ID or persistent ID"
title="Search for a node by ID, by table or view name, or by a substring of a persistent ID"
onkeydown={(e) => e.key === 'Enter' && handleSearch()}
class="input h-6 w-36 text-sm"
/>
Expand Down
12 changes: 10 additions & 2 deletions js-packages/profiler-lib/src/cytograph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import cytoscape, { type EdgeCollection, type EdgeDefinition, type ElementsDefinition, type EventObject, type NodeDefinition, type NodeSingular, type StylesheetJson } from 'cytoscape';
import dblclick from 'cytoscape-dblclick';
import { assert, Graph, OMap, Option, type EncodableAsString, NumericRange, Edge } from './util.js';
import { CircuitProfile, NodeAndMetric, PropertyValue, type NodeId } from './profile.js';
import { CircuitProfile, ComplexNode, NodeAndMetric, PropertyValue, type NodeId } from './profile.js';
import { CircuitSelection } from './selection.js';
import elk from 'cytoscape-elk';
import { Sources } from './dataflow.js';
Expand Down Expand Up @@ -320,7 +320,10 @@ export class Cytograph {
visibleParents.add(p);
}
let src = sources.toString(node.sourcePositions);
let operation = node.operation;
let operation = node instanceof ComplexNode
// node is complex only when drawn collapsed; show the tables and views hidden inside
? node.collapsedOperation()
: node.operation;
if (operation === CircuitProfile.Z1_TRACE_OUTPUT)
// These nodes were modified in the profile.fixZ1Nodes() function.
operation = CircuitProfile.Z1_TRACE;
Expand Down Expand Up @@ -565,6 +568,11 @@ export class CytographRendering {
return this.metadataSelection.metric;
}

/** Center the view on this node after the next layout completes. */
centerOnNextLayout(node: Option<NodeId>) {
this.lastNode = node;
}

/** Search a node by ID, return 'true' if found. */
search(value: string): boolean {
let el = this.cy.getElementById(value);
Expand Down
71 changes: 71 additions & 0 deletions js-packages/profiler-lib/src/profile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@ import {
BooleanValue,
BytesValue,
CircuitProfile,
ComplexNode,
CountValue,
MissingValue,
PercentValue,
PropertyValue,
SimpleNode,
StringValue,
TimeValue
} from './profile.js'
import type { Dataflow } from './dataflow.js'
import { NumericRange } from './util.js'

describe('CircuitProfile.isTop', () => {
Expand Down Expand Up @@ -342,3 +345,71 @@ describe('NumericRange cross-node normalization', () => {
expect(point.isPoint()).toBe(true)
})
})

describe('CircuitProfile.byName', () => {
const mirNode = (persistent_id: string, table: string | null, view: string | null) => ({
operation: 'op', table, view, inputs: [], calcite: {}, positions: [], persistent_id
})

const makeProfile = () => {
const profile = new CircuitProfile(1, 'n')
const source = new SimpleNode('n1', 'source', 1)
const sink = new SimpleNode('n2', 'sink', 1)
const port = new SimpleNode('n3', 'source', 1)
for (const [pid, node] of [['abc123', source], ['def456', sink], ['789fed', port]] as const) {
profile.simpleNodes.set(node.id, node)
profile.byPersistentId.set(pid, node)
}
const dataflow: Dataflow = {
calcite_plan: {},
mir: {
s1: mirNode('abc123', 'CUSTOMERS', null),
s2: mirNode('def456', null, 'report'),
s3: mirNode('789fed', 'port', null)
}
}
profile.setDataflow(dataflow)
return { profile, source, sink, port }
}

it('indexes input tables and output views by lowercase name', () => {
const { profile, source, sink } = makeProfile()
// Quoted uppercase table names are found by their lowercase key
expect(profile.byName.get('customers').unwrap()).toBe(source)
expect(profile.byName.get('report').unwrap()).toBe(sink)
expect(profile.byName.get('missing').isNone()).toBe(true)
})

it('findByName falls back to a substring match', () => {
const { profile, source, sink } = makeProfile()
expect(profile.findByName('CUSTOM').unwrap()).toBe(source)
expect(profile.findByName('epor').unwrap()).toBe(sink)
expect(profile.findByName('missing').isNone()).toBe(true)
})

it('findByName prefers an exact match over a substring match', () => {
const { profile, port } = makeProfile()
// 'port' is a substring of 'report', but the exact match wins
expect(profile.findByName('port').unwrap()).toBe(port)
})

it('propagates the name to ancestors for collapsed display', () => {
const profile = new CircuitProfile(1, 'n')
const outer = new ComplexNode('c1', 'region', 1)
const inner = new ComplexNode('c2', 'subregion', 1)
const source = new SimpleNode('n1', 'source', 1)
profile.complexNodes.set(outer.id, outer)
profile.complexNodes.set(inner.id, inner)
profile.simpleNodes.set(source.id, source)
profile.parents.set(inner.id, outer.id)
profile.parents.set(source.id, inner.id)
profile.byPersistentId.set('abc123', source)

profile.setDataflow({ calcite_plan: {}, mir: { s1: mirNode('abc123', 'customers', null) } })

expect(outer.collapsedOperation()).toBe('region customers')
expect(inner.collapsedOperation()).toBe('subregion customers')
// The expanded label stays unchanged
expect(outer.operation).toBe('region')
})
})
46 changes: 42 additions & 4 deletions js-packages/profiler-lib/src/profile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1286,12 +1286,23 @@ export class SimpleNode implements JsonSimpleCircuitNode {
export class ComplexNode extends SimpleNode {
children: Array<NodeId>;
depth: number = 0;
// Names of tables and views of descendant nodes.
readonly containedNames: Array<string> = [];

constructor(id: NodeId, label: string, worker_count: number) {
super(id, label, worker_count);
this.children = [];
}

/** Label to display when the node is drawn collapsed:
* the operation plus the names of the tables and views hidden inside. */
collapsedOperation(): string {
if (this.containedNames.length === 0) {
return this.operation;
}
return this.operation + " " + this.containedNames.join(", ");
}

addChild(node: SimpleNode) {
this.children.push(node.id);
}
Expand Down Expand Up @@ -1332,6 +1343,8 @@ export class CircuitProfile {
readonly dataRange: OMap<string, NumericRange> = new OMap();
// Index nodes by their persistent IDs
readonly byPersistentId: OMap<string, SimpleNode> = new OMap();
// Index nodes by table or view name (lowercase); filled from the dataflow graph.
readonly byName: OMap<string, SimpleNode> = new OMap();
// Source information; only available if the dataflow information is provided.
sources: Option<Sources> = Option.none();

Expand Down Expand Up @@ -1379,16 +1392,41 @@ export class CircuitProfile {
// This can happen for some Z nodes in recursive components
if (profileNode.isNone()) return;
let n = profileNode.unwrap();
if (mir.table !== null) {
n.operation += " " + mir.table;
} else if (mir.view !== null) {
n.operation += " " + mir.view;
const name = mir.table !== null ? mir.table : mir.view;
if (name !== null) {
n.operation += " " + name;
this.byName.set(name.toLowerCase(), n);
// Ancestors display the contained names when collapsed
let parent = this.parents.get(n.id);
while (parent.isSome()) {
const complex = this.complexNodes.get(parent.unwrap()).unwrap();
if (!complex.containedNames.includes(name)) {
complex.containedNames.push(name);
}
parent = this.parents.get(parent.unwrap());
}
}

n.setSourcePositions(new SourcePositionRanges(
mir.positions.map(p => new SourcePositionRange(p))));
}

/** Find the node of an input table or output view by name.
* The match is case-insensitive; an exact match wins over a substring match. */
findByName(name: string): Option<SimpleNode> {
const key = name.toLowerCase();
const exact = this.byName.get(key);
if (exact.isSome()) {
return exact;
}
for (const [n, node] of this.byName.entries()) {
if (n.includes(key)) {
return Option.some(node);
}
}
return Option.none();
}

// Get the topmost parent of a node which is not the toplevel graph node.
getTopParent(node: NodeId): NodeId {
while (this.parents.has(node)) {
Expand Down
62 changes: 35 additions & 27 deletions js-packages/profiler-lib/src/profiler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -297,47 +297,55 @@ export class Visualizer {
}

/**
* Search for a node by ID or a substring of the persistent ID.
* Center the view on the node with the given ID.
* If the node is hidden inside collapsed clusters, expand them;
* the layout is recomputed asynchronously, and the rendering
* centers on the node when the layout completes.
* Return 'true' if the node was found.
*/
private reveal(id: string): boolean {
if (this.rendering?.search(id)) {
return true;
}
if (!this.rendering || !this.circuitSelector) {
return false;
}
this.rendering.centerOnNextLayout(Option.some(id));
if (this.circuitSelector.expandAncestors(id)) {
return true;
}
// Nothing was expanded; do not center on the next unrelated layout.
this.rendering.centerOnNextLayout(Option.none());
return false;
}

/**
* Search for a node by ID, by input table or output view name,
* or by a substring of the persistent ID.
*/
search(query: string): void {
// First search by ID
let success = this.rendering?.search(query);
if (success) {
// First search by node ID
if (this.reveal(query)) {
return;
}
if (!this.profile) {
return;
}

// Find ID of node with given persistent ID
// Find node of an input table or output view with the given name
let named = this.profile.findByName(query);
if (named.isSome() && this.reveal(named.unwrap().id)) {
return;
}

// Find node with given persistent ID
for (const [pid, node] of this.profile.byPersistentId) {
if (pid.includes(query)) {
let success = this.rendering?.search(node.id);
if (success) {
if (this.reveal(node.id)) {
return;
}
}
}

// Neither succeeded, check if we are searching for an unexpanded node
let nodeN = this.profile.simpleNodes.get(query);
if (nodeN.isSome()) {
let node = nodeN.unwrap();
let current = node.id;
// Find the outermost parent
while (true) {
let parent = this.profile.parents.get(current);
if (parent.isNone()) {
break;
}
current = parent.unwrap();
}
// Current is the outermost parent
// Expansion will happen asynchronously,
// so we are not searching again, hopefully that is good
// enough to locate the node.
this.circuitSelector?.toggleExpand(current);
}
}

/**
Expand Down
Loading