diff --git a/docs.feldera.com/docs/pipelines/modifying.md b/docs.feldera.com/docs/pipelines/modifying.md index e92b928c0d4..a95b1fb4b6b 100644 --- a/docs.feldera.com/docs/pipelines/modifying.md +++ b/docs.feldera.com/docs/pipelines/modifying.md @@ -182,6 +182,12 @@ Concurrent bootstrapping proceeds in two phases: Concurrent bootstrapping is *mutually exclusive with `silent_bootstrap`**. Starting a pipeline with both `concurrent_bootstrap=true` and `silent_bootstrap=true` is rejected. +Both phases report progress: the `concurrent_bootstrap_phase` and +`concurrent_bootstrap_progress` fields returned by the +[`/stats`](/api/get-pipeline-stats) endpoint count the operators computed so far, and +the Web Console shows the same counts as a `Bootstrapping` progress bar in the +pipeline's Runtime tab. + ## Caveats and limitations ### Caveat 1: Feldera runtime upgrade can modify the pipeline diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte index 7a7560adcc3..dfc6fbeaa59 100644 --- a/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte +++ b/js-packages/web-console/src/lib/components/pipelines/editor/TabPerformance.svelte @@ -28,7 +28,7 @@ import type { ExtendedPipeline } from '$lib/services/pipelineManager' import type { TimeSeriesEntry } from '$lib/types/pipelineManager' import CheckpointsIndicator from './performance/CheckpointsIndicator.svelte' - import TransactionStatus from './performance/TransactionStatus.svelte' + import CommitProgressIndicator from './performance/CommitProgressIndicator.svelte' import Drawer from '$lib/components/layout/Drawer.svelte' import WarningBanner from './WarningBanner.svelte' import { sleep } from '$lib/functions/common/promise' @@ -360,7 +360,7 @@ {checkpointStatus} onShowCheckpoints={() => (openDrawer = { kind: 'checkpoints' })} /> - + {#if metrics.current.views.size || metrics.current.tables.size}
diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte new file mode 100644 index 00000000000..86e83c4f9ee --- /dev/null +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte @@ -0,0 +1,63 @@ + + + +
+ + + {#snippet detail()} +
+ ID:{transactionId} +
+ {/snippet} +
+ +
diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte.spec.ts b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte.spec.ts new file mode 100644 index 00000000000..4fbdd746877 --- /dev/null +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressIndicator.svelte.spec.ts @@ -0,0 +1,215 @@ +import { describe, expect, it } from 'vitest' +import { page } from 'vitest/browser' +import { render } from 'vitest-browser-svelte' +import type { PipelineMetrics } from '$lib/functions/pipelineMetrics' +import type { + CommitProgressSummary, + ConcurrentBootstrapPhase, + TransactionStatus +} from '$lib/services/manager' +import CommitProgressIndicator from './CommitProgressIndicator.svelte' + +const progress = (completed: number, in_progress: number, remaining: number) => + ({ + completed, + in_progress, + remaining, + in_progress_processed_records: 0, + in_progress_total_records: 0 + }) satisfies CommitProgressSummary + +function makeMetrics(global: { + transaction_status?: TransactionStatus + transaction_id?: number + commit_progress?: CommitProgressSummary | null + concurrent_bootstrap_phase?: ConcurrentBootstrapPhase + concurrent_bootstrap_progress?: CommitProgressSummary | null +}): { current: PipelineMetrics } { + return { + current: { + global: { + transaction_status: 'NoTransaction', + concurrent_bootstrap_phase: 'Inactive', + ...global + } + } as PipelineMetrics + } +} + +/** + * Height of the progress column - operator counts plus bar - of the transaction + * row, which is the first row of the tile. + */ +const transactionProgressHeight = (container: HTMLElement) => { + const transactionRow = container.firstElementChild!.firstElementChild! + return transactionRow.children[1].getBoundingClientRect().height +} + +/** The tile's rows, in document order. */ +const rows = (container: HTMLElement) => [...container.firstElementChild!.children] as HTMLElement[] + +/** Renders both rows inside a container of exactly `width`. */ +const renderBothRows = (width: string) => { + const result = render(CommitProgressIndicator, { + metrics: makeMetrics({ + transaction_status: 'TransactionInProgress', + transaction_id: 1, + commit_progress: progress(3, 2, 5), + concurrent_bootstrap_phase: 'ConcurrentBootstrapping', + concurrent_bootstrap_progress: progress(1, 1, 2) + }) + }) + result.container.style.width = width + return rows(result.container) +} + +const rem = () => parseFloat(getComputedStyle(document.documentElement).fontSize) + +/** A row's natural width, from its `basis-150` and `max-w-150`. */ +const rowMaxWidth = () => 37.5 * rem() + +/** The gap between side-by-side rows, from the tile's `gap-x-8`. */ +const rowGap = () => 2 * rem() + +/** + * The row title, which also carries the detail below the `sm` breakpoint, so the + * transaction row reads "Transaction ID:7" rather than "Transaction" there. + */ +const rowTitle = (label: string) => page.getByText(new RegExp(`^${label}\\b`)) + +describe('CommitProgressIndicator.svelte', () => { + it('reports no transaction and no bootstrapping row while neither is in progress', async () => { + const { container } = render(CommitProgressIndicator, { metrics: makeMetrics({}) }) + await expect.element(rowTitle('Transaction')).toBeInTheDocument() + await expect.element(page.getByText('None')).toBeInTheDocument() + await expect.element(rowTitle('Bootstrapping')).not.toBeInTheDocument() + // The transaction row holds its place; only the bootstrapping row comes and goes. + expect(rows(container)).toHaveLength(1) + }) + + describe('transaction only', () => { + const transacting = () => + render(CommitProgressIndicator, { + metrics: makeMetrics({ + transaction_status: 'CommitInProgress', + transaction_id: 7, + commit_progress: progress(3, 2, 5) + }) + }) + + it('shows the transaction status, its ID and its operator counts', async () => { + await transacting() + await expect.element(rowTitle('Transaction')).toBeInTheDocument() + await expect.element(page.getByText('Committing')).toBeInTheDocument() + await expect.element(page.getByText('ID:7')).toBeInTheDocument() + await expect.element(page.getByText(/Completed\s*3\s*out of\s*10/)).toBeInTheDocument() + }) + + it('hides the bootstrapping row', async () => { + await transacting() + await expect.element(rowTitle('Bootstrapping')).not.toBeInTheDocument() + // Only the transaction row's overlaid pair of bars is rendered. + expect(page.getByRole('progressbar').elements()).toHaveLength(2) + }) + }) + + describe('bootstrap only', () => { + const bootstrapping = () => + render(CommitProgressIndicator, { + metrics: makeMetrics({ + concurrent_bootstrap_phase: 'ConcurrentBootstrapping', + concurrent_bootstrap_progress: progress(1, 1, 2) + }) + }) + + it('shows the bootstrapping status and its operator counts', async () => { + await bootstrapping() + await expect.element(rowTitle('Bootstrapping')).toBeInTheDocument() + await expect.element(page.getByText('Backfilling')).toBeInTheDocument() + await expect.element(page.getByText(/Completed\s*1\s*out of\s*4/)).toBeInTheDocument() + }) + + it('keeps the transaction row in place, reporting no transaction', async () => { + await bootstrapping() + await expect.element(rowTitle('Transaction')).toBeInTheDocument() + await expect.element(page.getByText('None')).toBeInTheDocument() + await expect.element(page.getByText('ID:')).not.toBeInTheDocument() + // Both rows keep their pair of bars, so neither shifts vertically. + expect(page.getByRole('progressbar').elements()).toHaveLength(4) + // Only the bootstrapping row reports operator counts. + expect(page.getByTestId('box-label-completed').elements()).toHaveLength(1) + }) + + it('keeps the transaction bar in place when it reports no operator counts', async () => { + const transacting = render(CommitProgressIndicator, { + metrics: makeMetrics({ + transaction_status: 'CommitInProgress', + transaction_id: 7, + commit_progress: progress(3, 2, 5) + }) + }) + const heightWithCounts = transactionProgressHeight(transacting.container) + transacting.unmount() + + const idle = render(CommitProgressIndicator, { + metrics: makeMetrics({ concurrent_bootstrap_phase: 'ConcurrentBootstrapping' }) + }) + // Dropping the placeholder for the hidden counts shortens the column, + // shifting the bar up. + expect(transactionProgressHeight(idle.container)).toBe(heightWithCounts) + }) + }) + + it('flags the cutover pause while synchronizing', async () => { + await render(CommitProgressIndicator, { + metrics: makeMetrics({ concurrent_bootstrap_phase: 'Synchronizing' }) + }) + await expect.element(page.getByText('Synchronizing')).toBeInTheDocument() + }) + + it('shows both rows while a transaction and a bootstrap overlap', async () => { + await render(CommitProgressIndicator, { + metrics: makeMetrics({ + transaction_status: 'TransactionInProgress', + transaction_id: 1, + concurrent_bootstrap_phase: 'ConcurrentBootstrapping' + }) + }) + await expect.element(page.getByText('Started')).toBeInTheDocument() + await expect.element(page.getByText('Backfilling')).toBeInTheDocument() + }) + + describe('row layout', () => { + it('puts both rows on one line when there is room for both', () => { + const [transaction, bootstrap] = renderBothRows(`${2 * rowMaxWidth() + 4 * rem()}px`) + expect(bootstrap.getBoundingClientRect().top).toBe(transaction.getBoundingClientRect().top) + }) + + it('separates side-by-side rows by gap-x-8', () => { + const [transaction, bootstrap] = renderBothRows(`${2 * rowMaxWidth() + 8 * rem()}px`) + const gap = bootstrap.getBoundingClientRect().left - transaction.getBoundingClientRect().right + expect(gap).toBeCloseTo(rowGap(), 1) + }) + + it('wraps the second row when both no longer fit', () => { + // One row short of the pair's combined width, so only the first fits. + const [transaction, bootstrap] = renderBothRows(`${1.5 * rowMaxWidth()}px`) + expect(bootstrap.getBoundingClientRect().top).toBeGreaterThanOrEqual( + transaction.getBoundingClientRect().bottom + ) + }) + + it('keeps each row at its own max width rather than stretching', () => { + const [transaction, bootstrap] = renderBothRows('4000px') + expect(transaction.getBoundingClientRect().width).toBeCloseTo(rowMaxWidth(), 1) + expect(bootstrap.getBoundingClientRect().width).toBeCloseTo(rowMaxWidth(), 1) + }) + + it('shrinks a row below its max width in a narrow container', () => { + // The row only shrinks down to its own min-content width, since the + // operator counts do not wrap; below that it overflows rather than clip. + const [transaction] = renderBothRows(`${0.6 * rowMaxWidth()}px`) + expect(transaction.getBoundingClientRect().width).toBeLessThan(rowMaxWidth()) + }) + }) +}) diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressRow.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressRow.svelte new file mode 100644 index 00000000000..1e6f975d969 --- /dev/null +++ b/js-packages/web-console/src/lib/components/pipelines/editor/performance/CommitProgressRow.svelte @@ -0,0 +1,165 @@ + + +{#if !isIdle || idle === 'disable'} + +
+
+
+ {label} + {#if showDetail && !isScreenSm.current} + {@render detail!()} + {/if} +
+
+
+
+ {status?.label ?? 'None'} +
+
+
+ +
+ +
+ {#if isScreenSm.current} + + + {#if showDetail}{@render detail!()}{/if} + {/if} + {#if progress && !isIdle} + + Completed + Operators that have been fully flushed + {formatQty(progress.completed)} out of + {formatQty(total)} + · + In progress + Operators currently being flushed + {formatQty(progress.in_progress)} + {:else} +   + {/if} +
+
+
+ + + + + + + + + +
+
+
+
+{/if} diff --git a/js-packages/web-console/src/lib/components/pipelines/editor/performance/TransactionStatus.svelte b/js-packages/web-console/src/lib/components/pipelines/editor/performance/TransactionStatus.svelte deleted file mode 100644 index 66ee80966d9..00000000000 --- a/js-packages/web-console/src/lib/components/pipelines/editor/performance/TransactionStatus.svelte +++ /dev/null @@ -1,109 +0,0 @@ - - -{#if transactionStatus !== 'NoTransaction'} -
-
-
Transaction status
-
-
- {#if transactionStatus === 'TransactionInProgress'} -
Started
- {:else if transactionStatus === 'CommitInProgress'} -
Committing
- {/if} -
- ID:{transactionId} -
-
-
- -
-
- {#if commitProgress} - Operators: - Completed - {formatQty(commitProgress.completed)} out of - {formatQty(total)} - · - In progress - {formatQty(commitProgress.in_progress)} - {:else} -   - - {/if} -
-
-
- - - - - - - - - -
-
-
-
-{/if} diff --git a/js-packages/web-console/src/lib/compositions/layout/useIsMobile.svelte.ts b/js-packages/web-console/src/lib/compositions/layout/useIsMobile.svelte.ts index a174fa05d3d..426cea9e5cc 100644 --- a/js-packages/web-console/src/lib/compositions/layout/useIsMobile.svelte.ts +++ b/js-packages/web-console/src/lib/compositions/layout/useIsMobile.svelte.ts @@ -8,6 +8,10 @@ export const useIsMobile = () => { return new MediaQuery('not (min-width: 640px)') } +export const useIsScreenSm = () => { + return new MediaQuery('min-width: 640px') +} + export const useIsScreenMd = () => { return new MediaQuery('min-width: 768px') }