Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs.feldera.com/docs/pipelines/modifying.md
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,12 @@ Concurrent bootstrapping proceeds in two phases:
Concurrent bootstrapping is *mutually exclusive with `silent_bootstrap`**. Starting a
pipeline with both `concurrent_bootstrap=true` and `silent_bootstrap=true` is rejected.

Both phases report progress: the `concurrent_bootstrap_phase` and
`concurrent_bootstrap_progress` fields returned by the
[`/stats`](/api/get-pipeline-stats) endpoint count the operators computed so far, and
the Web Console shows the same counts as a `Bootstrapping` progress bar in the
pipeline's Runtime tab.

## Caveats and limitations

### Caveat 1: Feldera runtime upgrade can modify the pipeline
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import type { ExtendedPipeline } from '$lib/services/pipelineManager'
import type { TimeSeriesEntry } from '$lib/types/pipelineManager'
import CheckpointsIndicator from './performance/CheckpointsIndicator.svelte'
import TransactionStatus from './performance/TransactionStatus.svelte'
import CommitProgressIndicator from './performance/CommitProgressIndicator.svelte'
import Drawer from '$lib/components/layout/Drawer.svelte'
import WarningBanner from './WarningBanner.svelte'
import { sleep } from '$lib/functions/common/promise'
Expand Down Expand Up @@ -360,7 +360,7 @@
{checkpointStatus}
onShowCheckpoints={() => (openDrawer = { kind: 'checkpoints' })}
/>
<TransactionStatus {metrics} class="w-full"></TransactionStatus>
<CommitProgressIndicator {metrics} class="w-full"></CommitProgressIndicator>
</div>
{#if metrics.current.views.size || metrics.current.tables.size}
<div class="flex flex-wrap gap-4">
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<script lang="ts">
import { slide } from 'svelte/transition'
import { match } from 'ts-pattern'
import type { PipelineMetrics } from '$lib/functions/pipelineMetrics'
import CommitProgressRow from './CommitProgressRow.svelte'

let { metrics, class: _class = '' }: { metrics: { current: PipelineMetrics }; class?: string } =
$props()

const global = $derived(metrics.current.global)
const transactionId = $derived(global.transaction_id)
const bootstrapPhase = $derived(global.concurrent_bootstrap_phase)

const transactionStatus = $derived(
match(global.transaction_status)
.with('TransactionInProgress', () => ({ label: 'Started', class: 'bg-tertiary-50-950' }))
.with('CommitInProgress', () => ({ label: 'Committing', class: 'bg-warning-200-800' }))
// A pipeline that has not reported metrics yet has no status at all.
.otherwise(() => null)
)

const bootstrapStatus = $derived(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please document this UI

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This got de-prioritized again. Let's process the PR as-is, and the docs will be a separate issue to not hold up the feature

match(bootstrapPhase)
.with('ConcurrentBootstrapping', () => ({
label: 'Backfilling',
class: 'bg-blue-200 dark:bg-blue-800'
}))
// Warning coloring flags the cutover pause, matching the pipeline status chip.
.with('Synchronizing', () => ({
label: 'Synchronizing',
class: 'preset-filled-warning-200-800'
}))
.otherwise(() => null)
)
</script>

<!-- The rows sit side by side while both fit at their natural width and wrap to
their own lines when they do not. Each row carries its own basis and cap, so
neither stretches to fill a line it has to itself. -->
<div class="flex w-full flex-wrap items-start gap-x-8 gap-y-4 {_class}" transition:slide>
<!-- The transaction row is always present, reporting "None" when no transaction
is running. -->
<CommitProgressRow
label="Transaction"
status={transactionStatus}
progress={global.commit_progress}
idle="disable"
resetKey={transactionId}
>
{#snippet detail()}
<div class="font-dm-mono text-sm text-nowrap">
<span class="select-none">ID:</span>{transactionId}
</div>
{/snippet}
</CommitProgressRow>
<CommitProgressRow
label="Bootstrapping"
status={bootstrapStatus}
progress={global.concurrent_bootstrap_progress}
idle="hide"
resetKey={bootstrapPhase}
/>
</div>
Original file line number Diff line number Diff line change
@@ -0,0 +1,215 @@
import { describe, expect, it } from 'vitest'
import { page } from 'vitest/browser'
import { render } from 'vitest-browser-svelte'
import type { PipelineMetrics } from '$lib/functions/pipelineMetrics'
import type {
CommitProgressSummary,
ConcurrentBootstrapPhase,
TransactionStatus
} from '$lib/services/manager'
import CommitProgressIndicator from './CommitProgressIndicator.svelte'

const progress = (completed: number, in_progress: number, remaining: number) =>
({
completed,
in_progress,
remaining,
in_progress_processed_records: 0,
in_progress_total_records: 0
}) satisfies CommitProgressSummary

function makeMetrics(global: {
transaction_status?: TransactionStatus
transaction_id?: number
commit_progress?: CommitProgressSummary | null
concurrent_bootstrap_phase?: ConcurrentBootstrapPhase
concurrent_bootstrap_progress?: CommitProgressSummary | null
}): { current: PipelineMetrics } {
return {
current: {
global: {
transaction_status: 'NoTransaction',
concurrent_bootstrap_phase: 'Inactive',
...global
}
} as PipelineMetrics
}
}

/**
* Height of the progress column - operator counts plus bar - of the transaction
* row, which is the first row of the tile.
*/
const transactionProgressHeight = (container: HTMLElement) => {
const transactionRow = container.firstElementChild!.firstElementChild!
return transactionRow.children[1].getBoundingClientRect().height
}

/** The tile's rows, in document order. */
const rows = (container: HTMLElement) => [...container.firstElementChild!.children] as HTMLElement[]

/** Renders both rows inside a container of exactly `width`. */
const renderBothRows = (width: string) => {
const result = render(CommitProgressIndicator, {
metrics: makeMetrics({
transaction_status: 'TransactionInProgress',
transaction_id: 1,
commit_progress: progress(3, 2, 5),
concurrent_bootstrap_phase: 'ConcurrentBootstrapping',
concurrent_bootstrap_progress: progress(1, 1, 2)
})
})
result.container.style.width = width
return rows(result.container)
}

const rem = () => parseFloat(getComputedStyle(document.documentElement).fontSize)

/** A row's natural width, from its `basis-150` and `max-w-150`. */
const rowMaxWidth = () => 37.5 * rem()

/** The gap between side-by-side rows, from the tile's `gap-x-8`. */
const rowGap = () => 2 * rem()

/**
* The row title, which also carries the detail below the `sm` breakpoint, so the
* transaction row reads "Transaction ID:7" rather than "Transaction" there.
*/
const rowTitle = (label: string) => page.getByText(new RegExp(`^${label}\\b`))

describe('CommitProgressIndicator.svelte', () => {
it('reports no transaction and no bootstrapping row while neither is in progress', async () => {
const { container } = render(CommitProgressIndicator, { metrics: makeMetrics({}) })
await expect.element(rowTitle('Transaction')).toBeInTheDocument()
await expect.element(page.getByText('None')).toBeInTheDocument()
await expect.element(rowTitle('Bootstrapping')).not.toBeInTheDocument()
// The transaction row holds its place; only the bootstrapping row comes and goes.
expect(rows(container)).toHaveLength(1)
})

describe('transaction only', () => {
const transacting = () =>
render(CommitProgressIndicator, {
metrics: makeMetrics({
transaction_status: 'CommitInProgress',
transaction_id: 7,
commit_progress: progress(3, 2, 5)
})
})

it('shows the transaction status, its ID and its operator counts', async () => {
await transacting()
await expect.element(rowTitle('Transaction')).toBeInTheDocument()
await expect.element(page.getByText('Committing')).toBeInTheDocument()
await expect.element(page.getByText('ID:7')).toBeInTheDocument()
await expect.element(page.getByText(/Completed\s*3\s*out of\s*10/)).toBeInTheDocument()
})

it('hides the bootstrapping row', async () => {
await transacting()
await expect.element(rowTitle('Bootstrapping')).not.toBeInTheDocument()
// Only the transaction row's overlaid pair of bars is rendered.
expect(page.getByRole('progressbar').elements()).toHaveLength(2)
})
})

describe('bootstrap only', () => {
const bootstrapping = () =>
render(CommitProgressIndicator, {
metrics: makeMetrics({
concurrent_bootstrap_phase: 'ConcurrentBootstrapping',
concurrent_bootstrap_progress: progress(1, 1, 2)
})
})

it('shows the bootstrapping status and its operator counts', async () => {
await bootstrapping()
await expect.element(rowTitle('Bootstrapping')).toBeInTheDocument()
await expect.element(page.getByText('Backfilling')).toBeInTheDocument()
await expect.element(page.getByText(/Completed\s*1\s*out of\s*4/)).toBeInTheDocument()
})

it('keeps the transaction row in place, reporting no transaction', async () => {
await bootstrapping()
await expect.element(rowTitle('Transaction')).toBeInTheDocument()
await expect.element(page.getByText('None')).toBeInTheDocument()
await expect.element(page.getByText('ID:')).not.toBeInTheDocument()
// Both rows keep their pair of bars, so neither shifts vertically.
expect(page.getByRole('progressbar').elements()).toHaveLength(4)
// Only the bootstrapping row reports operator counts.
expect(page.getByTestId('box-label-completed').elements()).toHaveLength(1)
})

it('keeps the transaction bar in place when it reports no operator counts', async () => {
const transacting = render(CommitProgressIndicator, {
metrics: makeMetrics({
transaction_status: 'CommitInProgress',
transaction_id: 7,
commit_progress: progress(3, 2, 5)
})
})
const heightWithCounts = transactionProgressHeight(transacting.container)
transacting.unmount()

const idle = render(CommitProgressIndicator, {
metrics: makeMetrics({ concurrent_bootstrap_phase: 'ConcurrentBootstrapping' })
})
// Dropping the placeholder for the hidden counts shortens the column,
// shifting the bar up.
expect(transactionProgressHeight(idle.container)).toBe(heightWithCounts)
})
})

it('flags the cutover pause while synchronizing', async () => {
await render(CommitProgressIndicator, {
metrics: makeMetrics({ concurrent_bootstrap_phase: 'Synchronizing' })
})
await expect.element(page.getByText('Synchronizing')).toBeInTheDocument()
})

it('shows both rows while a transaction and a bootstrap overlap', async () => {
await render(CommitProgressIndicator, {
metrics: makeMetrics({
transaction_status: 'TransactionInProgress',
transaction_id: 1,
concurrent_bootstrap_phase: 'ConcurrentBootstrapping'
})
})
await expect.element(page.getByText('Started')).toBeInTheDocument()
await expect.element(page.getByText('Backfilling')).toBeInTheDocument()
})

describe('row layout', () => {
it('puts both rows on one line when there is room for both', () => {
const [transaction, bootstrap] = renderBothRows(`${2 * rowMaxWidth() + 4 * rem()}px`)
expect(bootstrap.getBoundingClientRect().top).toBe(transaction.getBoundingClientRect().top)
})

it('separates side-by-side rows by gap-x-8', () => {
const [transaction, bootstrap] = renderBothRows(`${2 * rowMaxWidth() + 8 * rem()}px`)
const gap = bootstrap.getBoundingClientRect().left - transaction.getBoundingClientRect().right
expect(gap).toBeCloseTo(rowGap(), 1)
})

it('wraps the second row when both no longer fit', () => {
// One row short of the pair's combined width, so only the first fits.
const [transaction, bootstrap] = renderBothRows(`${1.5 * rowMaxWidth()}px`)
expect(bootstrap.getBoundingClientRect().top).toBeGreaterThanOrEqual(
transaction.getBoundingClientRect().bottom
)
})

it('keeps each row at its own max width rather than stretching', () => {
const [transaction, bootstrap] = renderBothRows('4000px')
expect(transaction.getBoundingClientRect().width).toBeCloseTo(rowMaxWidth(), 1)
expect(bootstrap.getBoundingClientRect().width).toBeCloseTo(rowMaxWidth(), 1)
})

it('shrinks a row below its max width in a narrow container', () => {
// The row only shrinks down to its own min-content width, since the
// operator counts do not wrap; below that it overflows rather than clip.
const [transaction] = renderBothRows(`${0.6 * rowMaxWidth()}px`)
expect(transaction.getBoundingClientRect().width).toBeLessThan(rowMaxWidth())
})
})
})
Loading
Loading