diff --git a/__tests__/html2/livestream/html-chunk.html b/__tests__/html2/livestream/html-chunk.html new file mode 100644 index 0000000000..e4a1abddd0 --- /dev/null +++ b/__tests__/html2/livestream/html-chunk.html @@ -0,0 +1,218 @@ + + + + HTML Streaming: Streaming Support + + + + + + + + + +
+ + + diff --git a/__tests__/html2/livestream/html-chunk.html.snap-1.png b/__tests__/html2/livestream/html-chunk.html.snap-1.png new file mode 100644 index 0000000000..bd12486652 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.html.snap-1.png differ diff --git a/__tests__/html2/livestream/html-chunk.html.snap-2.png b/__tests__/html2/livestream/html-chunk.html.snap-2.png new file mode 100644 index 0000000000..441632b2b2 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.html.snap-2.png differ diff --git a/__tests__/html2/livestream/html-chunk.html.snap-3.png b/__tests__/html2/livestream/html-chunk.html.snap-3.png new file mode 100644 index 0000000000..e52b47ada0 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.html.snap-3.png differ diff --git a/__tests__/html2/livestream/html-chunk.html.snap-4.png b/__tests__/html2/livestream/html-chunk.html.snap-4.png new file mode 100644 index 0000000000..d8e818def0 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.html.snap-4.png differ diff --git a/__tests__/html2/livestream/html-chunk.no-streaming.html b/__tests__/html2/livestream/html-chunk.no-streaming.html new file mode 100644 index 0000000000..096b61b60e --- /dev/null +++ b/__tests__/html2/livestream/html-chunk.no-streaming.html @@ -0,0 +1,10 @@ + + + + HTML Streaming: No Streaming Support + + + + diff --git a/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-1.png b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-1.png new file mode 100644 index 0000000000..bd12486652 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-1.png differ diff --git a/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-2.png b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-2.png new file mode 100644 index 0000000000..441632b2b2 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-2.png differ diff --git a/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-3.png b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-3.png new file mode 100644 index 0000000000..e52b47ada0 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-3.png differ diff --git a/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-4.png b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-4.png new file mode 100644 index 0000000000..d8e818def0 Binary files /dev/null and b/__tests__/html2/livestream/html-chunk.no-streaming.html.snap-4.png differ diff --git a/packages/api-graph/src/private/GraphProvider.tsx b/packages/api-graph/src/private/GraphProvider.tsx index fc8bb12357..7f23dc060e 100644 --- a/packages/api-graph/src/private/GraphProvider.tsx +++ b/packages/api-graph/src/private/GraphProvider.tsx @@ -76,17 +76,21 @@ function GraphProvider(props: GraphProviderProps) { }; }, [graph, setOrderedActivityNodes]); - const orderedActivitiesState = useMemo( - () => - Object.freeze([ - Object.freeze( - orderedActivityNodes.map( - node => node['urn:microsoft:webchat:direct-line-activity:raw-json'][0]['@value'] as WebChatActivity - ) - ) - ] as const), - [orderedActivityNodes] - ); + const orderedActivitiesState = useMemo(() => { + // Filter out stale graph nodes for activities no longer in Redux (e.g. pruned livestream revisions). + // The graph does not support deletion, so stale nodes linger after computeSortedActivities prunes them. + const { activities: storeActivities } = store.getState(); + const validActivitySet = new Set(storeActivities); + const activities: WebChatActivity[] = []; + + for (const node of orderedActivityNodes) { + const activity = node['urn:microsoft:webchat:direct-line-activity:raw-json'][0]['@value'] as WebChatActivity; + + validActivitySet.has(activity) && activities.push(activity); + } + + return Object.freeze([Object.freeze(activities)] as const); + }, [orderedActivityNodes, store]); const context = useMemo( () => diff --git a/packages/api/src/providers/ActivityKeyer/ActivityKeyerComposer.tsx b/packages/api/src/providers/ActivityKeyer/ActivityKeyerComposer.tsx index c1e04de5a3..6d9e6e6be7 100644 --- a/packages/api/src/providers/ActivityKeyer/ActivityKeyerComposer.tsx +++ b/packages/api/src/providers/ActivityKeyer/ActivityKeyerComposer.tsx @@ -1,8 +1,9 @@ import { getActivityLivestreamingMetadata, type WebChatActivity } from 'botframework-webchat-core'; -import React, { useCallback, useMemo, useRef, type ReactNode } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, type ReactNode } from 'react'; import reduceIterable from '../../hooks/private/reduceIterable'; import useActivities from '../../hooks/useActivities'; +import usePonyfill from '../Ponyfill/usePonyfill'; import type { ActivityKeyerContextType } from './private/Context'; import ActivityKeyerContext from './private/Context'; import getActivityId from './private/getActivityId'; @@ -17,6 +18,12 @@ type ActivityToKeyMap = Map; type ClientActivityIdToKeyMap = Map; type KeyToActivitiesMap = Map; +/** After this many ms of no activity changes, verify that the frozen portion was not modified. */ +const FROZEN_CHECK_TIMEOUT = 10_000; + +/** Only the last N activities are compared reference-by-reference on each render. */ +const MUTABLE_ACTIVITY_WINDOW = 1_000; + /** * React context composer component to assign a perma-key to every activity. * This will support both `useGetActivityByKey` and `useGetKeyByActivity` custom hooks. @@ -32,6 +39,7 @@ type KeyToActivitiesMap = Map; * Local key are only persisted in memory. On refresh, they will be a new random key. */ const ActivityKeyerComposer = ({ children }: Readonly<{ children?: ReactNode | undefined }>) => { + const [{ cancelIdleCallback, clearTimeout, requestIdleCallback, setTimeout }] = usePonyfill(); const existingContext = useActivityKeyerContext(false); if (existingContext) { @@ -39,13 +47,111 @@ const ActivityKeyerComposer = ({ children }: Readonly<{ children?: ReactNode | u } const [activities] = useActivities(); - const activityIdToKeyMapRef = useRef>(Object.freeze(new Map())); - const activityToKeyMapRef = useRef>(Object.freeze(new Map())); - const clientActivityIdToKeyMapRef = useRef>(Object.freeze(new Map())); - const keyToActivitiesMapRef = useRef>(Object.freeze(new Map())); - // TODO: [P1] `useMemoWithPrevious` to check and cache the resulting array if it hasn't changed. + // Maps are intentionally mutable so the incremental fast path can append to them in-place. + const activityIdToKeyMapRef = useRef(new Map()); + const activityToKeyMapRef = useRef(new Map()); + const clientActivityIdToKeyMapRef = useRef(new Map()); + const keyToActivitiesMapRef = useRef(new Map()); + const prevActivitiesRef = useRef(Object.freeze([])); + const prevActivityKeysStateRef = useRef( + Object.freeze([Object.freeze([])]) as readonly [readonly string[]] + ); + const pendingFrozenCheckRef = useRef< + | { + readonly current: readonly WebChatActivity[]; + readonly frozenBoundary: number; + readonly prev: readonly WebChatActivity[]; + } + | undefined + >(); + const warnedPositionsRef = useRef>(new Set()); + + // Incremental keying: the fast path only processes newly-appended activities (O(delta) per render) + // instead of re-iterating all activities (O(n) per render, O(n²) total for n streaming pushes). const activityKeysState = useMemo(() => { + const prevActivities = prevActivitiesRef.current; + + // Only the last MUTABLE_ACTIVITY_WINDOW activities are compared each render. + // Activities before the frozen boundary are assumed unchanged — O(1) instead of O(n). + const frozenBoundary = Math.max(0, Math.min(prevActivities.length, activities.length) - MUTABLE_ACTIVITY_WINDOW); + let commonPrefixLength = frozenBoundary; + const maxPrefix = Math.min(prevActivities.length, activities.length); + + // eslint-disable-next-line security/detect-object-injection + while (commonPrefixLength < maxPrefix && prevActivities[commonPrefixLength] === activities[commonPrefixLength]) { + commonPrefixLength++; + } + + // Schedule deferred verification of the frozen portion if any was skipped. + pendingFrozenCheckRef.current = frozenBoundary + ? Object.freeze({ current: activities, frozenBoundary, prev: prevActivities }) + : undefined; + + const isAppendOnly = commonPrefixLength === prevActivities.length; + + if (isAppendOnly) { + // Fast path: only new activities were appended — process them incrementally. + if (commonPrefixLength === activities.length) { + // Array reference changed but content is identical. + prevActivitiesRef.current = activities; + + return prevActivityKeysStateRef.current; + } + + const { current: activityIdToKeyMap } = activityIdToKeyMapRef; + const { current: activityToKeyMap } = activityToKeyMapRef; + const { current: clientActivityIdToKeyMap } = clientActivityIdToKeyMapRef; + const { current: keyToActivitiesMap } = keyToActivitiesMapRef; + + const newKeys: string[] = []; + + for (let i = commonPrefixLength; i < activities.length; i++) { + // eslint-disable-next-line security/detect-object-injection + const activity = activities[i]; + const activityId = getActivityId(activity); + const clientActivityId = getClientActivityId(activity); + const typingActivityId = getActivityLivestreamingMetadata(activity)?.sessionId; + + // Since we mutate maps in-place, a single lookup covers both "previous" and + // "current-iteration" entries — equivalent to the slow path's dual-map check. + const key = + (clientActivityId && clientActivityIdToKeyMap.get(clientActivityId)) || + (typingActivityId && activityIdToKeyMap.get(typingActivityId)) || + (activityId && activityIdToKeyMap.get(activityId)) || + activityToKeyMap.get(activity) || + uniqueId(); + + activityId && activityIdToKeyMap.set(activityId, key); + typingActivityId && activityIdToKeyMap.set(typingActivityId, key); + clientActivityId && clientActivityIdToKeyMap.set(clientActivityId, key); + activityToKeyMap.set(activity, key); + + const activitiesForKey = keyToActivitiesMap.get(key); + + keyToActivitiesMap.set( + key, + activitiesForKey ? Object.freeze([...activitiesForKey, activity]) : Object.freeze([activity]) + ); + + !activitiesForKey && newKeys.push(key); + } + + prevActivitiesRef.current = activities; + + if (!newKeys.length) { + return prevActivityKeysStateRef.current; + } + + const nextKeys = Object.freeze([...prevActivityKeysStateRef.current[0], ...newKeys]); + const result = Object.freeze([nextKeys]) as readonly [readonly string[]]; + + prevActivityKeysStateRef.current = result; + + return result; + } + + // Slow path: activities were removed or reordered — full recalculation. const { current: activityIdToKeyMap } = activityIdToKeyMapRef; const { current: activityToKeyMap } = activityToKeyMapRef; const { current: clientActivityIdToKeyMap } = clientActivityIdToKeyMapRef; @@ -76,20 +182,80 @@ const ActivityKeyerComposer = ({ children }: Readonly<{ children?: ReactNode | u nextActivityToKeyMap.set(activity, key); nextActivityKeys.add(key); - const activities = nextKeyToActivitiesMap.has(key) ? [...nextKeyToActivitiesMap.get(key)] : []; + const activitiesForKey = nextKeyToActivitiesMap.has(key) ? [...nextKeyToActivitiesMap.get(key)] : []; - activities.push(activity); - nextKeyToActivitiesMap.set(key, Object.freeze(activities)); + activitiesForKey.push(activity); + nextKeyToActivitiesMap.set(key, Object.freeze(activitiesForKey)); }); - activityIdToKeyMapRef.current = Object.freeze(nextActivityIdToKeyMap); - activityToKeyMapRef.current = Object.freeze(nextActivityToKeyMap); - clientActivityIdToKeyMapRef.current = Object.freeze(nextClientActivityIdToKeyMap); - keyToActivitiesMapRef.current = Object.freeze(nextKeyToActivitiesMap); + activityIdToKeyMapRef.current = nextActivityIdToKeyMap; + activityToKeyMapRef.current = nextActivityToKeyMap; + clientActivityIdToKeyMapRef.current = nextClientActivityIdToKeyMap; + keyToActivitiesMapRef.current = nextKeyToActivitiesMap; + prevActivitiesRef.current = activities; + + // Slow path did a full recalculation — no frozen check needed, reset warnings. + pendingFrozenCheckRef.current = undefined; + warnedPositionsRef.current.clear(); + + const nextKeys = Object.freeze([...nextActivityKeys.values()]); + const result = Object.freeze([nextKeys]) as readonly [readonly string[]]; + + prevActivityKeysStateRef.current = result; + + return result; + }, [ + activities, + activityIdToKeyMapRef, + activityToKeyMapRef, + clientActivityIdToKeyMapRef, + keyToActivitiesMapRef, + pendingFrozenCheckRef, + prevActivitiesRef, + prevActivityKeysStateRef, + warnedPositionsRef + ]); + + // Deferred verification: after FROZEN_CHECK_TIMEOUT of quiet, validate that activities + // inside the frozen portion have not actually changed. Warn once per position if they did. + // Uses requestIdleCallback inside the timeout to avoid contending with the first post-stream repaint. + useEffect(() => { + const pending = pendingFrozenCheckRef.current; + + if (!pending) { + return; + } + + let idleHandle: ReturnType> | undefined; + + const runCheck = () => { + const { current: currentActivities, frozenBoundary, prev: prevFrozenActivities } = pending; + + for (let i = 0; i < frozenBoundary; i++) { + // eslint-disable-next-line security/detect-object-injection + if (prevFrozenActivities[i] !== currentActivities[i] && !warnedPositionsRef.current.has(i)) { + warnedPositionsRef.current.add(i); + + console.warn( + `botframework-webchat internal: change in activity at position ${i} was not applied because it is outside the mutable window of ${MUTABLE_ACTIVITY_WINDOW}.` + ); + } + } + }; + + const timer = setTimeout(() => { + if (requestIdleCallback) { + idleHandle = requestIdleCallback(runCheck); + } else { + runCheck(); + } + }, FROZEN_CHECK_TIMEOUT); - // `nextActivityKeys` could potentially same as `prevActivityKeys` despite reference differences, we should memoize it. - return Object.freeze([Object.freeze([...nextActivityKeys.values()])]) as readonly [readonly string[]]; - }, [activities, activityIdToKeyMapRef, activityToKeyMapRef, clientActivityIdToKeyMapRef, keyToActivitiesMapRef]); + return () => { + clearTimeout(timer); + idleHandle !== undefined && cancelIdleCallback?.(idleHandle); + }; + }, [activities, cancelIdleCallback, clearTimeout, requestIdleCallback, setTimeout]); const getActivitiesByKey: (key?: string | undefined) => readonly WebChatActivity[] | undefined = useCallback( (key?: string | undefined): readonly WebChatActivity[] | undefined => key && keyToActivitiesMapRef.current.get(key), diff --git a/packages/bundle/src/markdown/createStreamingRenderer.ts b/packages/bundle/src/markdown/createStreamingRenderer.ts new file mode 100644 index 0000000000..74b1be7479 --- /dev/null +++ b/packages/bundle/src/markdown/createStreamingRenderer.ts @@ -0,0 +1,350 @@ +/* eslint-disable no-magic-numbers */ +import katex from 'katex'; +import { compile, parse, postprocess, preprocess } from 'micromark'; +import { gfm, gfmHtml } from 'micromark-extension-gfm'; +import type { Event, Options } from 'micromark-util-types'; + +import { math, mathHtml } from './mathExtension'; +import betterLinkDocumentMod from './private/betterLinkDocumentMod'; +import extractDefinitionsFromEvents, { type MarkdownLinkDefinition } from './private/extractDefinitionsFromEvents'; +import { pre as respectCRLFPre } from './private/respectCRLF'; +import { createDecorate } from './private/createDecorate'; + +type StreamingRenderInit = Readonly<{ + externalLinkAlt: string; +}>; + +type StreamingRenderOptions = Readonly<{ + markdownRenderHTML?: boolean | undefined; + markdownRespectCRLF: boolean; +}>; + +type StreamingNextOptions = Readonly<{ + container: HTMLElement; + containerClassName?: string | undefined; + transformFragment?: ((fragment: DocumentFragment) => DocumentFragment) | undefined; +}>; + +type StreamingNextResult = Readonly<{ + definitions: readonly MarkdownLinkDefinition[]; +}>; + +type StreamingRenderer = Readonly<{ + finalize: (options: StreamingNextOptions) => StreamingNextResult; + next: (chunk: string, options: StreamingNextOptions) => void; + reset: () => void; +}>; + +// Top-level block token types emitted by micromark. +// An exit event at depth 0 for one of these types marks a committed block boundary. +const TOP_LEVEL_BLOCK_TYPES: ReadonlySet = new Set([ + 'atxHeading', + 'blockQuote', + 'codeFenced', + 'codeIndented', + 'content', + 'htmlFlow', + 'listOrdered', + 'listUnordered', + 'setextHeading', + 'table', + 'thematicBreak', + 'math' +]); + +type BlockBoundary = { + readonly endOffset: number; + readonly startOffset: number; + readonly type: string; +}; + +function findTopLevelBlocks(events: ReadonlyArray): readonly BlockBoundary[] { + const blocks: Array<{ endOffset: number; startOffset: number; type: string }> = []; + let depth = 0; + + for (const [action, token] of events) { + if (!TOP_LEVEL_BLOCK_TYPES.has(token.type)) { + continue; + } + + if (action === 'enter') { + if (!depth) { + blocks.push({ endOffset: -1, startOffset: token.start.offset, type: token.type }); + } + + depth++; + } else { + depth--; + + if (!depth && blocks.length) { + blocks[blocks.length - 1].endOffset = token.end.offset; + } + } + } + + return blocks; +} + +export default function createStreamingRenderer( + { markdownRenderHTML, markdownRespectCRLF }: StreamingRenderOptions, + { externalLinkAlt }: StreamingRenderInit +): StreamingRenderer { + const micromarkOptions: Options = { + allowDangerousHtml: markdownRenderHTML ?? true, + allowDangerousProtocol: true, + extensions: [gfm(), math()], + htmlExtensions: [ + gfmHtml(), + mathHtml({ + renderMath: (content, isDisplay) => + katex.renderToString(content, { + displayMode: isDisplay, + output: 'mathml' + }) + }) + ] + }; + + const domParser = new DOMParser(); + + // Parser state. + let activeBlockStartOffset = 0; + let previousMarkdown = ''; + const emptyDefinitions: readonly MarkdownLinkDefinition[] = Object.freeze([]); + + // DOM reconciliation state. + let wrapperDiv: HTMLDivElement | null = null; + let activeSentinel: Comment | null = null; + + function parseEvents(source: string): Event[] { + return postprocess( + parse(micromarkOptions) + .document() + .write(preprocess()(source, undefined, true)) + ); + } + + function applyTransform( + fragment: DocumentFragment, + transformFragment: ((fragment: DocumentFragment) => DocumentFragment) | undefined + ): DocumentFragment { + return transformFragment ? transformFragment(fragment) : fragment; + } + + function ensureWrapper(container: HTMLElement, containerClassName: string | undefined): HTMLDivElement { + if (wrapperDiv && container.contains(wrapperDiv)) { + wrapperDiv.className = containerClassName || ''; + + return wrapperDiv; + } + + const wrapper = document.createElement('div'); + + wrapper.className = containerClassName || ''; + container.textContent = ''; + container.appendChild(wrapper); + wrapperDiv = wrapper; + activeSentinel = null; + + return wrapper; + } + + function renderNext(chunk: string, options: StreamingNextOptions): void { + previousMarkdown += chunk; + + if (!previousMarkdown) { + activeBlockStartOffset = 0; + activeSentinel = null; + + const wrapper = ensureWrapper(options.container, options.containerClassName); + + wrapper.replaceChildren(); + + return; + } + + let processedMarkdown = previousMarkdown; + + if (markdownRespectCRLF) { + processedMarkdown = respectCRLFPre(processedMarkdown); + } + + // Incremental path: re-parse only from the last committed block boundary. + // Guard: if sentinel was lost (e.g. container changed), fall back to full reparse. + if (activeBlockStartOffset > 0) { + const wrapper = ensureWrapper(options.container, options.containerClassName); + + if (activeSentinel && wrapper.contains(activeSentinel)) { + const tailEvents = parseEvents(processedMarkdown.slice(activeBlockStartOffset)); + const tailBlocks = findTopLevelBlocks(tailEvents); + const tailHTML = compile(micromarkOptions)(tailEvents); + + if (tailBlocks.length <= 1) { + // Fast path: active block grew, no new committed blocks. + // Replace only the active zone (after sentinel). + const activeDoc = domParser.parseFromString(tailHTML, 'text/html'); + const activeFragment = activeDoc.createDocumentFragment(); + + activeFragment.append(...Array.from(activeDoc.body.childNodes)); + betterLinkDocumentMod(activeFragment, createDecorate(emptyDefinitions, externalLinkAlt)); + + const activeRange = document.createRange(); + + activeRange.setStartAfter(activeSentinel); + activeRange.setEndAfter(wrapper.lastChild!); + activeRange.deleteContents(); + + wrapper.append(applyTransform(activeFragment, options.transformFragment)); + } else { + // New block boundary in tail: commit newly-finished blocks, replace active. + const newActiveOffsetInTail = tailBlocks[tailBlocks.length - 1].startOffset; + const committedTailEvents = tailEvents.filter(([, token]) => token.start.offset < newActiveOffsetInTail); + const committedTailHTML = compile(micromarkOptions)(committedTailEvents); + + activeBlockStartOffset += newActiveOffsetInTail; + + const committedDoc = domParser.parseFromString(committedTailHTML, 'text/html'); + const committedFragment = committedDoc.createDocumentFragment(); + + committedFragment.append(...Array.from(committedDoc.body.childNodes)); + betterLinkDocumentMod(committedFragment, createDecorate(emptyDefinitions, externalLinkAlt)); + + const remainingHTML = tailHTML.slice(committedTailHTML.length); + const activeDoc = domParser.parseFromString(remainingHTML, 'text/html'); + const activeFragment = activeDoc.createDocumentFragment(); + + activeFragment.append(...Array.from(activeDoc.body.childNodes)); + betterLinkDocumentMod(activeFragment, createDecorate(emptyDefinitions, externalLinkAlt)); + + // Remove old sentinel and active zone. + const tailRange = document.createRange(); + + tailRange.setStartBefore(activeSentinel); + tailRange.setEndAfter(wrapper.lastChild!); + tailRange.deleteContents(); + + // Append newly committed, new sentinel, active. + activeSentinel = document.createComment(''); + + wrapper.append( + applyTransform(committedFragment, options.transformFragment), + activeSentinel, + applyTransform(activeFragment, options.transformFragment) + ); + } + + return; + } + + // Sentinel lost — reset and fall through to full reparse. + activeBlockStartOffset = 0; + activeSentinel = null; + } + + // Full reparse path. + const fullEvents = parseEvents(processedMarkdown); + const rawHTML = compile(micromarkOptions)(fullEvents); + const parsedDocument = domParser.parseFromString(rawHTML, 'text/html'); + const fragment = parsedDocument.createDocumentFragment(); + + fragment.append(...Array.from(parsedDocument.body.childNodes)); + + const blocks = findTopLevelBlocks(fullEvents); + + if (blocks.length >= 2) { + activeBlockStartOffset = blocks[blocks.length - 1].startOffset; + + const range = document.createRange(); + + range.setStartBefore(fragment.firstChild!); + range.setEndBefore(fragment.lastElementChild!); + + const committedFragment = range.extractContents(); + const decorate = createDecorate(emptyDefinitions, externalLinkAlt); + + betterLinkDocumentMod(committedFragment, decorate); + betterLinkDocumentMod(fragment, decorate); + + const wrapper = ensureWrapper(options.container, options.containerClassName); + + activeSentinel = document.createComment(''); + + wrapper.replaceChildren( + applyTransform(committedFragment, options.transformFragment), + activeSentinel, + applyTransform(fragment, options.transformFragment) + ); + + return; + } + + // Single block — full replace, no sentinel. + activeBlockStartOffset = 0; + activeSentinel = null; + + betterLinkDocumentMod(fragment, createDecorate(emptyDefinitions, externalLinkAlt)); + + const wrapper = ensureWrapper(options.container, options.containerClassName); + + wrapper.replaceChildren(applyTransform(fragment, options.transformFragment)); + } + + return Object.freeze({ + finalize(options: StreamingNextOptions): StreamingNextResult { + if (!previousMarkdown) { + const wrapper = ensureWrapper(options.container, options.containerClassName); + + wrapper.replaceChildren(); + + return Object.freeze({ definitions: Object.freeze([]) }); + } + + let processedMarkdown = previousMarkdown; + + if (markdownRespectCRLF) { + processedMarkdown = respectCRLFPre(processedMarkdown); + } + + const fullEvents = parseEvents(processedMarkdown); + const rawHTML = compile(micromarkOptions)(fullEvents); + const parsedDocument = domParser.parseFromString(rawHTML, 'text/html'); + const fragment = parsedDocument.createDocumentFragment(); + + fragment.append(...Array.from(parsedDocument.body.childNodes)); + + const definitions = extractDefinitionsFromEvents(fullEvents); + + betterLinkDocumentMod(fragment, createDecorate(definitions, externalLinkAlt)); + + activeBlockStartOffset = 0; + activeSentinel = null; + + // Full replace on finalize — no incremental path needed. + const wrapper = ensureWrapper(options.container, options.containerClassName); + const transformedFragment = applyTransform(fragment, options.transformFragment); + + wrapper.replaceChildren(transformedFragment); + + return Object.freeze({ definitions }); + }, + + next(chunk: string, options: StreamingNextOptions): void { + renderNext(chunk, options); + }, + + reset(): void { + previousMarkdown = ''; + activeBlockStartOffset = 0; + activeSentinel = null; + wrapperDiv = null; + } + }); +} + +export { + type MarkdownLinkDefinition, + type StreamingNextOptions, + type StreamingNextResult, + type StreamingRenderer, + type StreamingRenderOptions +}; diff --git a/packages/bundle/src/markdown/middleware/createSanitizeMiddleware.ts b/packages/bundle/src/markdown/middleware/createSanitizeMiddleware.ts index 2e7a5b6e7b..d2504c828f 100644 --- a/packages/bundle/src/markdown/middleware/createSanitizeMiddleware.ts +++ b/packages/bundle/src/markdown/middleware/createSanitizeMiddleware.ts @@ -16,6 +16,18 @@ export default function createSanitizeMiddleware(): HTMLContentTransformMiddlewa return () => () => request => { const { documentFragment } = request; + // preserve top level comment nodes + for (const node of documentFragment.querySelectorAll('webchat-preserve-comment')) { + node.remove(); + } + for (const node of documentFragment.childNodes) { + if (node.nodeType === Node.COMMENT_NODE) { + const comment = document.createElement('webchat-preserve-comment'); + comment.textContent = node.nodeValue; + node.replaceWith(comment); + } + } + const htmlAfterBetterLink = serializeDocumentFragmentIntoString(documentFragment); const htmlAfterSanitization = sanitizeHTML(htmlAfterBetterLink, { @@ -25,9 +37,18 @@ export default function createSanitizeMiddleware(): HTMLContentTransformMiddlewa ([tag, { attributes }]) => [tag, Array.from(attributes)] satisfies [string, string[]] ) ) satisfies Record, - allowedTags: Array.from(request.allowedTags.keys() satisfies Iterator) satisfies string[] + allowedTags: ['webchat-preserve-comment'].concat( + Array.from(request.allowedTags.keys() satisfies Iterator) satisfies string[] + ) }); - return parseDocumentFragmentFromString(htmlAfterSanitization); + const parsed = parseDocumentFragmentFromString(htmlAfterSanitization); + + for (const node of parsed.querySelectorAll('webchat-preserve-comment')) { + const comment = document.createComment(node.textContent || ''); + node.replaceWith(comment); + } + + return parsed; }; } diff --git a/packages/bundle/src/markdown/private/createDecorate.ts b/packages/bundle/src/markdown/private/createDecorate.ts new file mode 100644 index 0000000000..7586d71eac --- /dev/null +++ b/packages/bundle/src/markdown/private/createDecorate.ts @@ -0,0 +1,80 @@ +import { onErrorResumeNext } from 'botframework-webchat-core'; +import { BetterLinkDocumentModDecoration } from './betterLinkDocumentMod'; +import { MarkdownLinkDefinition } from './extractDefinitionsFromEvents'; +import { sanitizeUri } from 'micromark-util-sanitize-uri'; + +export const ALLOWED_SCHEMES = ['data', 'http', 'https', 'ftp', 'mailto', 'sip', 'tel']; + +export function createDecorate( + definitions: readonly MarkdownLinkDefinition[], + externalLinkAlt: string +): (href: string, textContent: string) => BetterLinkDocumentModDecoration { + const linkDefinitions = definitions.map(definition => + Object.freeze({ + ...definition, + markupUrl: sanitizeUri(definition.url), + parsedUrl: onErrorResumeNext(() => new URL(definition.url)) + }) + ); + + return (href: string, textContent: string): BetterLinkDocumentModDecoration => { + const decoration: BetterLinkDocumentModDecoration = { + rel: 'noopener noreferrer', + target: '_blank', + wrapZeroWidthSpace: true + }; + + const ariaLabelSegments: string[] = [textContent]; + const classes: Set = new Set(); + const linkDefinition = linkDefinitions.find(({ url, markupUrl }) => url === href || (href && markupUrl === href)); + const protocol = onErrorResumeNext(() => new URL(href).protocol); + + if (linkDefinition) { + ariaLabelSegments.push(linkDefinition.title || linkDefinition?.parsedUrl?.host || linkDefinition.url); + + // linkDefinition.identifier is uppercase, while linkDefinition.label is as-is. + linkDefinition.label === textContent && classes.add('render-markdown__pure-identifier'); + } + + // Let javascript: fell through. Our sanitizer will catch and remove it from . + // Otherwise, it will be turn into