Skip to content

Commit 138de3c

Browse files
jernoblejernoble
committed
Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
https://bugs.webkit.org/show_bug.cgi?id=175835 <rdar://problem/34022234> Reviewed by Eric Carlson. Source/WebCore: Test: platform/mac/media/media-source/videoplaybackquality-decompressionsession.html Track the total number of frames decoded, dropped, & corrupted, as well as the total delay imposed by decoding in the WebCoreDecompressionSession. Drive-by fix: implement frame dropping by skipping frames whose presentation times are before the video's current time and which aren't depended upon by other frames. * platform/cf/CoreMediaSoftLink.cpp: * platform/cf/CoreMediaSoftLink.h: * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm: (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics): * platform/graphics/cocoa/WebCoreDecompressionSession.h: (WebCore::WebCoreDecompressionSession::totalVideoFrames): (WebCore::WebCoreDecompressionSession::droppedVideoFrames): (WebCore::WebCoreDecompressionSession::corruptedVideoFrames): (WebCore::WebCoreDecompressionSession::totalFrameDelay): * platform/graphics/cocoa/WebCoreDecompressionSession.mm: (WebCore::WebCoreDecompressionSession::shouldDecodeSample): (WebCore::WebCoreDecompressionSession::decodeSample): (WebCore::WebCoreDecompressionSession::handleDecompressionOutput): LayoutTests: * platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt: Added. * platform/mac/media/media-source/videoplaybackquality-decompressionsession.html: Added. Canonical link: https://commits.webkit.org/192552@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@221098 268f45cc-cd09-0410-ab3c-d52691b4dbfc
1 parent 4846733 commit 138de3c

10 files changed

Lines changed: 198 additions & 12 deletions

File tree

‎LayoutTests/ChangeLog‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,14 @@
1+
2017-08-23 Jer Noble <jer.noble@apple.com>
2+
3+
Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
4+
https://bugs.webkit.org/show_bug.cgi?id=175835
5+
<rdar://problem/34022234>
6+
7+
Reviewed by Eric Carlson.
8+
9+
* platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt: Added.
10+
* platform/mac/media/media-source/videoplaybackquality-decompressionsession.html: Added.
11+
112
2017-08-23 Matt Lewis <jlewis3@apple.com>
213

314
Removed flaky timeout expectation for inspector/codemirror/prettyprinting-css.html.

‎LayoutTests/media/media-source/media-source-loader.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
function MediaSourceLoader(url)
1+
function MediaSourceLoader(url, prefix)
22
{
33
this._url = url;
4+
this._prefix = prefix;
45
setTimeout(this.loadManifest.bind(this));
56

67
this.onload = null;
@@ -40,7 +41,8 @@ MediaSourceLoader.prototype = {
4041
loadMediaData: function()
4142
{
4243
var request = new XMLHttpRequest();
43-
request.open('GET', this._manifest.url, true);
44+
var url = (this._prefix ? this._prefix : '') + this._manifest.url
45+
request.open('GET', url, true);
4446
request.responseType = 'arraybuffer';
4547
request.onload = this.loadMediaDataSucceeded.bind(this);
4648
request.onerror = this.loadMediaDataFailed.bind(this);

‎LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
RUN(video.src = URL.createObjectURL(source))
2+
EVENT(sourceopen)
3+
RUN(source.duration = loader.duration())
4+
RUN(sourceBuffer = source.addSourceBuffer(loader.type()))
5+
RUN(sourceBuffer.appendBuffer(loader.initSegment()))
6+
EVENT(update)
7+
RUN(sourceBuffer.appendBuffer(loader.mediaSegment(0)))
8+
EVENT(update)
9+
RUN(canvas = document.createElement("canvas"))
10+
RUN(gl = canvas.getContext("webgl"))
11+
RUN(texture = gl.createTexture())
12+
RUN(gl.bindTexture(gl.TEXTURE_2D, texture))
13+
RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR))
14+
RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR))
15+
RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE))
16+
RUN(gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE))
17+
RUN(gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video))
18+
RUN(video.play())
19+
Wait for currentTime to advance beyond 0s.
20+
EVENT(timeupdate)
21+
EXPECTED (video.getVideoPlaybackQuality().totalVideoFrames > '0') OK
22+
END OF TEST
23+

‎LayoutTests/platform/mac/media/media-source/videoplaybackquality-decompressionsession.html‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
<!DOCTYPE html>
2+
<html>
3+
<head>
4+
<title>audio-session-category-track-change</title>
5+
<script src="../../../../media/video-test.js"></script>
6+
<script src="../../../../media/media-source/media-source-loader.js"></script>
7+
<script>
8+
mediaElement = video = document.createElement('video');
9+
10+
loader = new MediaSourceLoader('../../../../media/media-source/content/test-fragmented-manifest.json', '../../../../media/media-source/');
11+
loader.onerror = () => {
12+
failTest('Media data loading failed');
13+
};
14+
15+
loader.onload = () => {
16+
source = new MediaSource();
17+
waitForEvent('sourceopen', sourceOpen, false, false, source);
18+
waitForEventAndFail('error');
19+
run('video.src = URL.createObjectURL(source)');
20+
}
21+
22+
function sourceOpen() {
23+
run('source.duration = loader.duration()');
24+
run('sourceBuffer = source.addSourceBuffer(loader.type())');
25+
waitForEventOn(sourceBuffer, 'update', sourceInitialized, false, true);
26+
run('sourceBuffer.appendBuffer(loader.initSegment())');
27+
}
28+
29+
function sourceInitialized() {
30+
waitForEventOn(sourceBuffer, 'update', mediaSegmentAppended, false, true);
31+
run('sourceBuffer.appendBuffer(loader.mediaSegment(0))');
32+
}
33+
34+
function mediaSegmentAppended() {
35+
run('canvas = document.createElement("canvas")');
36+
run('gl = canvas.getContext("webgl")');
37+
run('texture = gl.createTexture()');
38+
run('gl.bindTexture(gl.TEXTURE_2D, texture)');
39+
run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)');
40+
run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)');
41+
run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)');
42+
run('gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)');
43+
run('gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video)');
44+
run('video.play()');
45+
consoleWrite('Wait for currentTime to advance beyond 0s.');
46+
video.addEventListener('timeupdate', timeupdate);
47+
}
48+
49+
function timeupdate() {
50+
if (video.currentTime == 0)
51+
return;
52+
53+
consoleWrite('EVENT(timeupdate)');
54+
testExpected('video.getVideoPlaybackQuality().totalVideoFrames', '0', '>');
55+
endTest();
56+
}
57+
58+
</script>
59+
</head>
60+
</html>

‎Source/WebCore/ChangeLog‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,33 @@
1+
2017-08-23 Jer Noble <jer.noble@apple.com>
2+
3+
Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
4+
https://bugs.webkit.org/show_bug.cgi?id=175835
5+
<rdar://problem/34022234>
6+
7+
Reviewed by Eric Carlson.
8+
9+
Test: platform/mac/media/media-source/videoplaybackquality-decompressionsession.html
10+
11+
Track the total number of frames decoded, dropped, & corrupted, as well as the total
12+
delay imposed by decoding in the WebCoreDecompressionSession.
13+
14+
Drive-by fix: implement frame dropping by skipping frames whose presentation times are
15+
before the video's current time and which aren't depended upon by other frames.
16+
17+
* platform/cf/CoreMediaSoftLink.cpp:
18+
* platform/cf/CoreMediaSoftLink.h:
19+
* platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
20+
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics):
21+
* platform/graphics/cocoa/WebCoreDecompressionSession.h:
22+
(WebCore::WebCoreDecompressionSession::totalVideoFrames):
23+
(WebCore::WebCoreDecompressionSession::droppedVideoFrames):
24+
(WebCore::WebCoreDecompressionSession::corruptedVideoFrames):
25+
(WebCore::WebCoreDecompressionSession::totalFrameDelay):
26+
* platform/graphics/cocoa/WebCoreDecompressionSession.mm:
27+
(WebCore::WebCoreDecompressionSession::shouldDecodeSample):
28+
(WebCore::WebCoreDecompressionSession::decodeSample):
29+
(WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
30+
131
2017-08-23 Andy Estes <aestes@apple.com>
232

333
[Payment Request] Update to "In Development" in features.json

‎Source/WebCore/platform/cf/CoreMediaSoftLink.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeAdd, CMTime, (CMTime tim
4848
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeGetSeconds, Float64, (CMTime time), (time))
4949
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMake, CMTime, (int64_t value, int32_t timescale), (value, timescale))
5050
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
51+
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
5152
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
5253
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeMake, CMTimeRange, (CMTime start, CMTime duration), (start, duration))
5354
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeEqual, Boolean, (CMTimeRange range1, CMTimeRange range2), (range1, range2))
@@ -155,7 +156,6 @@ SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMAudioClockCreate, OSStatus,
155156
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMaximum, CMTime, (CMTime time1, CMTime time2), (time1, time2))
156157
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMinimum, CMTime, (CMTime time1, CMTime time2), (time1, time2))
157158
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
158-
SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
159159

160160
SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)
161161
#endif // PLATFORM(IOS)

‎Source/WebCore/platform/cf/CoreMediaSoftLink.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeMake, CMTime, (int64_t v
5959
#define CMTimeMake softLink_CoreMedia_CMTimeMake
6060
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
6161
#define CMTimeMakeWithSeconds softLink_CoreMedia_CMTimeMakeWithSeconds
62+
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
63+
#define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
6264
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
6365
#define CMTimeRangeGetEnd softLink_CoreMedia_CMTimeRangeGetEnd
6466
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeMake, CMTimeRange, (CMTime start, CMTime duration), (start, duration))
@@ -263,8 +265,6 @@ SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeMinimum, CMTime, (CMTime
263265
#define CMTimeMinimum softLink_CoreMedia_CMTimeMinimum
264266
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
265267
#define CMTimeRangeContainsTime softLink_CoreMedia_CMTimeRangeContainsTime
266-
SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
267-
#define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
268268

269269
SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)
270270
#define kCMTimeIndefinite get_CoreMedia_kCMTimeIndefinite()

‎Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -689,6 +689,14 @@ static void CMTimebaseEffectiveRateChangedCallback(CMNotificationCenterRef, cons
689689

690690
std::optional<PlatformVideoPlaybackQualityMetrics> MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics()
691691
{
692+
if (m_decompressionSession) {
693+
return PlatformVideoPlaybackQualityMetrics(
694+
m_decompressionSession->totalVideoFrames(),
695+
m_decompressionSession->droppedVideoFrames(),
696+
m_decompressionSession->corruptedVideoFrames(),
697+
m_decompressionSession->totalFrameDelay().toDouble()
698+
);
699+
}
692700

693701
auto metrics = [m_sampleBufferDisplayLayer videoPerformanceMetrics];
694702
if (!metrics)

‎Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
7070
RetainPtr<CVPixelBufferRef> imageForTime(const MediaTime&, ImageForTimeFlags = ExactTime);
7171
void flush();
7272

73+
unsigned long totalVideoFrames() { return m_totalVideoFrames; }
74+
unsigned long droppedVideoFrames() { return m_droppedVideoFrames; }
75+
unsigned long corruptedVideoFrames() { return m_corruptedVideoFrames; }
76+
MediaTime totalFrameDelay() { return m_totalFrameDelay; }
77+
7378
private:
7479
WebCoreDecompressionSession();
7580

@@ -79,6 +84,7 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
7984
RetainPtr<CVPixelBufferRef> getFirstVideoFrame();
8085
void resetAutomaticDequeueTimer();
8186
void automaticDequeue();
87+
bool shouldDecodeSample(CMSampleBufferRef, bool displaying);
8288

8389
static void decompressionOutputCallback(void* decompressionOutputRefCon, void* sourceFrameRefCon, OSStatus, VTDecodeInfoFlags, CVImageBufferRef, CMTime presentationTimeStamp, CMTime presentationDuration);
8490
static CMTime getDecodeTime(CMBufferRef, void* refcon);
@@ -106,6 +112,10 @@ class WebCoreDecompressionSession : public ThreadSafeRefCounted<WebCoreDecompres
106112

107113
bool m_invalidated { false };
108114
int m_framesBeingDecoded { 0 };
115+
unsigned long m_totalVideoFrames { 0 };
116+
unsigned long m_droppedVideoFrames { 0 };
117+
unsigned long m_corruptedVideoFrames { 0 };
118+
MediaTime m_totalFrameDelay;
109119
};
110120

111121
}

‎Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm‎

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,33 @@
180180
});
181181
}
182182

183+
bool WebCoreDecompressionSession::shouldDecodeSample(CMSampleBufferRef sample, bool displaying)
184+
{
185+
if (!displaying)
186+
return true;
187+
188+
if (!m_timebase)
189+
return true;
190+
191+
auto currentTime = CMTimebaseGetTime(m_timebase.get());
192+
auto presentationTimeStamp = CMSampleBufferGetPresentationTimeStamp(sample);
193+
if (CMTimeCompare(presentationTimeStamp, currentTime) >= 0)
194+
return true;
195+
196+
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false);
197+
if (!attachments)
198+
return true;
199+
200+
for (CFIndex index = 0, count = CFArrayGetCount(attachments); index < count; ++index) {
201+
CFDictionaryRef attachmentDict = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, index);
202+
CFBooleanRef dependedOn = (CFBooleanRef)CFDictionaryGetValue(attachmentDict, kCMSampleAttachmentKey_IsDependedOnByOthers);
203+
if (dependedOn && !CFBooleanGetValue(dependedOn))
204+
return false;
205+
}
206+
207+
return true;
208+
}
209+
183210
void WebCoreDecompressionSession::decodeSample(CMSampleBufferRef sample, bool displaying)
184211
{
185212
if (isInvalidated())
@@ -212,6 +239,12 @@
212239
if (!displaying)
213240
flags |= kVTDecodeFrame_DoNotOutputFrame;
214241

242+
if (!shouldDecodeSample(sample, displaying)) {
243+
++m_totalVideoFrames;
244+
++m_droppedVideoFrames;
245+
return;
246+
}
247+
215248
VTDecompressionSessionDecodeFrame(m_decompressionSession.get(), sample, flags, reinterpret_cast<void*>(displaying), nullptr);
216249
}
217250

@@ -224,12 +257,15 @@
224257

225258
void WebCoreDecompressionSession::handleDecompressionOutput(bool displaying, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef rawImageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration)
226259
{
227-
UNUSED_PARAM(status);
228-
UNUSED_PARAM(infoFlags);
260+
++m_totalVideoFrames;
261+
if (infoFlags & kVTDecodeInfo_FrameDropped)
262+
++m_droppedVideoFrames;
229263

230264
CMVideoFormatDescriptionRef rawImageBufferDescription = nullptr;
231-
if (noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription))
265+
if (status != noErr || noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription)) {
266+
++m_corruptedVideoFrames;
232267
return;
268+
}
233269
RetainPtr<CMVideoFormatDescriptionRef> imageBufferDescription = adoptCF(rawImageBufferDescription);
234270

235271
CMSampleTimingInfo imageBufferTiming {
@@ -239,11 +275,17 @@
239275
};
240276

241277
CMSampleBufferRef rawImageSampleBuffer = nullptr;
242-
if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer))
278+
if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer)) {
279+
++m_corruptedVideoFrames;
243280
return;
244-
RefPtr<WebCoreDecompressionSession> protectedThis { this };
245-
RetainPtr<CMSampleBufferRef> imageSampleBuffer = adoptCF(rawImageSampleBuffer);
246-
dispatch_async(m_enqueingQueue.get(), [protectedThis, imageSampleBuffer, displaying] {
281+
}
282+
283+
auto currentTime = CMTimebaseGetTime(m_timebase.get());
284+
if (m_timebase && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
285+
m_totalFrameDelay += toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
286+
287+
dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
288+
UNUSED_PARAM(infoFlags);
247289
protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
248290
});
249291
}

0 commit comments

Comments
 (0)