diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 1ab7fd1f..2c18270d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -822,19 +822,18 @@ private void ensureStreamsAreCached() throws ExtractionException { cachedVideoStreams = new ArrayList<>(); cachedVideoOnlyStreams = new ArrayList<>(); final String selectedClient = NewPipe.getYoutubePlayerClient(); - if (("mweb".equals(selectedClient) || "web".equals(selectedClient)) - && streamType != StreamType.LIVE_STREAM - && streamType != StreamType.POST_LIVE_STREAM - && hasSabrStreamingUrl()) { + final boolean useSabr = ("mweb".equals(selectedClient) + || "web".equals(selectedClient)) && hasSabrStreamingUrl(); + if (useSabr) { buildSabrStreams(videoId); } else if (!("tv_downgraded".equals(selectedClient) && streamType == StreamType.LIVE_STREAM)) { extractAdaptiveFormats(videoId); } - if (streamType == StreamType.POST_LIVE_STREAM + if (!useSabr && (streamType == StreamType.POST_LIVE_STREAM || (streamType == StreamType.LIVE_STREAM && "tv_downgraded".equals(selectedClient)) - || "web_safari".equals(selectedClient)) { + || "web_safari".equals(selectedClient))) { tryExtractHlsStreams(videoId); } streamsCached = true; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataNormalizer.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataNormalizer.java new file mode 100644 index 00000000..9b80049e --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataNormalizer.java @@ -0,0 +1,170 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Locale; + +final class SabrMediaDataNormalizer { + private static final int MP4_HEADER_SIZE = 8; + private static final int MP4_EXTENDED_HEADER_SIZE = 16; + private static final long EBML_HEADER_ID = 0x1A45DFA3L; + private static final long WEBM_SEGMENT_ID = 0x18538067L; + private static final long WEBM_CLUSTER_ID = 0x1F43B675L; + + private SabrMediaDataNormalizer() { + } + + @Nullable + static SabrMediaDataParts split(@Nullable final String mimeType, + @Nonnull final byte[] data) { + if (mimeType == null) { + return null; + } + final String container = mimeType.split(";", 2)[0].trim().toLowerCase(Locale.ROOT); + if (container.endsWith("/mp4")) { + return splitMp4(data); + } + if (container.endsWith("/webm")) { + return splitWebm(data); + } + return null; + } + + @Nullable + private static SabrMediaDataParts splitMp4(@Nonnull final byte[] data) { + int offset = 0; + int initializationEnd = -1; + while (offset + MP4_HEADER_SIZE <= data.length) { + final long size32 = readUnsignedInt(data, offset); + final String type = new String(data, offset + 4, 4, StandardCharsets.US_ASCII); + final int headerSize = size32 == 1 ? MP4_EXTENDED_HEADER_SIZE : MP4_HEADER_SIZE; + if (offset + headerSize > data.length) { + return null; + } + final long size = size32 == 0 ? data.length - (long) offset + : size32 == 1 ? readUnsignedLong(data, offset + MP4_HEADER_SIZE) : size32; + if (size < headerSize || size > data.length - (long) offset) { + return null; + } + final int end = offset + (int) size; + if ("moov".equals(type)) { + initializationEnd = end; + } else if ("moof".equals(type) && initializationEnd > 0) { + return splitAt(data, initializationEnd); + } + offset = end; + } + return null; + } + + @Nullable + private static SabrMediaDataParts splitWebm(@Nonnull final byte[] data) { + final EbmlElement ebml = readEbmlElement(data, 0); + if (ebml == null || ebml.id != EBML_HEADER_ID || ebml.size < 0) { + return null; + } + final long segmentOffset = ebml.payloadOffset + ebml.size; + if (segmentOffset > Integer.MAX_VALUE) { + return null; + } + final EbmlElement segment = readEbmlElement(data, (int) segmentOffset); + if (segment == null || segment.id != WEBM_SEGMENT_ID) { + return null; + } + int offset = segment.payloadOffset; + while (offset < data.length) { + final EbmlElement element = readEbmlElement(data, offset); + if (element == null) { + return null; + } + if (element.id == WEBM_CLUSTER_ID) { + return offset <= segment.payloadOffset ? null : splitAt(data, offset); + } + if (element.size < 0) { + return null; + } + final long next = element.payloadOffset + element.size; + if (next <= offset || next > data.length) { + return null; + } + offset = (int) next; + } + return null; + } + + @Nullable + private static EbmlElement readEbmlElement(@Nonnull final byte[] data, final int offset) { + final int idLength = vintLength(data, offset, 4); + if (idLength < 0) { + return null; + } + final long id = readRawValue(data, offset, idLength); + final int sizeOffset = offset + idLength; + final int sizeLength = vintLength(data, sizeOffset, 8); + if (sizeLength < 0) { + return null; + } + final long rawSize = readRawValue(data, sizeOffset, sizeLength); + final long marker = 1L << (7 * sizeLength); + final long sizeValue = rawSize & (marker - 1); + final long size = sizeValue == marker - 1 ? -1 : sizeValue; + return new EbmlElement(id, size, sizeOffset + sizeLength); + } + + private static int vintLength(@Nonnull final byte[] data, final int offset, + final int maximum) { + if (offset < 0 || offset >= data.length) { + return -1; + } + final int first = data[offset] & 0xFF; + if (first == 0) { + return -1; + } + final int length = Integer.numberOfLeadingZeros(first) - 23; + return length >= 1 && length <= maximum && offset + length <= data.length ? length : -1; + } + + private static long readUnsignedInt(@Nonnull final byte[] data, final int offset) { + return (long) (data[offset] & 0xFF) << 24 + | (long) (data[offset + 1] & 0xFF) << 16 + | (long) (data[offset + 2] & 0xFF) << 8 + | data[offset + 3] & 0xFFL; + } + + private static long readUnsignedLong(@Nonnull final byte[] data, final int offset) { + final long value = readRawValue(data, offset, 8); + return value < 0 ? -1 : value; + } + + private static long readRawValue(@Nonnull final byte[] data, final int offset, + final int length) { + if (length < 1 || length > 8 || offset < 0 || offset + length > data.length) { + return -1; + } + long value = 0; + for (int i = 0; i < length; i++) { + value = (value << 8) | (data[offset + i] & 0xFFL); + } + return value; + } + + @Nonnull + private static SabrMediaDataParts splitAt(@Nonnull final byte[] data, final int offset) { + return new SabrMediaDataParts(Arrays.copyOfRange(data, 0, offset), + Arrays.copyOfRange(data, offset, data.length)); + } + + private static final class EbmlElement { + private final long id; + private final long size; + private final int payloadOffset; + + private EbmlElement(final long id, final long size, final int payloadOffset) { + this.id = id; + this.size = size; + this.payloadOffset = payloadOffset; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataParts.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataParts.java new file mode 100644 index 00000000..62716a82 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaDataParts.java @@ -0,0 +1,26 @@ +package org.schabi.newpipe.extractor.services.youtube.sabr; + +import javax.annotation.Nonnull; + +final class SabrMediaDataParts { + @Nonnull + private final byte[] initializationData; + @Nonnull + private final byte[] mediaData; + + SabrMediaDataParts(@Nonnull final byte[] initializationData, + @Nonnull final byte[] mediaData) { + this.initializationData = initializationData; + this.mediaData = mediaData; + } + + @Nonnull + byte[] getInitializationData() { + return initializationData; + } + + @Nonnull + byte[] getMediaData() { + return mediaData; + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java index 968b3e37..aa945842 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaHeader.java @@ -181,6 +181,26 @@ static SabrMediaHeader decode(@Nonnull final byte[] data) throws SabrProtocolExc sequenceLastModified); } + @Nonnull + static SabrMediaHeader initializationFrom(@Nonnull final SabrMediaHeader source, + final int contentLength) { + return new SabrMediaHeader(source.headerId, source.videoId, source.itag, + source.lastModified, source.xtags, source.startRange, + source.compressionAlgorithm, true, -1, source.bitrateBps, 0, 0, + contentLength, 0, 0, source.timeRangeTimescale, source.sequenceLastModified); + } + + @Nonnull + static SabrMediaHeader mediaFrom(@Nonnull final SabrMediaHeader source, + final int contentLength) { + return new SabrMediaHeader(source.headerId, source.videoId, source.itag, + source.lastModified, source.xtags, source.startRange, + source.compressionAlgorithm, false, source.sequenceNumber, source.bitrateBps, + source.startMs, source.durationMs, contentLength, source.timeRangeStartTicks, + source.timeRangeDurationTicks, source.timeRangeTimescale, + source.sequenceLastModified); + } + @Nonnull private static FormatId decodeFormatId(@Nonnull final byte[] data) throws SabrProtocolException { int itag = -1; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaSegmentCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaSegmentCollector.java index c29c4c5a..0b20ab35 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaSegmentCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrMediaSegmentCollector.java @@ -145,6 +145,10 @@ public SabrMediaSegment onMediaEnd(@Nonnull final byte[] partData) return null; } + public boolean hasOpenSegments() { + return !openSegments.isEmpty(); + } + public void abort() { for (final OpenSegment segment : openSegments.values()) { segment.abort(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrStreamingResponseReader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrStreamingResponseReader.java index 4a3f8e13..592e63e2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrStreamingResponseReader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/SabrStreamingResponseReader.java @@ -42,6 +42,11 @@ public interface StoppableSegmentConsumer { boolean accept(@Nonnull SabrMediaSegment segment) throws SabrProtocolException; } + @FunctionalInterface + public interface LiveMetadataConsumer { + void accept(@Nonnull SabrLiveMetadata metadata) throws SabrProtocolException; + } + /** * Streams completed segments directly to {@code segmentConsumer}. When a consumer is supplied, * completed segments are not retained by the result, bounding the response reader to one open @@ -78,7 +83,7 @@ public static Result readUntil(@Nonnull final InputStream in, @Nullable final SegmentConsumer segmentStartConsumer, @Nullable final File spoolDirectory) throws SabrProtocolException, IOException { - return readUntil(in, segmentConsumer, segmentStartConsumer, spoolDirectory, + return readUntil(in, segmentConsumer, segmentStartConsumer, null, spoolDirectory, SabrMediaProtocol.builtin()); } @@ -89,6 +94,29 @@ public static Result readUntil(@Nonnull final InputStream in, @Nullable final File spoolDirectory, @Nonnull final SabrMediaProtocol mediaProtocol) throws SabrProtocolException, IOException { + return readUntil(in, segmentConsumer, segmentStartConsumer, null, spoolDirectory, + mediaProtocol); + } + + @Nonnull + public static Result readUntil(@Nonnull final InputStream in, + final StoppableSegmentConsumer segmentConsumer, + @Nullable final SegmentConsumer segmentStartConsumer, + @Nullable final LiveMetadataConsumer liveMetadataConsumer, + @Nullable final File spoolDirectory) + throws SabrProtocolException, IOException { + return readUntil(in, segmentConsumer, segmentStartConsumer, liveMetadataConsumer, + spoolDirectory, SabrMediaProtocol.builtin()); + } + + @Nonnull + public static Result readUntil(@Nonnull final InputStream in, + final StoppableSegmentConsumer segmentConsumer, + @Nullable final SegmentConsumer segmentStartConsumer, + @Nullable final LiveMetadataConsumer liveMetadataConsumer, + @Nullable final File spoolDirectory, + @Nonnull final SabrMediaProtocol mediaProtocol) + throws SabrProtocolException, IOException { final List controlParts = new ArrayList<>(); final List partSummaries = new ArrayList<>(); final List segments = new ArrayList<>(); @@ -100,6 +128,7 @@ public static Result readUntil(@Nonnull final InputStream in, final long[] maxPartBytes = {0}; final long[] maxMediaPartPayloadBytes = {0}; final long[] maxSegmentBytes = {0}; + final boolean[] stopAfterOpenSegments = {false}; // headerId -> total media bytes seen, so the decoded response passes the same integrity check // (getIntegrityIssues -> "missing-media") as the buffered path WITHOUT retaining the bytes. final Map mediaBytesByHeaderId = new HashMap<>(); @@ -160,16 +189,23 @@ public static Result readUntil(@Nonnull final InputStream in, maxSegmentBytes[0] = Math.max(maxSegmentBytes[0], segment.getLength()); if (segmentConsumer == null) { segments.add(segment); - } else { - return segmentConsumer.accept(segment); + } else if (!segmentConsumer.accept(segment)) { + stopAfterOpenSegments[0] = true; } } + } else if (type == SabrResponseDecoder.LIVE_METADATA) { + final byte[] payload = readPayloadBytes(payloadStream, size); + controlPayloadBytes[0] += payload.length; + controlParts.add(new UmpPart(type, payload.length, payload)); + if (liveMetadataConsumer != null) { + liveMetadataConsumer.accept(SabrLiveMetadata.decode(payload)); + } } else { final byte[] payload = readPayloadBytes(payloadStream, size); controlPayloadBytes[0] += payload.length; controlParts.add(new UmpPart(type, payload.length, payload)); } - return true; + return !stopAfterOpenSegments[0] || collector.hasOpenSegments(); }); } finally { // Any header left open after EOF, cancellation or failure must wake a growing-file diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java index ef03ff49..657f8a32 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrProbe.java @@ -263,10 +263,29 @@ static YoutubeSabrProbeResult probeFirstMediaResponseStreamingUntil( @Nullable final File segmentSpoolDirectory, @Nonnull final Localization localization) throws IOException, ExtractionException { + return probeFirstMediaResponseStreamingUntil(info, audioFormat, videoFormat, streamState, + serverAbrStreamingUrlOverride, segmentConsumer, segmentStartConsumer, null, + segmentSpoolDirectory, localization); + } + + @Nonnull + static YoutubeSabrProbeResult probeFirstMediaResponseStreamingUntil( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nullable final YoutubeSabrStreamState streamState, + @Nullable final String serverAbrStreamingUrlOverride, + @Nonnull final SabrStreamingResponseReader.StoppableSegmentConsumer segmentConsumer, + @Nonnull final SabrStreamingResponseReader.SegmentConsumer segmentStartConsumer, + @Nullable final SabrStreamingResponseReader.LiveMetadataConsumer liveMetadataConsumer, + @Nullable final File segmentSpoolDirectory, + @Nonnull final Localization localization) + throws IOException, ExtractionException { final byte[] requestBody = YoutubeSabrRequestBuilder.buildFirstMediaRequest( info, audioFormat, videoFormat, streamState); return postMediaRequest(info, requestBody, 0, serverAbrStreamingUrlOverride, - segmentConsumer, segmentStartConsumer, segmentSpoolDirectory, localization); + segmentConsumer, segmentStartConsumer, liveMetadataConsumer, + segmentSpoolDirectory, localization); } @Nonnull @@ -374,13 +393,33 @@ static YoutubeSabrProbeResult probeFollowUpMediaResponseStreamingUntil( @Nullable final File segmentSpoolDirectory, @Nonnull final Localization localization) throws IOException, ExtractionException { + return probeFollowUpMediaResponseStreamingUntil(info, audioFormat, videoFormat, streamState, + requestNumber, serverAbrStreamingUrlOverride, segmentConsumer, + segmentStartConsumer, null, segmentSpoolDirectory, localization); + } + + @Nonnull + static YoutubeSabrProbeResult probeFollowUpMediaResponseStreamingUntil( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final YoutubeSabrFormat audioFormat, + @Nonnull final YoutubeSabrFormat videoFormat, + @Nonnull final YoutubeSabrStreamState streamState, + final int requestNumber, + @Nullable final String serverAbrStreamingUrlOverride, + @Nonnull final SabrStreamingResponseReader.StoppableSegmentConsumer segmentConsumer, + @Nonnull final SabrStreamingResponseReader.SegmentConsumer segmentStartConsumer, + @Nullable final SabrStreamingResponseReader.LiveMetadataConsumer liveMetadataConsumer, + @Nullable final File segmentSpoolDirectory, + @Nonnull final Localization localization) + throws IOException, ExtractionException { if (requestNumber <= 0) { throw new SabrProtocolException("Follow-up request number must be positive"); } final byte[] requestBody = YoutubeSabrRequestBuilder.buildFollowUpMediaRequest( info, audioFormat, videoFormat, streamState); return postMediaRequest(info, requestBody, requestNumber, serverAbrStreamingUrlOverride, - segmentConsumer, segmentStartConsumer, segmentSpoolDirectory, localization); + segmentConsumer, segmentStartConsumer, liveMetadataConsumer, + segmentSpoolDirectory, localization); } @Nonnull @@ -444,8 +483,41 @@ static YoutubeSabrProbeResult postMediaRequest( @Nonnull final Localization localization) throws IOException, ExtractionException { return postMediaRequest(info, requestBody, requestNumber, serverAbrStreamingUrlOverride, - segmentConsumer, segmentStartConsumer, segmentSpoolDirectory, localization, - SabrMediaProtocol.builtin()); + segmentConsumer, segmentStartConsumer, null, segmentSpoolDirectory, localization); + } + + @Nonnull + static YoutubeSabrProbeResult postMediaRequest( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final byte[] requestBody, + final int requestNumber, + @Nullable final String serverAbrStreamingUrlOverride, + @Nullable final SabrStreamingResponseReader.StoppableSegmentConsumer segmentConsumer, + @Nullable final SabrStreamingResponseReader.SegmentConsumer segmentStartConsumer, + @Nullable final File segmentSpoolDirectory, + @Nonnull final Localization localization, + @Nonnull final SabrMediaProtocol mediaProtocol) + throws IOException, ExtractionException { + return postMediaRequest(info, requestBody, requestNumber, serverAbrStreamingUrlOverride, + segmentConsumer, segmentStartConsumer, null, segmentSpoolDirectory, localization, + mediaProtocol); + } + + @Nonnull + static YoutubeSabrProbeResult postMediaRequest( + @Nonnull final YoutubeSabrInfo info, + @Nonnull final byte[] requestBody, + final int requestNumber, + @Nullable final String serverAbrStreamingUrlOverride, + @Nullable final SabrStreamingResponseReader.StoppableSegmentConsumer segmentConsumer, + @Nullable final SabrStreamingResponseReader.SegmentConsumer segmentStartConsumer, + @Nullable final SabrStreamingResponseReader.LiveMetadataConsumer liveMetadataConsumer, + @Nullable final File segmentSpoolDirectory, + @Nonnull final Localization localization) + throws IOException, ExtractionException { + return postMediaRequest(info, requestBody, requestNumber, serverAbrStreamingUrlOverride, + segmentConsumer, segmentStartConsumer, liveMetadataConsumer, segmentSpoolDirectory, + localization, SabrMediaProtocol.builtin()); } @Nonnull @@ -456,6 +528,7 @@ static YoutubeSabrProbeResult postMediaRequest( @Nullable final String serverAbrStreamingUrlOverride, @Nullable final SabrStreamingResponseReader.StoppableSegmentConsumer segmentConsumer, @Nullable final SabrStreamingResponseReader.SegmentConsumer segmentStartConsumer, + @Nullable final SabrStreamingResponseReader.LiveMetadataConsumer liveMetadataConsumer, @Nullable final File segmentSpoolDirectory, @Nonnull final Localization localization, @Nonnull final SabrMediaProtocol mediaProtocol) @@ -499,10 +572,11 @@ static YoutubeSabrProbeResult postMediaRequest( final CountingInputStream body = new CountingInputStream(response.body()); final SabrStreamingResponseReader.Result streamed = timedConsumer == null && timedStartConsumer == null - ? SabrStreamingResponseReader.readUntil(body, null, null, null, - mediaProtocol) + ? SabrStreamingResponseReader.readUntil(body, null, null, + liveMetadataConsumer, null, mediaProtocol) : SabrStreamingResponseReader.readUntil(body, timedConsumer, - timedStartConsumer, segmentSpoolDirectory, mediaProtocol); + timedStartConsumer, liveMetadataConsumer, + segmentSpoolDirectory, mediaProtocol); final long requestElapsedMs = elapsedMs(requestStartNs); return new YoutubeSabrProbeResult(info, streamed.getDecodedResponse(), streamed.getSegments(), streamed.getSegmentCount(), response.responseCode(), diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrRequestBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrRequestBuilder.java index 759571b1..f31a7ff2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrRequestBuilder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrRequestBuilder.java @@ -50,10 +50,13 @@ private static byte[] buildFirstMediaRequestLocked(@Nonnull final YoutubeSabrInf final List bufferedRanges = streamState == null ? java.util.Collections.emptyList() : streamState.getBufferedRanges(); - final boolean includeInitialPlaybackState = playerTimeMs > 0 || !bufferedRanges.isEmpty(); + final boolean forcedInitialPlaybackState = streamState != null + && streamState.shouldWriteFirstRequestPlaybackState(); + final boolean includeInitialPlaybackState = forcedInitialPlaybackState + || playerTimeMs > 0 || !bufferedRanges.isEmpty(); final SabrProto.Writer request = new SabrProto.Writer(); request.writeMessage(1, buildClientAbrState(audioFormat, videoFormat, playerTimeMs, - includeInitialPlaybackState, + includeInitialPlaybackState && !forcedInitialPlaybackState, streamState == null ? ENABLED_TRACK_TYPES_VIDEO_AND_AUDIO : streamState.getEnabledTrackTypesBitfield(), @@ -270,21 +273,21 @@ private static void writePreferredFormats(@Nonnull final SabrProto.Writer reques return; } for (final YoutubeSabrFormat format : info.getFormats()) { - if (format.isAudio() && streamState.shouldSelectAudioFormat()) { + if (format.isAudio() && streamState.shouldPreferAudioFormat()) { request.writeMessage(16, SabrProto.formatId(format)); } } for (final YoutubeSabrFormat format : info.getFormats()) { - if (format.isVideo() && streamState.shouldSelectVideoFormat()) { + if (format.isVideo() && streamState.shouldPreferVideoFormat()) { request.writeMessage(17, SabrProto.formatId(format)); } } return; } - if (streamState == null || streamState.shouldSelectAudioFormat()) { + if (streamState == null || streamState.shouldPreferAudioFormat()) { request.writeMessage(16, SabrProto.formatId(audioFormat)); } - if (streamState == null || streamState.shouldSelectVideoFormat()) { + if (streamState == null || streamState.shouldPreferVideoFormat()) { request.writeMessage(17, SabrProto.formatId(videoFormat)); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java index dacbd90e..d2ca23af 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrSession.java @@ -182,6 +182,23 @@ public SabrMediaSegment fetchSegment(@Nonnull final SabrSegmentRequest request, if (cachedSegment != null) { return cachedSegment; } + final boolean initializationSegment = request.isInitializationSegment(); + if (initializationSegment) { + prepareInitializationRequest(request.getFormat()); + } + try { + return fetchUncachedSegment(request, localization); + } finally { + if (initializationSegment) { + clearInitializationRequest(); + } + } + } + + @Nonnull + private SabrMediaSegment fetchUncachedSegment(@Nonnull final SabrSegmentRequest request, + @Nonnull final Localization localization) + throws IOException, ExtractionException { failIfKnownOutOfBounds(request); boolean targetPrepared = maybePrepareForDistantMediaSegment(request); @@ -244,6 +261,28 @@ public SabrMediaSegment fetchSegment(@Nonnull final SabrSegmentRequest request, : ":seq=" + request.getSequenceNumber())); } + private void prepareInitializationRequest(@Nonnull final YoutubeSabrFormat format) { + streamState.setWriteFirstRequestPlaybackState(true); + streamState.setWriteTopLevelPlayerTimeMs(false); + streamState.setWriteLastManualSelectedResolution(format.isVideo()); + streamState.setBufferedRangesOverride(Collections.emptyList()); + streamState.setRequestTrackMode(format.isVideo() + ? YoutubeSabrStreamState.TRACK_MODE_VIDEO_ONLY + : YoutubeSabrStreamState.TRACK_MODE_AUDIO_ONLY, + false, false); + streamState.setPreferredTrackTypes(true, true); + streamState.setPlayerTimeMs(streamState.getPlayerTimeMs()); + } + + private void clearInitializationRequest() { + streamState.setWriteFirstRequestPlaybackState(false); + streamState.setWriteTopLevelPlayerTimeMs(true); + streamState.setWriteLastManualSelectedResolution(false); + streamState.setBufferedRangesOverride(null); + streamState.clearPlayerTimeMsOverride(); + streamState.setActiveTrackTypes(true, true); + } + @Nonnull public YoutubeSabrProbeResult fetchNextResponse(@Nonnull final Localization localization) throws IOException, ExtractionException { @@ -304,11 +343,14 @@ private YoutubeSabrProbeResult fetchNextResponseUntil( traceCurrentSegmentElapsedMs = -1; } }; + final SabrStreamingResponseReader.LiveMetadataConsumer liveMetadataConsumer = metadata -> { + streamState.ingestLiveMetadata(metadata); + }; final YoutubeSabrProbeResult result; try { result = YoutubeSabrProbe.postMediaRequest(info, requestBody, requestNumber, - serverAbrStreamingUrl, timedConsumer, startedConsumer, segmentSpoolDirectory, - localization, sessionPolicyHost.getMediaProtocol()); + serverAbrStreamingUrl, timedConsumer, startedConsumer, liveMetadataConsumer, + segmentSpoolDirectory, localization, sessionPolicyHost.getMediaProtocol()); } catch (final IOException | ExtractionException e) { addDiagnosticEvent("request_failed n=" + requestNumber + " type=" + e.getClass().getSimpleName() @@ -331,7 +373,9 @@ private YoutubeSabrProbeResult fetchNextResponseUntil( totalResponseBytes += result.getResponseBytes(); recordMemoryStats(result); recordTraceResponse(result); - updateBandwidthEstimate(result.getResponseBytes(), System.nanoTime() - requestStartNs); + if (result.getSegmentCount() > 0) { + updateBandwidthEstimate(result.getResponseBytes(), System.nanoTime() - requestStartNs); + } requestNumber++; sessionPolicyHost.commitAppliedState(requestPolicyResult, sessionPolicyState()); return result; @@ -447,11 +491,10 @@ public int pumpOnceStreaming(@Nonnull final Localization localization) /** * Like {@link #pumpOnceStreaming(Localization)}, but used by callers that are waiting on a - * concrete segment. Keep consuming the whole response: SABR/UMP response boundaries are part of - * the protocol state, and closing after the target segment can cut off a following media header - * whose body/end is still in the same response. Do not honor long server backoff here: the - * player loader is synchronously waiting for {@code target}, and the client data-source recovery - * loop needs short retries instead of a minutes-long buffering sleep. + * concrete segment. Stop once the target and every media segment already open in the response + * are complete, so a live response cannot hold the pump indefinitely. Do not honor long server + * backoff here: the player loader is synchronously waiting for {@code target}, and the client + * data-source recovery loop needs short retries instead of a minutes-long buffering sleep. */ public int pumpOnceStreamingUntilCached(@Nonnull final Localization localization, @Nonnull final SabrSegmentRequest target) @@ -489,7 +532,7 @@ public DemandResponseResult pumpOnceStreamingForDemand( } else if (!header.isInitSegment()) { returnedSegmentsTruncated[0] = true; } - return true; + return !target.matches(header); }, false); return new DemandResponseResult(result == null ? returnedSegments.size() : result.getSegmentCount(), targetTrackSegments[0], returnedSegments, @@ -759,13 +802,19 @@ private static void validateRedirectUrl(@Nonnull final String redirectUrl) } } - private void ingestAndCacheSegment(@Nonnull final SabrMediaSegment segment) { - final String key = cacheKey(segment); - if (cacheClosed || !segment.isComplete() || segment.hasFailed()) { - inFlightSegments.remove(key, segment); - segment.delete(); + private void ingestAndCacheSegment(@Nonnull final SabrMediaSegment sourceSegment) { + final String sourceKey = cacheKey(sourceSegment); + if (cacheClosed || !sourceSegment.isComplete() || sourceSegment.hasFailed()) { + inFlightSegments.remove(sourceKey, sourceSegment); + sourceSegment.delete(); return; } + final SabrMediaSegment segment = normalizeMediaSegment(sourceSegment); + inFlightSegments.remove(sourceKey, sourceSegment); + if (segment == null) { + return; + } + final String key = cacheKey(segment); streamState.ingest(segment); inFlightSegments.remove(key, segment); final SabrMediaSegment previous = segmentCache.putIfAbsent(key, segment); @@ -796,10 +845,55 @@ private void ingestAndCacheSegment(@Nonnull final SabrMediaSegment segment) { evictCacheIfNeeded(); } + @Nullable + private SabrMediaSegment normalizeMediaSegment(@Nonnull final SabrMediaSegment segment) { + final SabrMediaHeader header = segment.getHeader(); + if (header.isInitSegment()) { + return segment; + } + if (!streamState.isLive()) { + return segment; + } + final YoutubeSabrFormat format = formatForItag(header.getItag()); + if (format == null) { + return segment; + } + final SabrMediaDataParts parts = SabrMediaDataNormalizer.split( + format.getMimeType(), segment.getData()); + if (parts == null) { + return segment; + } + if (!streamState.isInitialized(format)) { + final byte[] initializationData = parts.getInitializationData(); + ingestAndCacheSegment(new SabrMediaSegment( + SabrMediaHeader.initializationFrom(header, initializationData.length), + initializationData)); + } + segment.delete(); + final byte[] mediaData = parts.getMediaData(); + return new SabrMediaSegment( + SabrMediaHeader.mediaFrom(header, mediaData.length), mediaData); + } + + @Nullable + private YoutubeSabrFormat formatForItag(final int itag) { + if (audioFormat.getItag() == itag) { + return audioFormat; + } + return videoFormat.getItag() == itag ? videoFormat : null; + } + private void publishInFlightSegment(@Nonnull final SabrMediaSegment segment) { if (segment.isComplete() || segment.getHeader().isInitSegment()) { return; } + if (streamState.isLive()) { + return; + } + final YoutubeSabrFormat format = formatForItag(segment.getHeader().getItag()); + if (format != null && !streamState.isInitialized(format)) { + return; + } if (cacheClosed) { segment.failProgressive(new IOException("SABR session cache is closed")); segment.delete(); @@ -1234,7 +1328,7 @@ public List getResponses() { /** True once the requested media segment is known to be past the last segment of the stream. */ public boolean isBeyondEnd(@Nonnull final SabrSegmentRequest request) { - if (request.isInitializationSegment()) { + if (request.isInitializationSegment() || streamState.isActiveLive()) { return false; } final long endSegment = streamState.getEndSegment(request.getFormat()); @@ -1245,16 +1339,31 @@ public boolean isComplete() { return streamState.isComplete(); } - /** True once the server has reported this is a live stream (foundation for live support). */ public boolean isLive() { return streamState.isLive(); } - /** Latest segment the live edge has reached, or -1 if unknown / not live. */ + public boolean isActiveLive() { + return streamState.isActiveLive(); + } + + /** Absolute broadcast head reported by live metadata, or -1 if unknown / not live. */ public long getLiveHeadSequenceNumber() { return streamState.getLiveHeadSequenceNumber(); } + public long getLiveHeadTimeMs() { + return streamState.getLiveHeadTimeMs(); + } + + public long getLiveSeekableStartTimeMs() { + return streamState.getLiveSeekableStartTimeMs(); + } + + public long getLiveSeekableEndTimeMs() { + return streamState.getLiveSeekableEndTimeMs(); + } + /** * True when playback has caught up to the live head: a live-aware pump should wait for the head * to advance rather than treating an empty response as the end. Always false for VOD. @@ -1443,7 +1552,7 @@ public void prepareForMissingSegment(@Nonnull final SabrSegmentRequest request) private void failIfKnownOutOfBounds(@Nonnull final SabrSegmentRequest request) throws SabrProtocolException { - if (request.isInitializationSegment()) { + if (request.isInitializationSegment() || streamState.isActiveLive()) { return; } final long endSegment = streamState.getEndSegment(request.getFormat()); @@ -1459,7 +1568,7 @@ private boolean maybePrepareForDistantMediaSegment( return false; } final YoutubeSabrFormat format = request.getFormat(); - if (streamState.getEndSegment(format) <= 0) { + if (streamState.isActiveLive() || streamState.getEndSegment(format) <= 0) { return false; } if (request.getSequenceNumber() <= streamState.getMaxSegment(format) + 1) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java index 59f1f909..322bedc2 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/sabr/YoutubeSabrStreamState.java @@ -28,7 +28,7 @@ public final class YoutubeSabrStreamState { private byte[] poToken; @Nullable private volatile SabrNextRequestPolicy nextRequestPolicy; - private long playerTimeMsOverride = -1; + private volatile long playerTimeMsOverride = -1; private boolean audioFullyBuffered; private boolean videoFullyBuffered; private boolean audioLastOnlyRange; @@ -37,6 +37,9 @@ public final class YoutubeSabrStreamState { private volatile int enabledTrackTypesBitfield = YoutubeSabrRequestBuilder.ENABLED_TRACK_TYPES_VIDEO_AND_AUDIO; private volatile boolean selectAudioFormat = true; private volatile boolean selectVideoFormat = true; + private volatile boolean preferAudioFormat = true; + private volatile boolean preferVideoFormat = true; + private boolean writeFirstRequestPlaybackState; private boolean writeTopLevelPlayerTimeMs = true; private int clientViewportWidth = -1; private int clientViewportHeight = -1; @@ -70,14 +73,13 @@ public final class YoutubeSabrStreamState { @Nullable private List bufferedRangesOverride; - // how close to the head counts as "at the live edge" (segments of slack before we wait) private static final long LIVE_EDGE_MARGIN_SEGMENTS = 2; - // live: foundation only. we record what the server tells us about the live edge (via - // LIVE_METADATA) so a future live-aware pump can follow the head. VOD never sets these. - private boolean live; - private boolean postLiveDvr; - private long liveHeadSequenceNumber = -1; - private long liveHeadTimeMs = -1; + private volatile boolean live; + private volatile boolean postLiveDvr; + private volatile long liveHeadSequenceNumber = -1; + private volatile long liveHeadTimeMs = -1; + private volatile long liveSeekableStartTimeMs = -1; + private volatile long liveSeekableEndTimeMs = -1; public YoutubeSabrStreamState(@Nonnull final YoutubeSabrFormat audioFormat, @Nonnull final YoutubeSabrFormat videoFormat) { @@ -99,14 +101,7 @@ public boolean ingest(@Nonnull final SabrResponseStatePatch patch) { playbackCookie = nextRequestPolicy.getRawPlaybackCookie().clone(); } for (final SabrLiveMetadata meta : patch.getLiveMetadata()) { - live = true; - postLiveDvr = meta.isPostLiveDvr(); - if (meta.getHeadSequenceNumber() >= 0) { - liveHeadSequenceNumber = meta.getHeadSequenceNumber(); - } - if (meta.getHeadTimeMs() >= 0) { - liveHeadTimeMs = meta.getHeadTimeMs(); - } + progressed |= ingestLiveMetadata(meta); } for (final SabrFormatInitializationMetadata metadata : patch.getFormatMetadata()) { @@ -130,6 +125,47 @@ public boolean ingest(@Nonnull final SabrResponseStatePatch patch) { return progressed; } + boolean ingestLiveMetadata(@Nonnull final SabrLiveMetadata metadata) { + final boolean wasLive = live; + final boolean wasPostLiveDvr = postLiveDvr; + final long previousHeadSequenceNumber = liveHeadSequenceNumber; + final long previousHeadTimeMs = liveHeadTimeMs; + final long previousSeekableStartTimeMs = liveSeekableStartTimeMs; + final long previousSeekableEndTimeMs = liveSeekableEndTimeMs; + live = true; + postLiveDvr |= metadata.isPostLiveDvr(); + liveHeadSequenceNumber = advance(liveHeadSequenceNumber, + metadata.getHeadSequenceNumber()); + liveHeadTimeMs = advance(liveHeadTimeMs, metadata.getHeadTimeMs()); + liveSeekableStartTimeMs = advance(liveSeekableStartTimeMs, + scaleToMilliseconds(metadata.getMinSeekableTimeTicks(), + metadata.getMinSeekableTimescale())); + liveSeekableEndTimeMs = advance(liveSeekableEndTimeMs, + scaleToMilliseconds(metadata.getMaxSeekableTimeTicks(), + metadata.getMaxSeekableTimescale())); + return wasLive != live + || wasPostLiveDvr != postLiveDvr + || previousHeadSequenceNumber != liveHeadSequenceNumber + || previousHeadTimeMs != liveHeadTimeMs + || previousSeekableStartTimeMs != liveSeekableStartTimeMs + || previousSeekableEndTimeMs != liveSeekableEndTimeMs; + } + + private static long advance(final long current, final long candidate) { + return candidate < 0 ? current : Math.max(current, candidate); + } + + private static long scaleToMilliseconds(final long ticks, final int timescale) { + if (ticks < 0 || timescale <= 0) { + return -1; + } + final long seconds = ticks / timescale; + if (seconds > Long.MAX_VALUE / 1000L) { + return Long.MAX_VALUE; + } + return seconds * 1000L + ticks % timescale * 1000L / timescale; + } + public boolean ingest(@Nonnull final SabrMediaSegment segment) { final FormatProgress progress = findProgressForItag(segment.getHeader().getItag()); return progress != null && progress.observeSegment(segment); @@ -184,6 +220,9 @@ public long getPlayerTimeMs() { if (playerTimeMsOverride >= 0) { return playerTimeMsOverride; } + if (isActiveLive()) { + return 0; + } return Math.max(audio.getBufferedEndMs(), video.getBufferedEndMs()); } @@ -281,21 +320,25 @@ Collection getUnsentSabrContextTypes() { } public boolean isComplete() { - return (!isAudioEnabled() || audio.isComplete()) + return !isActiveLive() + && (!isAudioEnabled() || audio.isComplete()) && (!isVideoEnabled() || video.isComplete()); } - /** True once the server has sent live metadata for this stream (foundation for live support). */ public boolean isLive() { return live; } + public boolean isActiveLive() { + return live && !postLiveDvr; + } + /** True for an ended live stream still seekable as DVR. */ public boolean isPostLiveDvr() { return postLiveDvr; } - /** Latest segment the live edge has reached, or -1 if unknown / not live. */ + /** Absolute broadcast head reported by live metadata, or -1 if unknown / not live. */ public long getLiveHeadSequenceNumber() { return liveHeadSequenceNumber; } @@ -305,6 +348,14 @@ public long getLiveHeadTimeMs() { return liveHeadTimeMs; } + public long getLiveSeekableStartTimeMs() { + return liveSeekableStartTimeMs; + } + + public long getLiveSeekableEndTimeMs() { + return liveSeekableEndTimeMs; + } + /** * True when we have fetched up to (within a small margin of) the live head: the slower track has * reached the edge, so a live-aware pump should wait for the head to advance instead of treating @@ -312,7 +363,7 @@ public long getLiveHeadTimeMs() { */ public boolean isAtLiveEdge(@Nonnull final YoutubeSabrFormat audioFormat, @Nonnull final YoutubeSabrFormat videoFormat) { - if (!live || liveHeadSequenceNumber < 0) { + if (!isActiveLive() || liveHeadSequenceNumber < 0) { return false; } final long slowerTrack = Math.min(getMaxSegment(audioFormat), getMaxSegment(videoFormat)); @@ -333,7 +384,7 @@ public boolean hasSegmentIndex(@Nonnull final YoutubeSabrFormat format) { } public boolean isComplete(@Nonnull final YoutubeSabrFormat format) { - return progressForItag(format.getItag()).isComplete(); + return !isActiveLive() && progressForItag(format.getItag()).isComplete(); } public void assumeBufferedUntil(@Nonnull final YoutubeSabrFormat format, @@ -409,6 +460,14 @@ public synchronized void setRequestTrackMode(final int enabledTrackTypesBitfield this.enabledTrackTypesBitfield = enabledTrackTypesBitfield; this.selectAudioFormat = selectAudioFormat; this.selectVideoFormat = selectVideoFormat; + this.preferAudioFormat = selectAudioFormat; + this.preferVideoFormat = selectVideoFormat; + } + + synchronized void setPreferredTrackTypes(final boolean videoActive, + final boolean audioActive) { + preferAudioFormat = audioActive; + preferVideoFormat = videoActive; } public void setActiveTrackTypes(final boolean videoActive, final boolean audioActive) { @@ -495,6 +554,22 @@ boolean shouldSelectVideoFormat() { return selectVideoFormat; } + boolean shouldPreferAudioFormat() { + return preferAudioFormat; + } + + boolean shouldPreferVideoFormat() { + return preferVideoFormat; + } + + void setWriteFirstRequestPlaybackState(final boolean writeFirstRequestPlaybackState) { + this.writeFirstRequestPlaybackState = writeFirstRequestPlaybackState; + } + + boolean shouldWriteFirstRequestPlaybackState() { + return writeFirstRequestPlaybackState; + } + public void setWriteTopLevelPlayerTimeMs(final boolean writeTopLevelPlayerTimeMs) { this.writeTopLevelPlayerTimeMs = writeTopLevelPlayerTimeMs; } @@ -753,10 +828,15 @@ private boolean observeMetadata(@Nonnull final SabrFormatInitializationMetadata } private boolean observeSegment(@Nonnull final SabrMediaSegment segment) { - if (!segment.getHeader().isInitSegment() || metadata == null || segmentIndex != null) { - return false; + if (!segment.getHeader().isInitSegment()) { + return observeHeader(segment.getHeader()); + } + final boolean changed = !initReceived; + initReceived = true; + if (segmentIndex != null) { + return changed; } - return observeInitializationData(segment.getData()); + return observeInitializationData(segment.getData()) || changed; } private boolean observeInitializationData(@Nonnull final byte[] data) {