diff --git a/README.md b/README.md index 54c349c..1270942 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # useless-umd-player -umd video player to enjoy limited dvd menus in 720x480 pixels - -A standalone player for PSP UMD Video ISO images: open a disc, render its real +a standalone player for PSP UMD Video ISO images: open a disc, render its real menu, and play the movie (H.264 video + ATRAC3+ audio) with mouse and keyboard/media-key controls. diff --git a/cli/src/main/java/dev/quoije/umd/cli/Main.java b/cli/src/main/java/dev/quoije/umd/cli/Main.java index aa6572a..8c756ce 100644 --- a/cli/src/main/java/dev/quoije/umd/cli/Main.java +++ b/cli/src/main/java/dev/quoije/umd/cli/Main.java @@ -7,6 +7,7 @@ import dev.quoije.umd.media.Atrac3pDecoder; import dev.quoije.umd.media.VideoDecoder; import dev.quoije.umd.mps.AudioFramer; import dev.quoije.umd.mps.MpsDemuxer; +import dev.quoije.umd.player.PlaybackEngine; import dev.quoije.umd.playlist.Playlist; import dev.quoije.umd.sfo.Sfo; @@ -19,7 +20,7 @@ import java.util.Map; /** Command-line front-end for the UMD player library. */ public final class Main { - private static final String USAGE = "usage: umdplayer dump \n umdplayer mps \n umdplayer decode \n umdplayer audio [count]\n umdplayer adecode [maxPackets]"; + private static final String USAGE = "usage: umdplayer dump \n umdplayer mps \n umdplayer decode \n umdplayer audio [count]\n umdplayer adecode [maxPackets]\n umdplayer play [seconds]"; private Main() { } @@ -38,6 +39,7 @@ public final class Main { case "decode" -> decode(requireArg(args, 1), requireArg(args, 2)); case "audio" -> audio(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Integer.parseInt(args[3]) : 8); case "adecode" -> adecode(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Integer.parseInt(args[3]) : 500); + case "play" -> play(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Double.parseDouble(args[3]) : 5.0); case "help", "--help", "-h" -> { System.out.println(USAGE); yield 0; @@ -333,6 +335,55 @@ public final class Main { return 0; } + private static int play(String imagePath, String streamName, double seconds) throws IOException { + long targetPts = (long) (seconds * PlaybackEngine.MPEG_TIMESTAMP); + try (UmdIsoReader iso = new UmdIsoReader(Path.of(imagePath)); + PlaybackEngine engine = PlaybackEngine.open(iso, streamName, 0)) { + + int frames = 0; + long firstPts = -1; + long lastPts = -1; + long audioClock = -1; + long maxDrift = 0; + + PlaybackEngine.VideoFrame frame; + while ((frame = engine.nextVideoFrame()) != null) { + if (firstPts < 0) { + firstPts = frame.pts(); + } + lastPts = frame.pts(); + frames++; + + // Advance the audio clock to at least this frame's timestamp. + while (audioClock < frame.pts()) { + PlaybackEngine.AudioChunk chunk = engine.nextAudioChunk(); + if (chunk == null) { + break; + } + audioClock = chunk.pts() + PlaybackEngine.AUDIO_FRAME_TICKS; + } + + long drift = frame.pts() - audioClock; + if (Math.abs(drift) > Math.abs(maxDrift)) { + maxDrift = drift; + } + + if (frame.pts() >= targetPts) { + break; + } + } + + double spanSeconds = (lastPts - firstPts) / (double) PlaybackEngine.MPEG_TIMESTAMP; + double frameDuration = frames > 1 ? (lastPts - firstPts) / (double) (frames - 1) : 0; + System.out.printf("video: %d frames, pts %d..%d (%.2fs), avg frame duration %.2f ticks%n", + frames, firstPts, lastPts, spanSeconds, frameDuration); + System.out.printf("audio clock: %d (%.2fs), max |video-audio| drift: %d ticks (%.1f ms)%n", + audioClock, audioClock / (double) PlaybackEngine.MPEG_TIMESTAMP, + maxDrift, maxDrift * 1000.0 / PlaybackEngine.MPEG_TIMESTAMP); + } + return 0; + } + private static String hexPrefix(byte[] data, int max) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < Math.min(data.length, max); i++) { diff --git a/core/src/main/java/dev/quoije/umd/media/AnnexBSplitter.java b/core/src/main/java/dev/quoije/umd/media/AnnexBSplitter.java new file mode 100644 index 0000000..168b2cc --- /dev/null +++ b/core/src/main/java/dev/quoije/umd/media/AnnexBSplitter.java @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package dev.quoije.umd.media; + +import java.util.Arrays; + +/** + * Incrementally splits an Annex-B H.264 byte stream into access units. + * + *

An access unit is emitted only once it is known to be complete: a slice + * NAL (type 1 or 5) terminates an access unit, and its end is confirmed by the + * presence of the following start code. Incomplete trailing data is buffered + * until more bytes arrive or until {@link #drain()} is called.

+ */ +public final class AnnexBSplitter { + + private byte[] buffer = new byte[0]; + + /** Appends raw Annex-B bytes. */ + public void feed(byte[] data) { + if (data == null || data.length == 0) { + return; + } + byte[] merged = Arrays.copyOf(buffer, buffer.length + data.length); + System.arraycopy(data, 0, merged, buffer.length, data.length); + buffer = merged; + } + + /** + * Returns the next complete access unit, or {@code null} if none is + * available yet. + */ + public byte[] nextAccessUnit() { + int i = 0; + while (i < buffer.length) { + int startCodeLength = startCodeLengthAt(buffer, i); + if (startCodeLength > 0) { + int nalu = i + startCodeLength; + if (nalu < buffer.length) { + int type = buffer[nalu] & 0x1F; + if (type == 1 || type == 5) { + int end = findNextStartCode(buffer, nalu); + if (end >= buffer.length) { + // Slice may still be incomplete; wait for more data. + return null; + } + byte[] accessUnit = Arrays.copyOfRange(buffer, 0, end); + buffer = Arrays.copyOfRange(buffer, end, buffer.length); + return accessUnit; + } + } + i = nalu; + } else { + i++; + } + } + return null; + } + + /** Returns any remaining buffered bytes, or {@code null} if empty. */ + public byte[] drain() { + if (buffer.length == 0) { + return null; + } + byte[] tail = buffer; + buffer = new byte[0]; + return tail; + } + + private static int startCodeLengthAt(byte[] b, int i) { + if (i + 2 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 1) { + return 3; + } + if (i + 3 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 0 && b[i + 3] == 1) { + return 4; + } + return -1; + } + + private static int findNextStartCode(byte[] b, int from) { + int j = from; + while (j < b.length) { + if (startCodeLengthAt(b, j) > 0) { + return j; + } + j++; + } + return b.length; + } +} diff --git a/core/src/main/java/dev/quoije/umd/media/VideoDecoder.java b/core/src/main/java/dev/quoije/umd/media/VideoDecoder.java index 86b310c..5f013d3 100644 --- a/core/src/main/java/dev/quoije/umd/media/VideoDecoder.java +++ b/core/src/main/java/dev/quoije/umd/media/VideoDecoder.java @@ -5,13 +5,13 @@ import org.bytedeco.ffmpeg.avcodec.AVCodec; import org.bytedeco.ffmpeg.avcodec.AVCodecContext; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.avutil.AVFrame; +import org.bytedeco.ffmpeg.avutil.AVRational; import org.bytedeco.ffmpeg.global.avcodec; import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacpp.BytePointer; import org.bytedeco.javacpp.PointerPointer; import java.util.ArrayDeque; -import java.util.Arrays; /** H.264 (Annex B) video decoder backed by FFmpeg. */ public final class VideoDecoder implements AutoCloseable { @@ -47,59 +47,24 @@ public final class VideoDecoder implements AutoCloseable { * produces one frame per packet. */ public void feedAnnexB(byte[] annexB) { - int accessUnitStart = 0; - int i = 0; - while (i < annexB.length) { - int startCodeLength = startCodeLengthAt(annexB, i); - if (startCodeLength > 0) { - int nalu = i + startCodeLength; - if (nalu < annexB.length) { - int type = annexB[nalu] & 0x1F; - if (type == 1 || type == 5) { - int end = findNextStartCode(annexB, nalu); - feedOne(Arrays.copyOfRange(annexB, accessUnitStart, end)); - accessUnitStart = end; - i = end; - continue; - } - } - i = nalu; - } else { - i++; - } + AnnexBSplitter splitter = new AnnexBSplitter(); + splitter.feed(annexB); + byte[] accessUnit; + while ((accessUnit = splitter.nextAccessUnit()) != null) { + feedAccessUnit(accessUnit); } - - if (accessUnitStart < annexB.length) { - feedOne(Arrays.copyOfRange(annexB, accessUnitStart, annexB.length)); + byte[] tail = splitter.drain(); + if (tail != null && tail.length > 0) { + feedAccessUnit(tail); } } - private static int startCodeLengthAt(byte[] b, int i) { - if (i + 2 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 1) { - return 3; - } - if (i + 3 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 0 && b[i + 3] == 1) { - return 4; - } - return -1; - } - - private static int findNextStartCode(byte[] b, int from) { - int j = from; - while (j < b.length) { - if (startCodeLengthAt(b, j) > 0) { - return j; - } - j++; - } - return b.length; - } - - private void feedOne(byte[] data) { - if (avcodec.av_new_packet(packet, data.length) < 0) { + /** Feeds one complete Annex-B access unit to the decoder. */ + public void feedAccessUnit(byte[] accessUnit) { + if (avcodec.av_new_packet(packet, accessUnit.length) < 0) { throw new IllegalStateException("av_new_packet failed"); } - packet.data().put(data); + packet.data().put(accessUnit); int ret = avcodec.avcodec_send_packet(context, packet); avcodec.av_packet_unref(packet); @@ -147,6 +112,18 @@ public final class VideoDecoder implements AutoCloseable { return frames.poll(); } + /** + * Returns the decoded frame rate reported by FFmpeg (from the SPS VUI), + * or 0.0 if not yet known. + */ + public double frameRate() { + AVRational fr = context.framerate(); + if (fr != null && !fr.isNull() && fr.num() > 0 && fr.den() > 0) { + return (double) fr.num() / fr.den(); + } + return 0.0; + } + @Override public void close() { avutil.av_frame_free(frame); diff --git a/core/src/main/java/dev/quoije/umd/player/PlaybackEngine.java b/core/src/main/java/dev/quoije/umd/player/PlaybackEngine.java new file mode 100644 index 0000000..35631ff --- /dev/null +++ b/core/src/main/java/dev/quoije/umd/player/PlaybackEngine.java @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package dev.quoije.umd.player; + +import dev.quoije.umd.iso.UmdIsoReader; +import dev.quoije.umd.media.AnnexBSplitter; +import dev.quoije.umd.media.Atrac3pDecoder; +import dev.quoije.umd.media.VideoDecoder; +import dev.quoije.umd.mps.AudioFramer; +import dev.quoije.umd.mps.MpsDemuxer; + +import java.io.IOException; +import java.util.ArrayDeque; + +/** + * Decodes an MPS stream into timestamped video frames and PCM audio. + * + *

The engine reads MPEG-PS packets, routes video to the H.264 decoder and + * the selected audio substream to the ATRAC3+ decoder, and exposes the results + * through bounded pull queues. Timestamps are in the 90 kHz MPEG clock.

+ * + *

Audio timestamps come directly from the PES headers (one ATRAC3+ frame + * is 2048 samples at 48 kHz = 3840 ticks). Video timestamps are anchored to + * the occasional PES PTS and interpolated using the frame rate reported by the + * decoder; most video packets carry no PTS.

+ */ +public final class PlaybackEngine implements AutoCloseable { + + public static final int MPEG_TIMESTAMP = 90000; + public static final int AUDIO_SAMPLE_RATE = 48000; + public static final int AUDIO_FRAME_SAMPLES = 2048; + public static final long AUDIO_FRAME_TICKS = (long) AUDIO_FRAME_SAMPLES * MPEG_TIMESTAMP / AUDIO_SAMPLE_RATE; + + /** A decoded video frame tagged with its presentation timestamp. */ + public record VideoFrame(long pts, int width, int height, int pixelFormat, byte[] yPlane, int yStride) { + } + + /** A chunk of interleaved S16 PCM tagged with its presentation timestamp. */ + public record AudioChunk(long pts, byte[] pcm) { + } + + private final MpsDemuxer demuxer; + private final int audioSubstream; + private final VideoDecoder videoDecoder = new VideoDecoder(); + private final AnnexBSplitter videoSplitter = new AnnexBSplitter(); + private final AudioFramer audioFramer = new AudioFramer(); + + private Atrac3pDecoder audioDecoder; + private final ArrayDeque videoQueue = new ArrayDeque<>(); + private final ArrayDeque audioQueue = new ArrayDeque<>(); + + private long pendingVideoPts = -1; + private long nextVideoPts = 0; + private double videoFrameDurationTicks = -1; + + private long audioFirstPts = 0; + private boolean audioPtsSet = false; + private long audioFramesEmitted = 0; + + private boolean eof = false; + + private PlaybackEngine(UmdIsoReader iso, String streamName, int audioSubstream) throws IOException { + this.demuxer = MpsDemuxer.open(iso, streamName); + this.audioSubstream = audioSubstream; + } + + /** Opens the given playlist stream for playback, using the given audio substream (0 or 1). */ + public static PlaybackEngine open(UmdIsoReader iso, String streamName, int audioSubstream) throws IOException { + return new PlaybackEngine(iso, streamName, audioSubstream); + } + + /** + * Decodes until the next video frame is available, or end of stream. + * Returns {@code null} at end of stream. + */ + public VideoFrame nextVideoFrame() throws IOException { + int budget = 1000; + while (videoQueue.isEmpty() && budget-- > 0) { + if (!routeOne()) { + break; + } + } + return videoQueue.poll(); + } + + /** + * Decodes until the next audio chunk is available, or end of stream. + * Returns {@code null} at end of stream. + */ + public AudioChunk nextAudioChunk() throws IOException { + int budget = 1000; + while (audioQueue.isEmpty() && budget-- > 0) { + if (!routeOne()) { + break; + } + } + return audioQueue.poll(); + } + + /** True once the stream has been fully decoded. */ + public boolean isEof() { + return eof && videoQueue.isEmpty() && audioQueue.isEmpty(); + } + + /** Reads and routes one packet. Returns false at end of stream. */ + private boolean routeOne() throws IOException { + if (eof) { + return false; + } + MpsDemuxer.Packet packet = demuxer.nextPacket(); + if (packet == null) { + finish(); + eof = true; + return false; + } + if (packet.isVideo()) { + handleVideo(packet); + } else if (packet.isAudio() && packet.channel() == audioSubstream) { + handleAudio(packet); + } + return true; + } + + private void handleVideo(MpsDemuxer.Packet packet) { + if (packet.pts() >= 0) { + if (pendingVideoPts < 0) { + nextVideoPts = packet.pts(); + } + pendingVideoPts = packet.pts(); + } + + videoSplitter.feed(packet.payload()); + byte[] accessUnit; + while ((accessUnit = videoSplitter.nextAccessUnit()) != null) { + videoDecoder.feedAccessUnit(accessUnit); + VideoDecoder.Frame frame; + while ((frame = videoDecoder.receive()) != null) { + videoQueue.add(new VideoFrame( + assignVideoPts(), frame.width(), frame.height(), + frame.pixelFormat(), frame.yPlane(), frame.yStride())); + } + } + } + + private long assignVideoPts() { + long pts; + if (pendingVideoPts >= 0) { + pts = pendingVideoPts; + pendingVideoPts = -1; + } else { + pts = nextVideoPts; + } + nextVideoPts = pts + Math.round(videoFrameDurationTicks()); + return pts; + } + + private double videoFrameDurationTicks() { + double frameRate = videoDecoder.frameRate(); + if (frameRate > 0) { + videoFrameDurationTicks = MPEG_TIMESTAMP / frameRate; + } else if (videoFrameDurationTicks < 0) { + videoFrameDurationTicks = 3003.0; // provisional NTSC default + } + return videoFrameDurationTicks; + } + + private void handleAudio(MpsDemuxer.Packet packet) { + if (packet.pts() >= 0) { + if (!audioPtsSet) { + audioFirstPts = packet.pts(); + audioPtsSet = true; + } + long expected = audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS; + if (Math.abs(packet.pts() - expected) >= AUDIO_FRAME_TICKS) { + audioFirstPts = packet.pts() - audioFramesEmitted * AUDIO_FRAME_TICKS; + } + } + + audioFramer.feed(packet.payload()); + byte[] frame; + while ((frame = audioFramer.nextFrame()) != null) { + if (audioDecoder == null) { + audioDecoder = new Atrac3pDecoder(AUDIO_SAMPLE_RATE, 2, Math.max(688, frame.length)); + } + long pts = audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS; + audioFramesEmitted++; + + audioDecoder.feedFrame(frame); + byte[] pcm; + while ((pcm = audioDecoder.receive()) != null) { + audioQueue.add(new AudioChunk(pts, pcm)); + } + } + } + + private void finish() { + byte[] tail = videoSplitter.drain(); + if (tail != null && tail.length > 0) { + videoDecoder.feedAccessUnit(tail); + } + videoDecoder.flush(); + VideoDecoder.Frame frame; + while ((frame = videoDecoder.receive()) != null) { + videoQueue.add(new VideoFrame( + assignVideoPts(), frame.width(), frame.height(), + frame.pixelFormat(), frame.yPlane(), frame.yStride())); + } + + if (audioDecoder != null) { + audioDecoder.flush(); + byte[] pcm; + while ((pcm = audioDecoder.receive()) != null) { + audioQueue.add(new AudioChunk( + audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS, pcm)); + } + } + } + + @Override + public void close() { + videoDecoder.close(); + if (audioDecoder != null) { + audioDecoder.close(); + } + demuxer.close(); + } +} diff --git a/core/src/test/java/dev/quoije/umd/media/AnnexBSplitterTest.java b/core/src/test/java/dev/quoije/umd/media/AnnexBSplitterTest.java new file mode 100644 index 0000000..3208062 --- /dev/null +++ b/core/src/test/java/dev/quoije/umd/media/AnnexBSplitterTest.java @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package dev.quoije.umd.media; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +class AnnexBSplitterTest { + + @Test + void splitsCompleteAccessUnitsAndHoldsPartialTail() { + // SPS + slice1 + slice2 (slice2 has no following start code). + byte[] stream = { + 0, 0, 0, 1, 0x67, 1, 2, 3, // SPS + 0, 0, 0, 1, 0x41, 4, 5, 6, // slice (type 1) + 0, 0, 0, 1, 0x41, 7, 8, 9, // slice (type 1), incomplete tail + }; + + AnnexBSplitter splitter = new AnnexBSplitter(); + splitter.feed(stream); + + assertThat(splitter.nextAccessUnit()).isEqualTo(Arrays.copyOfRange(stream, 0, 16)); + assertThat(splitter.nextAccessUnit()).isNull(); + assertThat(splitter.drain()).isEqualTo(Arrays.copyOfRange(stream, 16, 24)); + } + + @Test + void holdsIncompleteSliceAcrossFeeds() { + byte[] stream = { + 0, 0, 0, 1, 0x67, 1, 2, 3, + 0, 0, 0, 1, 0x41, 4, 5, 6, + 0, 0, 0, 1, 0x41, 7, 8, 9, + }; + + AnnexBSplitter splitter = new AnnexBSplitter(); + splitter.feed(Arrays.copyOfRange(stream, 0, 15)); + // The slice NAL's end (the next start code) has not arrived yet. + assertThat(splitter.nextAccessUnit()).isNull(); + + splitter.feed(Arrays.copyOfRange(stream, 15, stream.length)); + assertThat(splitter.nextAccessUnit()).isEqualTo(Arrays.copyOfRange(stream, 0, 16)); + assertThat(splitter.nextAccessUnit()).isNull(); + assertThat(splitter.drain()).isEqualTo(Arrays.copyOfRange(stream, 16, 24)); + } +}