Add playback engine with A/V timestamping

- AnnexBSplitter incrementally splits the Annex-B stream into access
  units, holding partial slices across packet feeds.
- VideoDecoder exposes feedAccessUnit and the decoded frame rate.
- PlaybackEngine demuxes and decodes in lockstep, tagging video frames
  and audio chunks with 90 kHz PTS (audio from PES headers, video from
  PES anchors interpolated by decoder frame rate).
- CLI 'play' command runs the engine headlessly and reports A/V drift.
- Validated against Batman Begins: 95 frames in 4.0s, max drift -78.6 ms
  (video trails audio at stream start), avg frame duration 3833.6 ticks.
- Tests for AnnexBSplitter.

README.md

@@ -1,8 +1,6 @@
 # useless-umd-player
 
-umd video player to enjoy limited dvd menus in 720x480 pixels
+a standalone player for PSP UMD Video ISO images: open a disc, render its real
-
-A standalone player for PSP UMD Video ISO images: open a disc, render its real
 menu, and play the movie (H.264 video + ATRAC3+ audio) with mouse and
 keyboard/media-key controls.
 

cli/src/main/java/dev/quoije/umd/cli/Main.java

@@ -7,6 +7,7 @@ import dev.quoije.umd.media.Atrac3pDecoder;
 import dev.quoije.umd.media.VideoDecoder;
 import dev.quoije.umd.mps.AudioFramer;
 import dev.quoije.umd.mps.MpsDemuxer;
+import dev.quoije.umd.player.PlaybackEngine;
 import dev.quoije.umd.playlist.Playlist;
 import dev.quoije.umd.sfo.Sfo;
 
@@ -19,7 +20,7 @@ import java.util.Map;
 /** Command-line front-end for the UMD player library. */
 public final class Main {
 
-    private static final String USAGE = "usage: umdplayer dump <image.iso>\n       umdplayer mps <image.iso> <streamName>\n       umdplayer decode <image.iso> <streamName>\n       umdplayer audio <image.iso> <streamName> [count]\n       umdplayer adecode <image.iso> <streamName> [maxPackets]";
+    private static final String USAGE = "usage: umdplayer dump <image.iso>\n       umdplayer mps <image.iso> <streamName>\n       umdplayer decode <image.iso> <streamName>\n       umdplayer audio <image.iso> <streamName> [count]\n       umdplayer adecode <image.iso> <streamName> [maxPackets]\n       umdplayer play <image.iso> <streamName> [seconds]";
 
     private Main() {
     }
@@ -38,6 +39,7 @@ public final class Main {
                 case "decode" -> decode(requireArg(args, 1), requireArg(args, 2));
                 case "audio" -> audio(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Integer.parseInt(args[3]) : 8);
                 case "adecode" -> adecode(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Integer.parseInt(args[3]) : 500);
+                case "play" -> play(requireArg(args, 1), requireArg(args, 2), args.length > 3 ? Double.parseDouble(args[3]) : 5.0);
                 case "help", "--help", "-h" -> {
                     System.out.println(USAGE);
                     yield 0;
@@ -333,6 +335,55 @@ public final class Main {
         return 0;
     }
 
+    private static int play(String imagePath, String streamName, double seconds) throws IOException {
+        long targetPts = (long) (seconds * PlaybackEngine.MPEG_TIMESTAMP);
+        try (UmdIsoReader iso = new UmdIsoReader(Path.of(imagePath));
+             PlaybackEngine engine = PlaybackEngine.open(iso, streamName, 0)) {
+
+            int frames = 0;
+            long firstPts = -1;
+            long lastPts = -1;
+            long audioClock = -1;
+            long maxDrift = 0;
+
+            PlaybackEngine.VideoFrame frame;
+            while ((frame = engine.nextVideoFrame()) != null) {
+                if (firstPts < 0) {
+                    firstPts = frame.pts();
+                }
+                lastPts = frame.pts();
+                frames++;
+
+                // Advance the audio clock to at least this frame's timestamp.
+                while (audioClock < frame.pts()) {
+                    PlaybackEngine.AudioChunk chunk = engine.nextAudioChunk();
+                    if (chunk == null) {
+                        break;
+                    }
+                    audioClock = chunk.pts() + PlaybackEngine.AUDIO_FRAME_TICKS;
+                }
+
+                long drift = frame.pts() - audioClock;
+                if (Math.abs(drift) > Math.abs(maxDrift)) {
+                    maxDrift = drift;
+                }
+
+                if (frame.pts() >= targetPts) {
+                    break;
+                }
+            }
+
+            double spanSeconds = (lastPts - firstPts) / (double) PlaybackEngine.MPEG_TIMESTAMP;
+            double frameDuration = frames > 1 ? (lastPts - firstPts) / (double) (frames - 1) : 0;
+            System.out.printf("video: %d frames, pts %d..%d (%.2fs), avg frame duration %.2f ticks%n",
+                    frames, firstPts, lastPts, spanSeconds, frameDuration);
+            System.out.printf("audio clock: %d (%.2fs), max |video-audio| drift: %d ticks (%.1f ms)%n",
+                    audioClock, audioClock / (double) PlaybackEngine.MPEG_TIMESTAMP,
+                    maxDrift, maxDrift * 1000.0 / PlaybackEngine.MPEG_TIMESTAMP);
+        }
+        return 0;
+    }
+
     private static String hexPrefix(byte[] data, int max) {
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < Math.min(data.length, max); i++) {

core/src/main/java/dev/quoije/umd/media/AnnexBSplitter.java

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+package dev.quoije.umd.media;
+
+import java.util.Arrays;
+
+/**
+ * Incrementally splits an Annex-B H.264 byte stream into access units.
+ *
+ * <p>An access unit is emitted only once it is known to be complete: a slice
+ * NAL (type 1 or 5) terminates an access unit, and its end is confirmed by the
+ * presence of the following start code. Incomplete trailing data is buffered
+ * until more bytes arrive or until {@link #drain()} is called.</p>
+ */
+public final class AnnexBSplitter {
+
+    private byte[] buffer = new byte[0];
+
+    /** Appends raw Annex-B bytes. */
+    public void feed(byte[] data) {
+        if (data == null || data.length == 0) {
+            return;
+        }
+        byte[] merged = Arrays.copyOf(buffer, buffer.length + data.length);
+        System.arraycopy(data, 0, merged, buffer.length, data.length);
+        buffer = merged;
+    }
+
+    /**
+     * Returns the next complete access unit, or {@code null} if none is
+     * available yet.
+     */
+    public byte[] nextAccessUnit() {
+        int i = 0;
+        while (i < buffer.length) {
+            int startCodeLength = startCodeLengthAt(buffer, i);
+            if (startCodeLength > 0) {
+                int nalu = i + startCodeLength;
+                if (nalu < buffer.length) {
+                    int type = buffer[nalu] & 0x1F;
+                    if (type == 1 || type == 5) {
+                        int end = findNextStartCode(buffer, nalu);
+                        if (end >= buffer.length) {
+                            // Slice may still be incomplete; wait for more data.
+                            return null;
+                        }
+                        byte[] accessUnit = Arrays.copyOfRange(buffer, 0, end);
+                        buffer = Arrays.copyOfRange(buffer, end, buffer.length);
+                        return accessUnit;
+                    }
+                }
+                i = nalu;
+            } else {
+                i++;
+            }
+        }
+        return null;
+    }
+
+    /** Returns any remaining buffered bytes, or {@code null} if empty. */
+    public byte[] drain() {
+        if (buffer.length == 0) {
+            return null;
+        }
+        byte[] tail = buffer;
+        buffer = new byte[0];
+        return tail;
+    }
+
+    private static int startCodeLengthAt(byte[] b, int i) {
+        if (i + 2 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 1) {
+            return 3;
+        }
+        if (i + 3 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 0 && b[i + 3] == 1) {
+            return 4;
+        }
+        return -1;
+    }
+
+    private static int findNextStartCode(byte[] b, int from) {
+        int j = from;
+        while (j < b.length) {
+            if (startCodeLengthAt(b, j) > 0) {
+                return j;
+            }
+            j++;
+        }
+        return b.length;
+    }
+}

core/src/main/java/dev/quoije/umd/media/VideoDecoder.java

@@ -5,13 +5,13 @@ import org.bytedeco.ffmpeg.avcodec.AVCodec;
 import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
 import org.bytedeco.ffmpeg.avcodec.AVPacket;
 import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.avutil.AVRational;
 import org.bytedeco.ffmpeg.global.avcodec;
 import org.bytedeco.ffmpeg.global.avutil;
 import org.bytedeco.javacpp.BytePointer;
 import org.bytedeco.javacpp.PointerPointer;
 
 import java.util.ArrayDeque;
-import java.util.Arrays;
 
 /** H.264 (Annex B) video decoder backed by FFmpeg. */
 public final class VideoDecoder implements AutoCloseable {
@@ -47,59 +47,24 @@ public final class VideoDecoder implements AutoCloseable {
      * produces one frame per packet.
      */
     public void feedAnnexB(byte[] annexB) {
-        int accessUnitStart = 0;
+        AnnexBSplitter splitter = new AnnexBSplitter();
-        int i = 0;
+        splitter.feed(annexB);
-        while (i < annexB.length) {
+        byte[] accessUnit;
-            int startCodeLength = startCodeLengthAt(annexB, i);
+        while ((accessUnit = splitter.nextAccessUnit()) != null) {
-            if (startCodeLength > 0) {
+            feedAccessUnit(accessUnit);
-                int nalu = i + startCodeLength;
-                if (nalu < annexB.length) {
-                    int type = annexB[nalu] & 0x1F;
-                    if (type == 1 || type == 5) {
-                        int end = findNextStartCode(annexB, nalu);
-                        feedOne(Arrays.copyOfRange(annexB, accessUnitStart, end));
-                        accessUnitStart = end;
-                        i = end;
-                        continue;
         }
-                }
+        byte[] tail = splitter.drain();
-                i = nalu;
+        if (tail != null && tail.length > 0) {
-            } else {
+            feedAccessUnit(tail);
-                i++;
         }
     }
 
-        if (accessUnitStart < annexB.length) {
+    /** Feeds one complete Annex-B access unit to the decoder. */
-            feedOne(Arrays.copyOfRange(annexB, accessUnitStart, annexB.length));
+    public void feedAccessUnit(byte[] accessUnit) {
-        }
+        if (avcodec.av_new_packet(packet, accessUnit.length) < 0) {
-    }
-
-    private static int startCodeLengthAt(byte[] b, int i) {
-        if (i + 2 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 1) {
-            return 3;
-        }
-        if (i + 3 < b.length && b[i] == 0 && b[i + 1] == 0 && b[i + 2] == 0 && b[i + 3] == 1) {
-            return 4;
-        }
-        return -1;
-    }
-
-    private static int findNextStartCode(byte[] b, int from) {
-        int j = from;
-        while (j < b.length) {
-            if (startCodeLengthAt(b, j) > 0) {
-                return j;
-            }
-            j++;
-        }
-        return b.length;
-    }
-
-    private void feedOne(byte[] data) {
-        if (avcodec.av_new_packet(packet, data.length) < 0) {
             throw new IllegalStateException("av_new_packet failed");
         }
-        packet.data().put(data);
+        packet.data().put(accessUnit);
 
         int ret = avcodec.avcodec_send_packet(context, packet);
         avcodec.av_packet_unref(packet);
@@ -147,6 +112,18 @@ public final class VideoDecoder implements AutoCloseable {
         return frames.poll();
     }
 
+    /**
+     * Returns the decoded frame rate reported by FFmpeg (from the SPS VUI),
+     * or 0.0 if not yet known.
+     */
+    public double frameRate() {
+        AVRational fr = context.framerate();
+        if (fr != null && !fr.isNull() && fr.num() > 0 && fr.den() > 0) {
+            return (double) fr.num() / fr.den();
+        }
+        return 0.0;
+    }
+
     @Override
     public void close() {
         avutil.av_frame_free(frame);

core/src/main/java/dev/quoije/umd/player/PlaybackEngine.java

@@ -0,0 +1,226 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+package dev.quoije.umd.player;
+
+import dev.quoije.umd.iso.UmdIsoReader;
+import dev.quoije.umd.media.AnnexBSplitter;
+import dev.quoije.umd.media.Atrac3pDecoder;
+import dev.quoije.umd.media.VideoDecoder;
+import dev.quoije.umd.mps.AudioFramer;
+import dev.quoije.umd.mps.MpsDemuxer;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+
+/**
+ * Decodes an MPS stream into timestamped video frames and PCM audio.
+ *
+ * <p>The engine reads MPEG-PS packets, routes video to the H.264 decoder and
+ * the selected audio substream to the ATRAC3+ decoder, and exposes the results
+ * through bounded pull queues. Timestamps are in the 90 kHz MPEG clock.</p>
+ *
+ * <p>Audio timestamps come directly from the PES headers (one ATRAC3+ frame
+ * is 2048 samples at 48 kHz = 3840 ticks). Video timestamps are anchored to
+ * the occasional PES PTS and interpolated using the frame rate reported by the
+ * decoder; most video packets carry no PTS.</p>
+ */
+public final class PlaybackEngine implements AutoCloseable {
+
+    public static final int MPEG_TIMESTAMP = 90000;
+    public static final int AUDIO_SAMPLE_RATE = 48000;
+    public static final int AUDIO_FRAME_SAMPLES = 2048;
+    public static final long AUDIO_FRAME_TICKS = (long) AUDIO_FRAME_SAMPLES * MPEG_TIMESTAMP / AUDIO_SAMPLE_RATE;
+
+    /** A decoded video frame tagged with its presentation timestamp. */
+    public record VideoFrame(long pts, int width, int height, int pixelFormat, byte[] yPlane, int yStride) {
+    }
+
+    /** A chunk of interleaved S16 PCM tagged with its presentation timestamp. */
+    public record AudioChunk(long pts, byte[] pcm) {
+    }
+
+    private final MpsDemuxer demuxer;
+    private final int audioSubstream;
+    private final VideoDecoder videoDecoder = new VideoDecoder();
+    private final AnnexBSplitter videoSplitter = new AnnexBSplitter();
+    private final AudioFramer audioFramer = new AudioFramer();
+
+    private Atrac3pDecoder audioDecoder;
+    private final ArrayDeque<VideoFrame> videoQueue = new ArrayDeque<>();
+    private final ArrayDeque<AudioChunk> audioQueue = new ArrayDeque<>();
+
+    private long pendingVideoPts = -1;
+    private long nextVideoPts = 0;
+    private double videoFrameDurationTicks = -1;
+
+    private long audioFirstPts = 0;
+    private boolean audioPtsSet = false;
+    private long audioFramesEmitted = 0;
+
+    private boolean eof = false;
+
+    private PlaybackEngine(UmdIsoReader iso, String streamName, int audioSubstream) throws IOException {
+        this.demuxer = MpsDemuxer.open(iso, streamName);
+        this.audioSubstream = audioSubstream;
+    }
+
+    /** Opens the given playlist stream for playback, using the given audio substream (0 or 1). */
+    public static PlaybackEngine open(UmdIsoReader iso, String streamName, int audioSubstream) throws IOException {
+        return new PlaybackEngine(iso, streamName, audioSubstream);
+    }
+
+    /**
+     * Decodes until the next video frame is available, or end of stream.
+     * Returns {@code null} at end of stream.
+     */
+    public VideoFrame nextVideoFrame() throws IOException {
+        int budget = 1000;
+        while (videoQueue.isEmpty() && budget-- > 0) {
+            if (!routeOne()) {
+                break;
+            }
+        }
+        return videoQueue.poll();
+    }
+
+    /**
+     * Decodes until the next audio chunk is available, or end of stream.
+     * Returns {@code null} at end of stream.
+     */
+    public AudioChunk nextAudioChunk() throws IOException {
+        int budget = 1000;
+        while (audioQueue.isEmpty() && budget-- > 0) {
+            if (!routeOne()) {
+                break;
+            }
+        }
+        return audioQueue.poll();
+    }
+
+    /** True once the stream has been fully decoded. */
+    public boolean isEof() {
+        return eof && videoQueue.isEmpty() && audioQueue.isEmpty();
+    }
+
+    /** Reads and routes one packet. Returns false at end of stream. */
+    private boolean routeOne() throws IOException {
+        if (eof) {
+            return false;
+        }
+        MpsDemuxer.Packet packet = demuxer.nextPacket();
+        if (packet == null) {
+            finish();
+            eof = true;
+            return false;
+        }
+        if (packet.isVideo()) {
+            handleVideo(packet);
+        } else if (packet.isAudio() && packet.channel() == audioSubstream) {
+            handleAudio(packet);
+        }
+        return true;
+    }
+
+    private void handleVideo(MpsDemuxer.Packet packet) {
+        if (packet.pts() >= 0) {
+            if (pendingVideoPts < 0) {
+                nextVideoPts = packet.pts();
+            }
+            pendingVideoPts = packet.pts();
+        }
+
+        videoSplitter.feed(packet.payload());
+        byte[] accessUnit;
+        while ((accessUnit = videoSplitter.nextAccessUnit()) != null) {
+            videoDecoder.feedAccessUnit(accessUnit);
+            VideoDecoder.Frame frame;
+            while ((frame = videoDecoder.receive()) != null) {
+                videoQueue.add(new VideoFrame(
+                        assignVideoPts(), frame.width(), frame.height(),
+                        frame.pixelFormat(), frame.yPlane(), frame.yStride()));
+            }
+        }
+    }
+
+    private long assignVideoPts() {
+        long pts;
+        if (pendingVideoPts >= 0) {
+            pts = pendingVideoPts;
+            pendingVideoPts = -1;
+        } else {
+            pts = nextVideoPts;
+        }
+        nextVideoPts = pts + Math.round(videoFrameDurationTicks());
+        return pts;
+    }
+
+    private double videoFrameDurationTicks() {
+        double frameRate = videoDecoder.frameRate();
+        if (frameRate > 0) {
+            videoFrameDurationTicks = MPEG_TIMESTAMP / frameRate;
+        } else if (videoFrameDurationTicks < 0) {
+            videoFrameDurationTicks = 3003.0; // provisional NTSC default
+        }
+        return videoFrameDurationTicks;
+    }
+
+    private void handleAudio(MpsDemuxer.Packet packet) {
+        if (packet.pts() >= 0) {
+            if (!audioPtsSet) {
+                audioFirstPts = packet.pts();
+                audioPtsSet = true;
+            }
+            long expected = audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS;
+            if (Math.abs(packet.pts() - expected) >= AUDIO_FRAME_TICKS) {
+                audioFirstPts = packet.pts() - audioFramesEmitted * AUDIO_FRAME_TICKS;
+            }
+        }
+
+        audioFramer.feed(packet.payload());
+        byte[] frame;
+        while ((frame = audioFramer.nextFrame()) != null) {
+            if (audioDecoder == null) {
+                audioDecoder = new Atrac3pDecoder(AUDIO_SAMPLE_RATE, 2, Math.max(688, frame.length));
+            }
+            long pts = audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS;
+            audioFramesEmitted++;
+
+            audioDecoder.feedFrame(frame);
+            byte[] pcm;
+            while ((pcm = audioDecoder.receive()) != null) {
+                audioQueue.add(new AudioChunk(pts, pcm));
+            }
+        }
+    }
+
+    private void finish() {
+        byte[] tail = videoSplitter.drain();
+        if (tail != null && tail.length > 0) {
+            videoDecoder.feedAccessUnit(tail);
+        }
+        videoDecoder.flush();
+        VideoDecoder.Frame frame;
+        while ((frame = videoDecoder.receive()) != null) {
+            videoQueue.add(new VideoFrame(
+                    assignVideoPts(), frame.width(), frame.height(),
+                    frame.pixelFormat(), frame.yPlane(), frame.yStride()));
+        }
+
+        if (audioDecoder != null) {
+            audioDecoder.flush();
+            byte[] pcm;
+            while ((pcm = audioDecoder.receive()) != null) {
+                audioQueue.add(new AudioChunk(
+                        audioFirstPts + audioFramesEmitted * AUDIO_FRAME_TICKS, pcm));
+            }
+        }
+    }
+
+    @Override
+    public void close() {
+        videoDecoder.close();
+        if (audioDecoder != null) {
+            audioDecoder.close();
+        }
+        demuxer.close();
+    }
+}

core/src/test/java/dev/quoije/umd/media/AnnexBSplitterTest.java

@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+package dev.quoije.umd.media;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class AnnexBSplitterTest {
+
+    @Test
+    void splitsCompleteAccessUnitsAndHoldsPartialTail() {
+        // SPS + slice1 + slice2 (slice2 has no following start code).
+        byte[] stream = {
+                0, 0, 0, 1, 0x67, 1, 2, 3,       // SPS
+                0, 0, 0, 1, 0x41, 4, 5, 6,       // slice (type 1)
+                0, 0, 0, 1, 0x41, 7, 8, 9,       // slice (type 1), incomplete tail
+        };
+
+        AnnexBSplitter splitter = new AnnexBSplitter();
+        splitter.feed(stream);
+
+        assertThat(splitter.nextAccessUnit()).isEqualTo(Arrays.copyOfRange(stream, 0, 16));
+        assertThat(splitter.nextAccessUnit()).isNull();
+        assertThat(splitter.drain()).isEqualTo(Arrays.copyOfRange(stream, 16, 24));
+    }
+
+    @Test
+    void holdsIncompleteSliceAcrossFeeds() {
+        byte[] stream = {
+                0, 0, 0, 1, 0x67, 1, 2, 3,
+                0, 0, 0, 1, 0x41, 4, 5, 6,
+                0, 0, 0, 1, 0x41, 7, 8, 9,
+        };
+
+        AnnexBSplitter splitter = new AnnexBSplitter();
+        splitter.feed(Arrays.copyOfRange(stream, 0, 15));
+        // The slice NAL's end (the next start code) has not arrived yet.
+        assertThat(splitter.nextAccessUnit()).isNull();
+
+        splitter.feed(Arrays.copyOfRange(stream, 15, stream.length));
+        assertThat(splitter.nextAccessUnit()).isEqualTo(Arrays.copyOfRange(stream, 0, 16));
+        assertThat(splitter.nextAccessUnit()).isNull();
+        assertThat(splitter.drain()).isEqualTo(Arrays.copyOfRange(stream, 16, 24));
+    }
+}