From 6c7cc4b32266ccab53d275b8af3838996e85a8a8 Mon Sep 17 00:00:00 2001 From: Sads Date: Fri, 12 Jun 2026 18:13:03 +0200 Subject: [PATCH] Video: show real duration, stay on replay prompt at end Fix the playback HUD showing "/ 0:00": duration is re-polled in the render loop until sceAvPlayerGetStreamInfo reports it, instead of being latched once before async parsing finishes. When a video reaches its end, hold on the last frame with an "X replay / O back" prompt instead of returning to the detail screen (which, for server videos, forced a full re-download to watch again). X replays from the still-local file via re-init; the decoder claim is held across replays. Also add the agentic loop tooling/docs: AgenticLoop.md runbook (build, install, drive Vita3K via kwin-mcp, zero-context-cost frame capture), controls.MD key map, deploy.sh, .mcp.json, and gitignore build-native/ and agent-run/. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 2 + .mcp.json | 8 ++++ AgenticLoop.md | 124 +++++++++++++++++++++++++++++++++++++++++++++++++ controls.MD | 29 ++++++++++++ deploy.sh | 25 ++++++++++ src/main.c | 60 +++++++++++++++++++++--- 6 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 .mcp.json create mode 100644 AgenticLoop.md create mode 100644 controls.MD create mode 100755 deploy.sh diff --git a/.gitignore b/.gitignore index 567609b..0db3778 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ build/ +build-native/ +agent-run/ diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..2a633cf --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "kwin-mcp": { + "command": "uvx", + "args": ["kwin-mcp"] + } + } +} diff --git a/AgenticLoop.md b/AgenticLoop.md new file mode 100644 index 0000000..37f8de6 --- /dev/null +++ b/AgenticLoop.md @@ -0,0 +1,124 @@ +# Agentic loop: build, run & drive vitaImmich in Vita3K + +This is the loop an agent uses to make a change, run it in the Vita3K emulator, +and verify it by sending input + reading screenshots — all headless, on KDE +Plasma (Wayland). + +## One-time setup + +- VitaSDK (community): native toolchain at `~/vitasdk` (`arm-vita-eabi-gcc`, + `vita-pack-vpk`, toolchain at `share/vita.toolchain.cmake`). +- Vita3K installed and on `PATH` (`/usr/bin/vita3k`). +- The MCP that drives the GUI: `uv tool install kwin-mcp` (configured in + `.mcp.json` as the `kwin-mcp` server). + +## 1. Build the VPK → `build/vitaImmich.vpk` + +Preferred (Docker, reproducible): + +```bash +./build.sh # writes build/vitaImmich.vpk (build/ ends up root-owned) +``` + +Fallback when the Docker runtime is broken (e.g. containerd shim error +`readlink /proc/self/exe: no such file or directory`), build natively with the +host SDK into `build-native/` (git-ignored): + +```bash +export VITASDK="$HOME/vitasdk" +export PATH=$VITASDK/bin:$PATH +cmake -B build-native -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$VITASDK/share/vita.toolchain.cmake . +cmake --build build-native --parallel "$(nproc)" +``` + +## 2. Install the VPK into Vita3K + +Installing = launch Vita3K once with the `.vpk` as an argument; it extracts and +auto-boots. Gotchas: + +- Run it **from a script file**, backgrounded with a wait then killed. Inline + backgrounding from the agent's shell does not persist. +- Guard `pkill` with `|| true` — a no-match exit code aborts the chain. +- Vita3K extracts files preserving the **zip's internal mtime**, so to confirm a + fresh install compare the installed `eboot.bin` mtime against + `unzip -l | grep eboot`, not wall-clock time. + +```bash +#!/bin/bash +cd "$(git rev-parse --show-toplevel)" +pkill -9 -f vita3k 2>/dev/null || true +sleep 1 +vita3k ./build-native/vitaImmich.vpk >/tmp/vinst.log 2>&1 & # or ./build/... +sleep 8 +pkill -9 -f vita3k 2>/dev/null || true +sleep 1 +grep -i "successfully" /tmp/vinst.log | tail -1 +stat -c 'eboot: %y' ~/.local/share/Vita3K/Vita3K/ux0/app/VIMM00001/eboot.bin +``` + +App title id: **`VIMM00001`**. To wipe a stale install: `vita3k -d VIMM00001`. + +## 3. Launch + drive via kwin-mcp + +Launch the app **inside the MCP's own isolated KWin session** — its screenshot +tool works there. (Connecting to the live desktop with `session_connect` does +*not* work for capture: `spectacle` can't save non-interactively, and KWin's +`ScreenShot2` DBus API rejects unauthorized callers — so the isolated session is +the only reliable way to capture, and it is headless: the human cannot watch it +live on their monitor.) + +- `session_start` with `app_command: "vita3k -r VIMM00001"` **and + `keep_screenshots: true`** — starts an isolated Wayland session, launches the + app, and keeps every captured frame on disk after `session_stop`. Give it ~12s + to boot. +- **`focus_window` with `app_name: "Vita3K"` BEFORE sending keys.** Without focus + the keypresses are dropped. Presses also drop intermittently — re-focus and + re-press. Send one key per call; rapid bursts get coalesced/dropped. +- `keyboard_key` with `screenshot_after_ms: [ms]` after **every** command, so the + frame folder is a complete visual record of the run for the human to scrub. +- `session_stop` when done (and `pkill -9 -f vita3k`). Frames survive in + `/tmp/kwin-mcp-screenshots-*/`; copy them to `./agent-run/` (git-ignored) and + tell the human where they are. + +### Zero-context-cost capture (the standing rule) + +Capturing a frame (writing the PNG) costs the agent **nothing** — files on disk +are not in context. The cost is only incurred when the agent **Reads** a PNG +back. So: + +- **Capture every command** (`screenshot_after_ms`) — free, and it gives the + human the full record. +- **Do not Read frames back** into context during the loop. Verify progress from + the app's own **text** log instead (cheap), not from images: + `~/.local/share/Vita3K/Vita3K/ux0/data/vitaimmich/log.txt`. +- Only Read a frame when a screenshot is genuinely the only way to resolve an + ambiguity (rare). Reading every frame is what bloats context and slows the + session — that is exactly what this rule avoids. + +### Key mapping (full list in `controls.MD`) + +Vita3K maps host keys → Vita buttons. The ones this app uses: + +| Vita button | host key (kwin-mcp) | in-app action | +|-------------|---------------------|--------------------------| +| D-pad | `Right` `Left` `Up` `Down` | move grid / browse | +| Cross (X) | `x` | view / play / replay | +| Circle (O) | `c` | back | +| Select | `Shift_R` | sync | +| Start | `Return` | exit | + +### Typical verify loop (capture everything, Read nothing) + +1. `session_start` (`keep_screenshots: true`) → wait ~12s. +2. `focus_window "Vita3K"`. +3. `keyboard_key "Right"` (with `screenshot_after_ms`) … stepping to the target + tile, one key per call. +4. `keyboard_key "x"` to open, `x` again to play. +5. Confirm what happened from the **text** log, not the frames: + - `~/.local/share/Vita3K/Vita3K/ux0/data/vitaimmich/log.txt` (the app's own log + — records page fetches, video open/play, scans, errors) + - MCP `read_app_log` with the launched PID, or `/tmp/vita3k-runtime.log`. +6. On `session_stop`, the captured frames remain in + `/tmp/kwin-mcp-screenshots-*/` — hand that folder (or a copy in `./agent-run/`) + to the human as the visual record. diff --git a/controls.MD b/controls.MD new file mode 100644 index 0000000..613aed4 --- /dev/null +++ b/controls.MD @@ -0,0 +1,29 @@ +D-pad + up: ArrowUp + left : ArrowLeft + right : ArrowRight + down : ArrowDown +Left Stick + Up : W + Left : A + Right : D + Down : S +L1 : Q +L2 : E +Face Buttons: + Triangle : V + Square : Z + Circle : C + Cross : X +Right Stick: + Up : I + Left : J + Right : L + Down: K +PS Button : P +Select : Shift Right +Start : Enter +L2 : U +L3 : F +R3 : H +R2 : O diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..4dc62be --- /dev/null +++ b/deploy.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +TITLE="VIMM00001" + +cd "$(dirname "$0")" + +pkill -9 -f vita3k 2>/dev/null || true +sleep 1 + +# Install +vita3k ./build/vitaImmich.vpk >/tmp/vita3k-install.log 2>&1 & +sleep 4 + +pkill -9 -f vita3k 2>/dev/null || true +sleep 1 + +# Launch detached +nohup vita3k -r "$TITLE" \ + >/tmp/vita3k-runtime.log 2>&1 \ + = 0) - duration = sinfo.duration; - } /* four rotating frame wrappers (matching numOutputVideoFrameBuffers): * while the GPU still samples one frame the decoder fills another, and @@ -1694,6 +1697,7 @@ static void play_video_file(const char *path) vita2d_texture *cur = NULL; int paused = 0; + int ended_eos = 0; unsigned int prev = 0xffffffff; /* swallow the X press that got us here */ while (active) { @@ -1721,8 +1725,10 @@ static void play_video_file(const char *path) sceAvPlayerJumpToTime(g_avp, t); } - if (!sceAvPlayerIsActive(g_avp)) + if (!sceAvPlayerIsActive(g_avp)) { + ended_eos = 1; break; /* end of stream */ + } if (sceAvPlayerGetVideoData(g_avp, &vframe[buf_idx])) { sceGxmTextureInitLinear(&vtex[buf_idx].gxm_tex, @@ -1740,6 +1746,13 @@ static void play_video_file(const char *path) if (cur) draw_texture_fitted(cur, 0, 0, SCREEN_W, SCREEN_H); + if (duration == 0) { + SceAvPlayerStreamInfo sinfo; + memset(&sinfo, 0, sizeof(sinfo)); + if (sceAvPlayerGetStreamInfo(g_avp, 0, &sinfo) >= 0) + duration = sinfo.duration; + } + unsigned int cs = (unsigned int)(sceAvPlayerCurrentTime(g_avp) / 1000); unsigned int ds = (unsigned int)(duration / 1000); char hud[160]; @@ -1761,6 +1774,36 @@ static void play_video_file(const char *path) "check Immich transcoding settings."); } + /* reached the end of the video: hold on the last frame with a replay + * prompt instead of returning (which, for a server video, drops the user + * back to the detail screen and would re-download to play it again). X + * replays from the still-local file; O leaves. The decoder buffers behind + * `cur` are still valid here — teardown happens below, after this loop. */ + if (ended_eos) { + unsigned int eprev = 0xffffffff; + for (;;) { + SceCtrlData pad; + sceCtrlPeekBufferPositive(0, &pad, 1); + unsigned int pressed = pad.buttons & ~eprev; + eprev = pad.buttons; + + if (pressed & SCE_CTRL_CIRCLE) + break; + if (pressed & SCE_CTRL_CROSS) { + want_replay = 1; + break; + } + + vita2d_start_drawing(); + vita2d_clear_screen(); + if (cur) + draw_texture_fitted(cur, 0, 0, SCREEN_W, SCREEN_H); + draw_hud("ended X replay O back"); + vita2d_end_drawing(); + vita2d_swap_buffers(); + } + } + g_av_audio_run = 0; if (audio_thid >= 0) { sceKernelWaitThreadEnd(audio_thid, NULL, NULL); @@ -1771,6 +1814,9 @@ static void play_video_file(const char *path) sceAvPlayerStop(g_avp); sceAvPlayerClose(g_avp); + if (want_replay) + goto replay; /* re-init from the same file; no re-download */ + /* release the decoder so the worker can resume poster extraction */ g_player_active = 0; __sync_synchronize();