From c868b30523aa89dd1041019428ef31e317f4199c Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 9 Oct 2026 10:13:14 +0200 Subject: [PATCH 01/10] Add fls-wh: a block-device write head for remote system writes Runs on a remote host, opens a raw block device (e.g. /dev/emmc0), and writes framed commands to it over stdin/stdout. the remote counterpart of the local O_DIRECT writer (SYNC = fdatasync; fsync on macOS). Builds as flswh-aarch64-qnx7 (cross) and flswh-- (native); CI runs it first and keeps both binaries as artifacts for later fls testing. Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- .github/workflows/ci.yml | 55 +++++++ Makefile | 13 +- remote/.gitignore | 3 + remote/Containerfile | 12 ++ remote/Makefile | 76 +++++++++ remote/README.md | 49 ++++++ remote/fls-wh.c | 325 +++++++++++++++++++++++++++++++++++++ remote/include/errno.h | 6 + remote/include/fcntl.h | 9 + remote/include/string.h | 6 + remote/include/sys/types.h | 7 + remote/include/unistd.h | 13 ++ remote/start.S | 11 ++ remote/stubs/libc.min.json | 16 ++ remote/stubs/so_stub.py | 176 ++++++++++++++++++++ remote/test.py | 120 ++++++++++++++ 16 files changed, 896 insertions(+), 1 deletion(-) create mode 100644 remote/.gitignore create mode 100644 remote/Containerfile create mode 100644 remote/Makefile create mode 100644 remote/README.md create mode 100644 remote/fls-wh.c create mode 100644 remote/include/errno.h create mode 100644 remote/include/fcntl.h create mode 100644 remote/include/string.h create mode 100644 remote/include/sys/types.h create mode 100644 remote/include/unistd.h create mode 100644 remote/start.S create mode 100644 remote/stubs/libc.min.json create mode 100644 remote/stubs/so_stub.py create mode 100644 remote/test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a21dd3a..cf20933 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,59 @@ env: CARGO_TERM_COLOR: always jobs: + # runs first: builds the write-head binaries and leaves them as artifacts + # for later fls jobs (add `needs: write-head` + actions/download-artifact + # in a job that wants them) + write-head: + name: Write head (fls-wh) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # build the QNX cross binary in the fedora container (remote/ has its + # own Containerfile with the cross toolchain) + - name: Build in fedora container (podman) + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq podman + podman build -t fls-wh-builder remote/ + podman run --rm -v "$GITHUB_WORKSPACE":/src -w /src fls-wh-builder \ + make -C remote LD=aarch64-linux-gnu-ld STUB_LD=aarch64-linux-gnu-ld + + # the QNX binary can't run on a linux runner, so verify the artifact: + # aarch64 ELF, QNX interpreter, correct SONAME and NEEDED entries + - name: Verify artifact + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq binutils-aarch64-linux-gnu + file remote/flswh-aarch64-qnx7 | grep -q 'ARM aarch64' + file remote/flswh-aarch64-qnx7 | grep -q '/usr/lib/ldqnx-64.so.2' + aarch64-linux-gnu-readelf -d remote/flswh-aarch64-qnx7 | grep -q 'NEEDED.*libc.so.5' + aarch64-linux-gnu-readelf -d remote/stubs/libc.min.so_stub.so | grep -q 'SONAME.*libc.so.5' + + # runtime test: build the same source natively and drive it with a + # framed command script against a temp file; assert records + bytes + - name: Native test + run: python3 remote/test.py + + # build the native (runner) binary too, kept for later fls testing + - name: Build native + run: make -C remote native + + - name: Upload flswh-aarch64-qnx7 + uses: actions/upload-artifact@v4 + with: + name: flswh-aarch64-qnx7 + path: remote/flswh-aarch64-qnx7 + if-no-files-found: error + + - name: Upload flswh-x86_64-linux + uses: actions/upload-artifact@v4 + with: + name: flswh-x86_64-linux + path: remote/flswh-x86_64-linux + if-no-files-found: error + lint: name: Lint runs-on: ubuntu-latest @@ -152,3 +205,5 @@ jobs: path: ${{ matrix.artifact_name }} if-no-files-found: error + + diff --git a/Makefile b/Makefile index a2a344f..ef25efd 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Makefile for fls - ARM64 static builds -.PHONY: all build clean test arm64-static lint fmt clippy check help +.PHONY: all build clean test arm64-static remote-wh remote-wh-native lint fmt clippy check help # Default target all: build @@ -24,6 +24,14 @@ arm64-static: all-targets: ./build.sh +# Build the remote write head (fls/remote) for QNX aarch64 +remote-wh: + $(MAKE) -C remote + +# Build the remote write head for the host (native) +remote-wh-native: + $(MAKE) -C remote native + # Run tests test: cargo test @@ -55,6 +63,7 @@ check: clean: cargo clean rm -f fls-* + $(MAKE) -C remote clean # Install dependencies for ARM64 cross-compilation install-deps: @@ -72,6 +81,8 @@ help: @echo " build - Build for native target" @echo " arm64-static - Build for ARM64 static (musl)" @echo " all-targets - Build for native + ARM64 static" + @echo " remote-wh - Build the remote write head for QNX aarch64" + @echo " remote-wh-native - Build the remote write head for the host" @echo "" @echo "Testing and Quality:" @echo " test - Run tests" diff --git a/remote/.gitignore b/remote/.gitignore new file mode 100644 index 0000000..a63f436 --- /dev/null +++ b/remote/.gitignore @@ -0,0 +1,3 @@ +*.o +flswh-* +stubs/*.so_stub.so* diff --git a/remote/Containerfile b/remote/Containerfile new file mode 100644 index 0000000..90e48fc --- /dev/null +++ b/remote/Containerfile @@ -0,0 +1,12 @@ +FROM fedora:latest + +# gcc: native compiler (runs the native test; system headers) +# gcc-aarch64-linux-gnu: cross compiler (provides aarch64-linux-gnu-gcc) +# binutils-aarch64-linux-gnu: GNU ld (provides aarch64-linux-gnu-ld) +# python3: runs stubs/so_stub.py; make: the build itself +RUN dnf install -y gcc gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu python3 make \ + && dnf clean all + +# The Makefile and so_stub.py hardcode the QNX-style tool name; map it +# to Fedora's. (ld.lld is already named correctly by the lld package.) +RUN ln -s /usr/bin/aarch64-linux-gnu-gcc /usr/local/bin/aarch64-elf-gcc diff --git a/remote/Makefile b/remote/Makefile new file mode 100644 index 0000000..048747f --- /dev/null +++ b/remote/Makefile @@ -0,0 +1,76 @@ +# fls-wh — a block-device write head, built for a QNX aarch64 target. +# +# Built from: +# - this source (fls-wh.c, start.S) +# - standard C/POSIX headers (include/) +# - a minimal JSON symbol manifest (stubs/*.min.json) <- source of truth +# - open tooling: aarch64-elf-gcc (GCC) + ld.lld (LLVM) + stubs/so_stub.py +# +# The stub .so files are build artifacts (gitignored). At runtime, the +# target's dynamic linker resolves the SONAME (libc.so.5) to the library +# present on the system. +# +# Container build: `make container-build` builds the Containerfile with +# podman and runs the whole build inside it (fedora + cross toolchain). +# +# `make help` prints this file's `##`-annotated targets, grouped by +# `##@` section headers. + +CC := aarch64-elf-gcc +LD := ld.lld +CFLAGS := -I include +LDFLAGS := -dynamic-linker /usr/lib/ldqnx-64.so.2 +STUBDIR := stubs + +CONTAINER ?= podman +IMAGE ?= fls-wh-builder + +# host arch/os for the native binary name (flswh--) +UNAME_M := $(shell uname -m) +UNAME_S := $(shell uname -s) +NATIVE_ARCH := $(if $(filter arm64,$(UNAME_M)),aarch64,$(UNAME_M)) +NATIVE_OS := $(if $(filter Darwin,$(UNAME_S)),darwin,$(if $(filter Linux,$(UNAME_S)),linux,$(UNAME_S))) +NATIVE_BIN := flswh-$(NATIVE_ARCH)-$(NATIVE_OS) + +.PHONY: all clean stubs native container container-build test help + +##@ build +all: flswh-aarch64-qnx7 ## build fls-wh for QNX aarch64 (default) + +flswh-aarch64-qnx7: fls-wh.o start.o stubs ## link the QNX aarch64 binary + $(LD) -o $@ fls-wh.o start.o -L$(STUBDIR) -l:libc.min.so_stub.so $(LDFLAGS) + +native: $(NATIVE_BIN) ## build fls-wh for the host (system compiler, no stubs) + +$(NATIVE_BIN): fls-wh.c + cc -O2 $< -o $@ + +# build the minimal stub .so from the minimal JSON (artifacts, gitignored) +stubs: ## build the stub .so files from the JSON manifests + python3 stubs/so_stub.py $(STUBDIR)/libc.min.json -o $(STUBDIR)/libc.min.so_stub.so + +fls-wh.o: fls-wh.c + $(CC) $(CFLAGS) -c $< -o $@ + +start.o: start.S + $(CC) -c $< -o $@ + +clean: ## remove all build artifacts + rm -f flswh-* fls-wh.o start.o $(STUBDIR)/*.so_stub.so* + +##@ test +test: ## native test: build with the system compiler, run, assert + python3 test.py + +##@ container (podman) +container: ## build the fedora container image + $(CONTAINER) build -t $(IMAGE) . + +container-build: container ## build fls-wh inside the container (GNU ld) + $(CONTAINER) run --rm -v $(PWD):/src -w /src $(IMAGE) make \ + LD=aarch64-linux-gnu-ld STUB_LD=aarch64-linux-gnu-ld + +help: ## show this help + @awk 'BEGIN {FS = ":.*##"} \ + /^##@/ { printf "\n\033[1;33m%s\033[0m\n", substr($$0, 5) } \ + /^[a-zA-Z0-9_-]+:.*##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) diff --git a/remote/README.md b/remote/README.md new file mode 100644 index 0000000..b73faef --- /dev/null +++ b/remote/README.md @@ -0,0 +1,49 @@ +# fls-wh — the remote write head + +`fls-wh` is a small C program that runs on a remote system (QNX today; +extensible to others, e.g. a U-Boot command talking over the network) and +writes framed commands to a raw block device (e.g. `/dev/emmc0`). It is the +remote counterpart of the local writer in `src/fls/block_writer.rs` (which +uses `O_DIRECT` on Linux/macOS). QNX has no `O_DIRECT`, so durability comes +from an explicit `SYNC` (`fdatasync`; `fsync` on macOS, which has no +`fdatasync`). + +## Usage + +``` +./flswh-- +``` + +Opens `` (`O_RDWR`), reports its size (`READY`), then reads framed +commands from stdin and writes them to the device, emitting framed records to +stdout. Intended to be driven by a remote client (e.g. over SSH). QUIT or +EOF exits. +Device I/O errors are reported (`ERR`) and processing continues. + +## Protocol + +Frame (both directions): `[magic:4][opcode:1][payload_len:u32 LE][payload]`, +little-endian throughout. The magic (`FLSW`) is a sync header: the reader +scans for it to resync after any corruption. + +| | opcode | | opcode | +|---|---|---|---| +| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` | +| **Reserved** | `AUTH 0x07` `HELLO 0x08` | | `AUTH_RESULT 0x87` `HELLO 0x88` | + +The `READY` payload carries a protocol version byte (currently 1) for +forward compatibility. `DATA` is `size:u32 + content + crc:u32` (CRC32-IEEE); +`READ` is `len:u64` (reads from the current offset, advances it, capped at +1 MiB) and returns `READ_DATA` with the bytes. + +## Build + +Binaries are named `flswh--` (matching the `fls` artifact scheme): + +- **QNX aarch64** (cross): `make` (or `make -C remote` from the repo root) → `flswh-aarch64-qnx7`. Needs an aarch64 cross gcc toolchain (the Makefile's `aarch64-elf-gcc` + `ld.lld` names map to any gcc cross toolchain; `make container-build` uses Fedora's in a container). +- **Native** (host, incl. macOS): `make native` → `flswh--` (e.g. `flswh-aarch64-darwin`, `flswh-x86_64-linux`). Uses the system compiler and system headers, no stubs. + +## Test + +`python3 test.py` — builds the source with the system compiler, drives it with +a framed command script against a temp file, and asserts the records and bytes. diff --git a/remote/fls-wh.c b/remote/fls-wh.c new file mode 100644 index 0000000..200a7b5 --- /dev/null +++ b/remote/fls-wh.c @@ -0,0 +1,325 @@ +#include +#include +#include +#include +#include + +/* macOS has no fdatasync(); fsync() is the equivalent. */ +#ifdef __APPLE__ +#define sync_file(fd) fsync(fd) +#else +#define sync_file(fd) fdatasync(fd) +#endif + +/* fls-wh: a block-device write head. + * + * Usage: ./fls-wh + * + * Opens (O_RDWR), reports its size, then reads framed commands from + * fd 0 (stdin) and writes them to the device, emitting framed records to fd 1 + * (stdout). Intended to be driven over SSH by a remote client (the Rust `fls` + * tool). The device is a raw block device (e.g. /dev/emmc0); there is no + * O_DIRECT on QNX, so durability comes from an explicit SYNC (fdatasync; + * fsync on macOS, which has no fdatasync). + * + * Frame (both directions): [magic:4][opcode:1][payload_len:u32 LE][payload] + * Little-endian throughout. The magic is a sync header: the reader scans for + * it to resync after any corruption. + * + * Requests: DATA(0x01) SKIP(0x02) SEEK(0x03) SYNC(0x04) QUIT(0x05) + * READ(0x06) [implemented] + * AUTH(0x07) HELLO(0x08) [reserved, spec only] + * Responses: READY(0x81) OK(0x82) PROG(0x83) ERR(0x84) DONE(0x85) + * READ_DATA(0x86) [implemented] + * AUTH_RESULT(0x87) HELLO(0x88) [reserved] + * + * DATA payload: size:u32 + content[size] + crc:u32 (CRC32-IEEE of content) + * SKIP payload: len:u64 + * SEEK payload: offset:u64 + * SYNC/QUIT: (no payload) + * READ payload: len:u64 (read len bytes from the current offset, advance it; + * len must be <= 1 MiB) + * + * READY payload: version:u8 + size:u64 + * OK payload: op:1 + offset:u64 + bytes:u64 + * PROG payload: offset:u64 + bytes_done:u64 (every 1 MiB of a DATA) + * ERR payload: code:1 + op:1 + errno:u32 + offset:u64 + * code: 0=device I/O error, 1=CRC mismatch, 2=not implemented + * DONE payload: total_bytes:u64 + * READ_DATA payload: content (the bytes read; the frame's payload_len is the + * count, which may be < requested near end of device) + * + * Uses read()/write() on fds directly (portable; some libc variants do not + * export stdin/stdout/stderr). No userspace buffering, so every record is + * written to the kernel immediately (no flush needed over SSH). + */ + +enum { + OP_DATA = 0x01, OP_SKIP = 0x02, OP_SEEK = 0x03, OP_SYNC = 0x04, + OP_QUIT = 0x05, + OP_READ = 0x06, OP_AUTH = 0x07, OP_HELLO = 0x08, /* reserved */ + R_READY = 0x81, R_ACK = 0x82, R_PROG = 0x83, R_ERR = 0x84, R_DONE = 0x85, + R_READ_DATA = 0x86, R_AUTH_RESULT = 0x87, R_HELLO = 0x88 /* reserved */ +}; +enum { ERR_DEV = 0, ERR_CRC = 1, ERR_NI = 2 }; /* NI = not implemented */ + +static const unsigned char MAGIC[4] = { 0x46, 0x4c, 0x53, 0x57 }; /* "FLSW" */ +static const unsigned char PROTO_VER = 1; +static const size_t PROG_STEP = 1u << 20; /* emit PROG every 1 MiB */ +#define MAX_READ (1u << 20) /* cap a single READ at 1 MiB */ +static unsigned char dbuf[MAX_READ]; /* device read buffer */ + +/* --- little-endian packing --- */ +static unsigned int rd_u32(const unsigned char *p) +{ return p[0] | p[1] << 8 | p[2] << 16 | (unsigned)p[3] << 24; } +static unsigned long long rd_u64(const unsigned char *p) +{ return (unsigned long long)rd_u32(p) | ((unsigned long long)rd_u32(p + 4) << 32); } +static void wr_u32(unsigned char *p, unsigned int v) +{ p[0] = v; p[1] = v >> 8; p[2] = v >> 16; p[3] = v >> 24; } +static void wr_u64(unsigned char *p, unsigned long long v) +{ wr_u32(p, (unsigned int)v); wr_u32(p + 4, (unsigned int)(v >> 32)); } + +/* --- CRC32 (IEEE), running state so it can be computed while streaming --- */ +static unsigned int crc_tab[256]; +static unsigned int crc_state; +static void crc_init(void) +{ + for (unsigned int i = 0; i < 256; i++) { + unsigned int c = i; + for (int k = 0; k < 8; k++) + c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; + crc_tab[i] = c; + } +} +static void crc_reset(void) { crc_state = 0xFFFFFFFFu; } +static void crc_update(const unsigned char *b, size_t n) +{ + unsigned int c = crc_state; + for (size_t i = 0; i < n; i++) + c = crc_tab[(c ^ b[i]) & 0xFF] ^ (c >> 8); + crc_state = c; +} +static unsigned int crc_final(void) { return crc_state ^ 0xFFFFFFFFu; } + +/* --- framed output (stdout) --- */ +static void emit(int op, const unsigned char *payload, size_t len) +{ + unsigned char hdr[9]; + hdr[0] = MAGIC[0]; hdr[1] = MAGIC[1]; + hdr[2] = MAGIC[2]; hdr[3] = MAGIC[3]; + hdr[4] = (unsigned char)op; + wr_u32(hdr + 5, (unsigned int)len); + (void)write(1, hdr, 9); + if (len) (void)write(1, payload, len); +} +static void ok_rec(int op, unsigned long long off, unsigned long long bytes) +{ + unsigned char p[17]; + p[0] = (unsigned char)op; wr_u64(p + 1, off); wr_u64(p + 9, bytes); + emit(R_ACK, p, sizeof p); +} +static void err_rec(int code, int op, unsigned int e, unsigned long long off) +{ + unsigned char p[14]; + p[0] = (unsigned char)code; p[1] = (unsigned char)op; + wr_u32(p + 2, e); wr_u64(p + 6, off); + emit(R_ERR, p, sizeof p); +} + +/* --- buffered reader (stdin) --- */ +static unsigned char rbuf[1 << 16]; +static size_t rlen = 0, rpos = 0; +static int rfill(void) +{ + if (rpos < rlen) return 1; + ssize_t n = read(0, rbuf, sizeof rbuf); + if (n <= 0) { rlen = 0; rpos = 0; return 0; } + rlen = (size_t)n; rpos = 0; return 1; +} +static int rnext(unsigned char *b) +{ + if (!rfill()) return 0; + *b = rbuf[rpos++]; + return 1; +} +static int rreadn(unsigned char *dst, size_t n) +{ + while (n > 0) { + if (rpos >= rlen && !rfill()) return 0; + size_t avail = rlen - rpos; + size_t take = (n < avail) ? n : avail; + memcpy(dst, rbuf + rpos, take); + dst += take; rpos += take; n -= take; + } + return 1; +} +/* scan for the 4-byte magic (resyncs after corruption); 0 on EOF */ +static int read_magic(void) +{ + unsigned char w[3] = { 0, 0, 0 }; + for (;;) { + unsigned char b; + if (!rnext(&b)) return 0; + if (w[0] == MAGIC[0] && w[1] == MAGIC[1] && + w[2] == MAGIC[2] && b == MAGIC[3]) + return 1; + w[0] = w[1]; w[1] = w[2]; w[2] = b; + } +} +/* skip n bytes from the stream (drop a malformed frame's remainder); 0 on EOF */ +static int skip(unsigned long long n) +{ + unsigned char b[1 << 16]; + while (n > 0) { + size_t take = (n > sizeof b) ? sizeof b : (size_t)n; + if (!rreadn(b, take)) return 0; + n -= take; + } + return 1; +} + +/* write all, retrying on partial writes; -1 on error (errno set) */ +static int write_all(int fd, const unsigned char *buf, size_t len) +{ + size_t off = 0; + while (off < len) { + ssize_t n = write(fd, buf + off, len - off); + if (n < 0) return -1; + off += (size_t)n; + } + return 0; +} + +/* read all, retrying on partial reads; -1 on error (errno set), else the bytes + * read (which may be < len near end of device) */ +static int read_all(int fd, unsigned char *buf, size_t len) +{ + size_t off = 0; + while (off < len) { + ssize_t n = read(fd, buf + off, len - off); + if (n < 0) return -1; + if (n == 0) return (int)off; /* EOF: partial read */ + off += (size_t)n; + } + return (int)len; +} + +int main(int argc, char **argv) +{ + if (argc < 2) { err_rec(ERR_DEV, 0, 22, 0); return 1; } /* 22 = EINVAL */ + int fd = open(argv[1], O_RDWR); + if (fd < 0) { err_rec(ERR_DEV, 0, (unsigned)errno, 0); return 1; } + + long long sz = lseek(fd, 0, SEEK_END); + lseek(fd, 0, SEEK_SET); /* the size query left the offset at the end */ + unsigned char rp[9]; + rp[0] = PROTO_VER; wr_u64(rp + 1, (unsigned long long)(sz < 0 ? 0 : sz)); + emit(R_READY, rp, sizeof rp); + + unsigned long long cur = 0; /* current logical write position */ + unsigned long long total = 0; + crc_init(); + + for (;;) { + if (!read_magic()) break; /* EOF */ + unsigned char op; + if (!rnext(&op)) break; + unsigned char pl[4]; + if (!rreadn(pl, 4)) break; + unsigned long long plen = rd_u32(pl); + + if (op == OP_QUIT) + break; + + else if (op == OP_DATA) { + unsigned char szf[4]; + if (!rreadn(szf, 4)) break; + unsigned int size = rd_u32(szf); + if (plen != 8 + (unsigned long long)size) { /* malformed frame */ + if (plen >= 4) skip(plen - 4); + err_rec(ERR_NI, OP_DATA, 0, cur); + continue; + } + unsigned long long start = cur; + crc_reset(); + unsigned long long done = 0, next_prog = PROG_STEP; + int status = 0; /* 0=ok, 1=eof, 2=write error */ + unsigned char chunk[1 << 16]; + while (done < size) { + size_t want = (size_t)(size - done); + if (want > sizeof chunk) want = sizeof chunk; + if (!rreadn(chunk, want)) { status = 1; break; } + crc_update(chunk, want); + if (write_all(fd, chunk, want) < 0) { status = 2; break; } + done += want; + while (done >= next_prog) { + unsigned char pp[16]; + wr_u64(pp, start + done); wr_u64(pp + 8, done); + emit(R_PROG, pp, sizeof pp); + next_prog += PROG_STEP; + } + } + cur = start + done; + total += done; + if (status == 1) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + if (status == 2) { err_rec(ERR_DEV, OP_DATA, (unsigned)errno, start); } + else if (done < size) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + else { + unsigned char crcf[4]; + if (!rreadn(crcf, 4)) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + if (crc_final() != rd_u32(crcf)) + err_rec(ERR_CRC, OP_DATA, 0, start); + else + ok_rec(OP_DATA, start, done); + } + } + + else if (op == OP_SKIP) { + if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_SKIP, 0, cur); continue; } + unsigned char lf[8]; + if (!rreadn(lf, 8)) break; + unsigned long long len = rd_u64(lf); + if (lseek(fd, cur + len, SEEK_SET) < 0) + err_rec(ERR_DEV, OP_SKIP, (unsigned)errno, cur); + else { ok_rec(OP_SKIP, cur, len); cur += len; } + } + + else if (op == OP_SEEK) { + if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_SEEK, 0, cur); continue; } + unsigned char of[8]; + if (!rreadn(of, 8)) break; + unsigned long long off = rd_u64(of); + if (lseek(fd, off, SEEK_SET) < 0) + err_rec(ERR_DEV, OP_SEEK, (unsigned)errno, off); + else { ok_rec(OP_SEEK, off, 0); cur = off; } + } + + else if (op == OP_READ) { + if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_READ, 0, cur); continue; } + unsigned char lf[8]; + if (!rreadn(lf, 8)) break; + unsigned long long len = rd_u64(lf); + if (len > MAX_READ) { err_rec(ERR_DEV, OP_READ, 22, cur); continue; } /* 22 = EINVAL */ + int n = read_all(fd, dbuf, (size_t)len); + if (n < 0) { err_rec(ERR_DEV, OP_READ, (unsigned)errno, cur); continue; } + emit(R_READ_DATA, dbuf, (size_t)n); /* the bytes read (may be empty) */ + cur += (unsigned long long)n; + } + + else if (op == OP_SYNC) { + if (plen != 0) { skip(plen); err_rec(ERR_NI, OP_SYNC, 0, cur); continue; } + (void)sync_file(fd); + ok_rec(OP_SYNC, cur, 0); + } + + else { /* reserved (AUTH/HELLO) or unknown opcode */ + skip(plen); + err_rec(ERR_NI, op, 0, cur); + } + } + + unsigned char dp[8]; + wr_u64(dp, total); + emit(R_DONE, dp, sizeof dp); + return 0; +} diff --git a/remote/include/errno.h b/remote/include/errno.h new file mode 100644 index 0000000..45d47ec --- /dev/null +++ b/remote/include/errno.h @@ -0,0 +1,6 @@ +#ifndef _ERRNO_H +#define _ERRNO_H +/* QNX: errno is a per-thread location reached via __get_errno_ptr(). */ +int *__get_errno_ptr(void); +#define errno (*__get_errno_ptr()) +#endif diff --git a/remote/include/fcntl.h b/remote/include/fcntl.h new file mode 100644 index 0000000..b51af87 --- /dev/null +++ b/remote/include/fcntl.h @@ -0,0 +1,9 @@ +#ifndef _FCNTL_H +#define _FCNTL_H + +int open(const char *path, int flags); + +#define O_RDONLY 0 +#define O_WRONLY 1 +#define O_RDWR 2 +#endif diff --git a/remote/include/string.h b/remote/include/string.h new file mode 100644 index 0000000..405c5a0 --- /dev/null +++ b/remote/include/string.h @@ -0,0 +1,6 @@ +#ifndef _STRING_H +#define _STRING_H +#include + +void *memcpy(void *dst, const void *src, size_t n); +#endif diff --git a/remote/include/sys/types.h b/remote/include/sys/types.h new file mode 100644 index 0000000..f817ef1 --- /dev/null +++ b/remote/include/sys/types.h @@ -0,0 +1,7 @@ +#ifndef _SYS_TYPES_H +#define _SYS_TYPES_H +/* Basic types (POSIX-compatible). */ +typedef unsigned long size_t; +typedef long ssize_t; +typedef long off_t; +#endif diff --git a/remote/include/unistd.h b/remote/include/unistd.h new file mode 100644 index 0000000..c7b430d --- /dev/null +++ b/remote/include/unistd.h @@ -0,0 +1,13 @@ +#ifndef _UNISTD_H +#define _UNISTD_H +#include + +ssize_t read(int fd, void *buf, size_t count); +ssize_t write(int fd, const void *buf, size_t count); +off_t lseek(int fd, off_t offset, int whence); +int fdatasync(int fd); + +#define SEEK_SET 0 +#define SEEK_CUR 1 +#define SEEK_END 2 +#endif diff --git a/remote/start.S b/remote/start.S new file mode 100644 index 0000000..b22ca03 --- /dev/null +++ b/remote/start.S @@ -0,0 +1,11 @@ + .globl _start + .type _start, @function +_start: + /* initial stack: [argc, argv..., NULL, envp..., NULL] */ + ldr x0, [sp] /* x0 = argc */ + add x1, sp, #8 /* x1 = argv = sp+8 */ + add x2, x1, x0, lsl #3 /* x2 = argv + argc*8 */ + add x2, x2, #8 /* x2 = envp (skip NULL terminator) */ + bl main /* main(argc, argv, envp) */ + bl exit /* exit(main's return) */ + .size _start, .-_start diff --git a/remote/stubs/libc.min.json b/remote/stubs/libc.min.json new file mode 100644 index 0000000..d2f4d94 --- /dev/null +++ b/remote/stubs/libc.min.json @@ -0,0 +1,16 @@ +{ + "version": 1, + "soname": "libc.so.5", + "needed": [], + "funcs": [ + "exit", + "open", + "lseek", + "write", + "read", + "fdatasync", + "memcpy", + "__get_errno_ptr" + ], + "vars": [] +} diff --git a/remote/stubs/so_stub.py b/remote/stubs/so_stub.py new file mode 100644 index 0000000..b45a068 --- /dev/null +++ b/remote/stubs/so_stub.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +""" +Build a minimal stub .so from a JSON symbol manifest. + +Given a JSON file describing a library (its SONAME plus the symbol names +it exports), this tool assembles and links a standard, well-formed shared +library that exports exactly those names. The stub contains no real code +(each function is a `ret` no-op) and no real data (each variable is 8 +zero bytes) just enough for a linker to resolve symbol names against +the library's SONAME. + +JSON input format (version 1): + { + "version": 1, + "soname": "libz.so.2", + "needed": [], # reserved, currently unused + "funcs": ["inflate", ...], # exported as STT_FUNC + "vars": ["errno", ...] # exported as STT_OBJECT, 8 bytes each + } + +Usage: + python3 stubs/so_stub.py [-o out.so] + e.g. python3 stubs/so_stub.py stubs/libz.min.json -o stubs/libz.min.so_stub.so +""" +import json +import os +import struct +import subprocess +import sys + +# --- ELF constants (64-bit, little-endian) -------------------------------- +PT_LOAD = 1 # program header: loadable segment (vaddr -> file offset) +PT_DYNAMIC = 2 # program header: dynamic section (DT_* entries) +DT_NULL = 0 # end of the dynamic section +DT_STRTAB = 5 # virtual address of the dynamic string table +DT_SONAME = 11 # string-table offset of the library's SONAME + +# Offsets within a 64-bit ELF header +E_PHOFF = 0x20 # e_phoff: file offset of the program header table +E_PHNUM = 0x38 # e_phnum: number of program headers +E_PHENTSIZE = 0x36 # e_phentsize: size of one program header + + +class Elf: + """Minimal ELF reader: find a segment by type, map vaddrs to file offsets.""" + + def __init__(self, data): + self.d = data + self.e_phoff = struct.unpack_from(' {out}") + print("link against it: ld.lld -o prog start.o prog.o -L. -l:" + + os.path.basename(out)) + + +if __name__ == '__main__': + main() diff --git a/remote/test.py b/remote/test.py new file mode 100644 index 0000000..4bb86e3 --- /dev/null +++ b/remote/test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Native test for fls-wh: build with the system compiler, run it against a +temp file (standing in for a block device), feed a framed command script, and +assert the framed records + the file bytes. + +Runs on Linux and macOS: `SYNC` uses `fdatasync` where available and falls +back to `fsync` on macOS (which has no `fdatasync`). +""" +import os +import shutil +import struct +import subprocess +import sys +import zlib + +MAGIC = b"FLSW" +OP_DATA, OP_SKIP, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ = 1, 2, 3, 4, 5, 6 +R_READY, R_ACK, R_PROG, R_ERR, R_DONE, R_READ_DATA = 0x81, 0x82, 0x83, 0x84, 0x85, 0x86 + + +def frame(op, payload): + return MAGIC + bytes([op]) + struct.pack(" Date: Fri, 9 Oct 2026 10:44:03 +0200 Subject: [PATCH 02/10] fix: fls-wh protocol robustness and bump write chunks to 8 MiB - drain remaining DATA payload after device write error (stream desync) - emit() uses write_all() to handle partial stdout writes over SSH - guard against malformed DATA frames with plen < 8 - read_all() returns ssize_t instead of int - bump disk write chunks from 64 KiB to 8 MiB (WRITE_CHUNK_SIZE, compile-time) - test.py uses tempfile instead of hardcoded /tmp paths Assisted-by: OpenCode Co-Authored-By: Claude Opus 4.6 --- remote/fls-wh.c | 51 ++++++++++++++++++++++++++++--------------------- remote/test.py | 6 ++++-- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 200a7b5..1ad6364 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -65,7 +65,10 @@ enum { ERR_DEV = 0, ERR_CRC = 1, ERR_NI = 2 }; /* NI = not implemented */ static const unsigned char MAGIC[4] = { 0x46, 0x4c, 0x53, 0x57 }; /* "FLSW" */ static const unsigned char PROTO_VER = 1; -static const size_t PROG_STEP = 1u << 20; /* emit PROG every 1 MiB */ +#ifndef WRITE_CHUNK_SIZE +#define WRITE_CHUNK_SIZE (8u << 20) /* 8 MiB disk write chunks (-DWRITE_CHUNK_SIZE=N to override) */ +#endif +static const size_t PROG_STEP = WRITE_CHUNK_SIZE; /* emit PROG every chunk */ #define MAX_READ (1u << 20) /* cap a single READ at 1 MiB */ static unsigned char dbuf[MAX_READ]; /* device read buffer */ @@ -101,6 +104,18 @@ static void crc_update(const unsigned char *b, size_t n) } static unsigned int crc_final(void) { return crc_state ^ 0xFFFFFFFFu; } +/* write all, retrying on partial writes; -1 on error (errno set) */ +static int write_all(int fd, const unsigned char *buf, size_t len) +{ + size_t off = 0; + while (off < len) { + ssize_t n = write(fd, buf + off, len - off); + if (n < 0) return -1; + off += (size_t)n; + } + return 0; +} + /* --- framed output (stdout) --- */ static void emit(int op, const unsigned char *payload, size_t len) { @@ -109,8 +124,8 @@ static void emit(int op, const unsigned char *payload, size_t len) hdr[2] = MAGIC[2]; hdr[3] = MAGIC[3]; hdr[4] = (unsigned char)op; wr_u32(hdr + 5, (unsigned int)len); - (void)write(1, hdr, 9); - if (len) (void)write(1, payload, len); + (void)write_all(1, hdr, 9); + if (len) (void)write_all(1, payload, len); } static void ok_rec(int op, unsigned long long off, unsigned long long bytes) { @@ -178,30 +193,18 @@ static int skip(unsigned long long n) return 1; } -/* write all, retrying on partial writes; -1 on error (errno set) */ -static int write_all(int fd, const unsigned char *buf, size_t len) -{ - size_t off = 0; - while (off < len) { - ssize_t n = write(fd, buf + off, len - off); - if (n < 0) return -1; - off += (size_t)n; - } - return 0; -} - /* read all, retrying on partial reads; -1 on error (errno set), else the bytes * read (which may be < len near end of device) */ -static int read_all(int fd, unsigned char *buf, size_t len) +static ssize_t read_all(int fd, unsigned char *buf, size_t len) { size_t off = 0; while (off < len) { ssize_t n = read(fd, buf + off, len - off); if (n < 0) return -1; - if (n == 0) return (int)off; /* EOF: partial read */ + if (n == 0) return (ssize_t)off; /* EOF: partial read */ off += (size_t)n; } - return (int)len; + return (ssize_t)len; } int main(int argc, char **argv) @@ -232,11 +235,12 @@ int main(int argc, char **argv) break; else if (op == OP_DATA) { + if (plen < 8) { skip(plen); err_rec(ERR_NI, OP_DATA, 0, cur); continue; } unsigned char szf[4]; if (!rreadn(szf, 4)) break; unsigned int size = rd_u32(szf); if (plen != 8 + (unsigned long long)size) { /* malformed frame */ - if (plen >= 4) skip(plen - 4); + skip(plen - 4); err_rec(ERR_NI, OP_DATA, 0, cur); continue; } @@ -244,7 +248,7 @@ int main(int argc, char **argv) crc_reset(); unsigned long long done = 0, next_prog = PROG_STEP; int status = 0; /* 0=ok, 1=eof, 2=write error */ - unsigned char chunk[1 << 16]; + static unsigned char chunk[WRITE_CHUNK_SIZE]; while (done < size) { size_t want = (size_t)(size - done); if (want > sizeof chunk) want = sizeof chunk; @@ -262,7 +266,10 @@ int main(int argc, char **argv) cur = start + done; total += done; if (status == 1) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } - if (status == 2) { err_rec(ERR_DEV, OP_DATA, (unsigned)errno, start); } + if (status == 2) { + skip((unsigned long long)(size - done) + 4); /* drain remaining content + CRC */ + err_rec(ERR_DEV, OP_DATA, (unsigned)errno, start); + } else if (done < size) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } else { unsigned char crcf[4]; @@ -300,7 +307,7 @@ int main(int argc, char **argv) if (!rreadn(lf, 8)) break; unsigned long long len = rd_u64(lf); if (len > MAX_READ) { err_rec(ERR_DEV, OP_READ, 22, cur); continue; } /* 22 = EINVAL */ - int n = read_all(fd, dbuf, (size_t)len); + ssize_t n = read_all(fd, dbuf, (size_t)len); if (n < 0) { err_rec(ERR_DEV, OP_READ, (unsigned)errno, cur); continue; } emit(R_READ_DATA, dbuf, (size_t)n); /* the bytes read (may be empty) */ cur += (unsigned long long)n; diff --git a/remote/test.py b/remote/test.py index 4bb86e3..dcce5b5 100644 --- a/remote/test.py +++ b/remote/test.py @@ -11,6 +11,7 @@ import struct import subprocess import sys +import tempfile import zlib MAGIC = b"FLSW" @@ -54,8 +55,9 @@ def find_cc(): def main(): here = os.path.dirname(os.path.abspath(__file__)) src = os.path.join(here, "fls-wh.c") - binp = "/tmp/fls-wh-native" - blk = "/tmp/fls-wh-test-blk" + tmpdir = tempfile.mkdtemp(prefix="fls-wh-test-") + binp = os.path.join(tmpdir, "fls-wh-native") + blk = os.path.join(tmpdir, "blk") # 1. build with the system compiler (system headers, not include/) subprocess.run([find_cc(), "-O2", src, "-o", binp], check=True) From 1c0651d31d589177ca8bbe599106ac10d0ca0794 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 9 Oct 2026 10:48:39 +0200 Subject: [PATCH 03/10] fix: DATA error drain over-consumed the failed chunk The failed chunk is already read from the stream when the write fails, so drain (size - rd) + 4 based on bytes read, not bytes written. Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- remote/fls-wh.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 1ad6364..cc06c50 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -246,13 +246,14 @@ int main(int argc, char **argv) } unsigned long long start = cur; crc_reset(); - unsigned long long done = 0, next_prog = PROG_STEP; + unsigned long long done = 0, rd = 0, next_prog = PROG_STEP; /* rd = bytes read from the stream */ int status = 0; /* 0=ok, 1=eof, 2=write error */ static unsigned char chunk[WRITE_CHUNK_SIZE]; while (done < size) { size_t want = (size_t)(size - done); if (want > sizeof chunk) want = sizeof chunk; if (!rreadn(chunk, want)) { status = 1; break; } + rd += want; crc_update(chunk, want); if (write_all(fd, chunk, want) < 0) { status = 2; break; } done += want; @@ -267,7 +268,7 @@ int main(int argc, char **argv) total += done; if (status == 1) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } if (status == 2) { - skip((unsigned long long)(size - done) + 4); /* drain remaining content + CRC */ + skip((unsigned long long)(size - rd) + 4); /* drain unread content + CRC (the failed chunk is already consumed) */ err_rec(ERR_DEV, OP_DATA, (unsigned)errno, start); } else if (done < size) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } From d40f6199ba809c75ccb7149bfebd407eba1b0a51 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Fri, 9 Oct 2026 11:18:09 +0200 Subject: [PATCH 04/10] Add ZERO command; CRC from gityf/crc ZERO (0x09, len:u64) writes zeros from the current offset without sending bytes over the wire. CRC32 base table and per-byte update copied from gityf/crc (Wang Yaofu, Apache-2.0), adapted to a running state; verified against zlib in test.py. Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- remote/README.md | 6 ++- remote/fls-wh.c | 102 ++++++++++++++++++++++++++++++++++++++++------- remote/test.py | 15 ++++--- 3 files changed, 102 insertions(+), 21 deletions(-) diff --git a/remote/README.md b/remote/README.md index b73faef..bb232f4 100644 --- a/remote/README.md +++ b/remote/README.md @@ -28,13 +28,15 @@ scans for it to resync after any corruption. | | opcode | | opcode | |---|---|---|---| -| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` | +| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` `ZERO 0x09` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` | | **Reserved** | `AUTH 0x07` `HELLO 0x08` | | `AUTH_RESULT 0x87` `HELLO 0x88` | The `READY` payload carries a protocol version byte (currently 1) for forward compatibility. `DATA` is `size:u32 + content + crc:u32` (CRC32-IEEE); `READ` is `len:u64` (reads from the current offset, advances it, capped at -1 MiB) and returns `READ_DATA` with the bytes. +1 MiB) and returns `READ_DATA` with the bytes. `ZERO` is `len:u64` — writes +that many zeros from the current offset and advances it, sending only the 8 +length bytes over the wire (for filling regions without transferring bytes). ## Build diff --git a/remote/fls-wh.c b/remote/fls-wh.c index cc06c50..380b22b 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -27,7 +27,7 @@ * it to resync after any corruption. * * Requests: DATA(0x01) SKIP(0x02) SEEK(0x03) SYNC(0x04) QUIT(0x05) - * READ(0x06) [implemented] + * READ(0x06) ZERO(0x09) [implemented] * AUTH(0x07) HELLO(0x08) [reserved, spec only] * Responses: READY(0x81) OK(0x82) PROG(0x83) ERR(0x84) DONE(0x85) * READ_DATA(0x86) [implemented] @@ -57,7 +57,7 @@ enum { OP_DATA = 0x01, OP_SKIP = 0x02, OP_SEEK = 0x03, OP_SYNC = 0x04, OP_QUIT = 0x05, - OP_READ = 0x06, OP_AUTH = 0x07, OP_HELLO = 0x08, /* reserved */ + OP_READ = 0x06, OP_AUTH = 0x07, OP_HELLO = 0x08, OP_ZERO = 0x09, R_READY = 0x81, R_ACK = 0x82, R_PROG = 0x83, R_ERR = 0x84, R_DONE = 0x85, R_READ_DATA = 0x86, R_AUTH_RESULT = 0x87, R_HELLO = 0x88 /* reserved */ }; @@ -82,24 +82,72 @@ static void wr_u32(unsigned char *p, unsigned int v) static void wr_u64(unsigned char *p, unsigned long long v) { wr_u32(p, (unsigned int)v); wr_u32(p + 4, (unsigned int)(v >> 32)); } -/* --- CRC32 (IEEE), running state so it can be computed while streaming --- */ -static unsigned int crc_tab[256]; +/* --- CRC32 (IEEE), running state so it can be computed while streaming. + * Base table and per-byte update from gityf/crc (Wang Yaofu, + * Apache License 2.0, https://github.com/gityf/crc), adapted to a + * running state. */ +static const unsigned int crc_tab[256] = { + 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, + 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, + 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, + 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, + 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, + 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, + 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, + 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, + 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, + 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, + 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, + 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, + 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, + 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, + 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, + 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, + 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, + 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, + 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, + 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, + 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, + 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, + 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, + 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, + 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, + 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, + 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, + 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, + 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, + 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, + 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, + 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, + 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, + 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, + 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, + 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, + 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, + 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, + 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, + 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, + 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, + 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, + 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, + 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, + 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, + 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, + 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, + 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, + 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, + 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, + 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, + 0x2d02ef8dL +}; static unsigned int crc_state; -static void crc_init(void) -{ - for (unsigned int i = 0; i < 256; i++) { - unsigned int c = i; - for (int k = 0; k < 8; k++) - c = (c & 1) ? 0xEDB88320u ^ (c >> 1) : c >> 1; - crc_tab[i] = c; - } -} +static void crc_init(void) { } /* table is static */ static void crc_reset(void) { crc_state = 0xFFFFFFFFu; } static void crc_update(const unsigned char *b, size_t n) { unsigned int c = crc_state; for (size_t i = 0; i < n; i++) - c = crc_tab[(c ^ b[i]) & 0xFF] ^ (c >> 8); + c = crc_tab[(c ^ b[i]) & 0xFF] ^ ((c >> 8) & 0x00FFFFFF); crc_state = c; } static unsigned int crc_final(void) { return crc_state ^ 0xFFFFFFFFu; } @@ -292,6 +340,32 @@ int main(int argc, char **argv) else { ok_rec(OP_SKIP, cur, len); cur += len; } } + else if (op == OP_ZERO) { + if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_ZERO, 0, cur); continue; } + unsigned char lf[8]; + if (!rreadn(lf, 8)) break; + unsigned long long len = rd_u64(lf); + static unsigned char zbuf[WRITE_CHUNK_SIZE]; /* zero-initialized */ + unsigned long long start = cur, done = 0, next_prog = PROG_STEP; + int status = 0; + while (done < len) { + size_t want = (size_t)(len - done); + if (want > sizeof zbuf) want = sizeof zbuf; + if (write_all(fd, zbuf, want) < 0) { status = errno; break; } + done += want; + while (done >= next_prog) { + unsigned char pp[16]; + wr_u64(pp, start + done); wr_u64(pp + 8, done); + emit(R_PROG, pp, sizeof pp); + next_prog += PROG_STEP; + } + } + cur = start + done; + total += done; + if (status) err_rec(ERR_DEV, OP_ZERO, (unsigned)status, start); /* no drain: nothing left in the stream */ + else ok_rec(OP_ZERO, start, done); + } + else if (op == OP_SEEK) { if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_SEEK, 0, cur); continue; } unsigned char of[8]; diff --git a/remote/test.py b/remote/test.py index dcce5b5..9dba803 100644 --- a/remote/test.py +++ b/remote/test.py @@ -15,7 +15,7 @@ import zlib MAGIC = b"FLSW" -OP_DATA, OP_SKIP, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ = 1, 2, 3, 4, 5, 6 +OP_DATA, OP_SKIP, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ, OP_ZERO = 1, 2, 3, 4, 5, 6, 9 R_READY, R_ACK, R_PROG, R_ERR, R_DONE, R_READ_DATA = 0x81, 0x82, 0x83, 0x84, 0x85, 0x86 @@ -67,7 +67,7 @@ def main(): f.truncate(1 << 20) # 3. build the command script: SEEK 0, DATA 16, SEEK 0, READ 16, SKIP 8, - # SYNC, QUIT + # ZERO 256, SYNC, QUIT content = bytes(range(16)) crc = zlib.crc32(content) & 0xFFFFFFFF script = b"" @@ -77,6 +77,7 @@ def main(): script += frame(OP_SEEK, struct.pack(" Date: Fri, 9 Oct 2026 12:36:03 +0200 Subject: [PATCH 05/10] Fix remote I/O recovery and SYNC errors Restore offsets after partial I/O, preserve errno, and reject zero-progress writes. Cover failure paths and restrict CI credentials. Assisted-by: OpenCode Co-Authored-By: GPT-6.1 Sol --- .github/workflows/ci.yml | 5 ++++ remote/fls-wh.c | 37 +++++++++++++++++++---- remote/include/errno.h | 2 ++ remote/test.py | 63 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf20933..dbab6d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,9 @@ on: env: CARGO_TERM_COLOR: always +permissions: + contents: read + jobs: # runs first: builds the write-head binaries and leaves them as artifacts # for later fls jobs (add `needs: write-head` + actions/download-artifact @@ -18,6 +21,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + persist-credentials: false # build the QNX cross binary in the fedora container (remote/ has its # own Containerfile with the cross toolchain) diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 380b22b..6d97dd4 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -159,6 +159,7 @@ static int write_all(int fd, const unsigned char *buf, size_t len) while (off < len) { ssize_t n = write(fd, buf + off, len - off); if (n < 0) return -1; + if (n == 0) { errno = EIO; return -1; } off += (size_t)n; } return 0; @@ -316,8 +317,16 @@ int main(int argc, char **argv) total += done; if (status == 1) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } if (status == 2) { - skip((unsigned long long)(size - rd) + 4); /* drain unread content + CRC (the failed chunk is already consumed) */ - err_rec(ERR_DEV, OP_DATA, (unsigned)errno, start); + int e = errno; + if (!skip((unsigned long long)(size - rd) + 4)) { /* unread content + CRC; failed chunk already consumed */ + err_rec(ERR_DEV, OP_DATA, (unsigned)e, start); + break; + } + if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { + err_rec(ERR_DEV, OP_DATA, (unsigned)errno, cur); + break; + } + err_rec(ERR_DEV, OP_DATA, (unsigned)e, start); } else if (done < size) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } else { @@ -362,7 +371,13 @@ int main(int argc, char **argv) } cur = start + done; total += done; - if (status) err_rec(ERR_DEV, OP_ZERO, (unsigned)status, start); /* no drain: nothing left in the stream */ + if (status) { + if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { + err_rec(ERR_DEV, OP_ZERO, (unsigned)errno, cur); + break; + } + err_rec(ERR_DEV, OP_ZERO, (unsigned)status, start); /* no drain: nothing left in the stream */ + } else ok_rec(OP_ZERO, start, done); } @@ -383,15 +398,25 @@ int main(int argc, char **argv) unsigned long long len = rd_u64(lf); if (len > MAX_READ) { err_rec(ERR_DEV, OP_READ, 22, cur); continue; } /* 22 = EINVAL */ ssize_t n = read_all(fd, dbuf, (size_t)len); - if (n < 0) { err_rec(ERR_DEV, OP_READ, (unsigned)errno, cur); continue; } + if (n < 0) { + int e = errno; + if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { + err_rec(ERR_DEV, OP_READ, (unsigned)errno, cur); + break; + } + err_rec(ERR_DEV, OP_READ, (unsigned)e, cur); + continue; + } emit(R_READ_DATA, dbuf, (size_t)n); /* the bytes read (may be empty) */ cur += (unsigned long long)n; } else if (op == OP_SYNC) { if (plen != 0) { skip(plen); err_rec(ERR_NI, OP_SYNC, 0, cur); continue; } - (void)sync_file(fd); - ok_rec(OP_SYNC, cur, 0); + if (sync_file(fd) < 0) + err_rec(ERR_DEV, OP_SYNC, (unsigned)errno, cur); /* durability not achieved; don't claim it */ + else + ok_rec(OP_SYNC, cur, 0); } else { /* reserved (AUTH/HELLO) or unknown opcode */ diff --git a/remote/include/errno.h b/remote/include/errno.h index 45d47ec..6a9c91f 100644 --- a/remote/include/errno.h +++ b/remote/include/errno.h @@ -3,4 +3,6 @@ /* QNX: errno is a per-thread location reached via __get_errno_ptr(). */ int *__get_errno_ptr(void); #define errno (*__get_errno_ptr()) +/* only the constants the code assigns */ +#define EIO 5 #endif diff --git a/remote/test.py b/remote/test.py index 9dba803..fae5a25 100644 --- a/remote/test.py +++ b/remote/test.py @@ -6,6 +6,7 @@ Runs on Linux and macOS: `SYNC` uses `fdatasync` where available and falls back to `fsync` on macOS (which has no `fdatasync`). """ +import errno import os import shutil import struct @@ -120,6 +121,68 @@ def main(): assert head == content, f"file bytes mismatch: {head.hex()} != {content.hex()}" assert zeros == b"\0" * 256, f"ZERO region not zeroed: {zeros[:16].hex()}..." + # Inject short I/O followed by EIO, zero-progress writes, and SYNC failure. + # Successful lseek deliberately changes errno to catch lost error codes. + fault_src = os.path.join(tmpdir, "faults.c") + fault_bin = os.path.join(tmpdir, "fls-wh-faults") + with open(fault_src, "w") as f: + f.write('''#include +#include +static ssize_t fault_write(int fd, const void *buf, size_t n) { + if (fd > 2) { + if (n == 32) return 0; + if (n == 8) { errno = EIO; return -1; } + if (n == 16) n = 8; + } + return write(fd, buf, n); +} +static ssize_t fault_read(int fd, void *buf, size_t n) { + if (fd > 2) { + if (n == 8) { errno = EIO; return -1; } + if (n == 16) n = 8; + } + return read(fd, buf, n); +} +static off_t fault_seek(int fd, off_t off, int whence) { + off_t result = lseek(fd, off, whence); + if (result >= 0) errno = EINVAL; + return result; +} +static int fault_sync(int fd) { (void)fd; errno = EIO; return -1; } +#define write fault_write +#define read fault_read +#define lseek fault_seek +#define fsync fault_sync +#define fdatasync fault_sync +#include "fls-wh.c" +''') + subprocess.run([find_cc(), "-O2", "-I", here, fault_src, "-o", fault_bin], check=True) + + def data(content): + return frame(OP_DATA, struct.pack(" Date: Fri, 9 Oct 2026 12:53:29 +0200 Subject: [PATCH 06/10] Add remote SIZE query Return total device bytes without moving the cursor. Document image-size preflight and test query/error paths. Assisted-by: OpenCode Co-Authored-By: GPT-6.1 Sol --- remote/README.md | 15 ++++++++++++--- remote/fls-wh.c | 20 ++++++++++++++++---- remote/test.py | 27 +++++++++++++++++++++++---- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/remote/README.md b/remote/README.md index bb232f4..499d086 100644 --- a/remote/README.md +++ b/remote/README.md @@ -28,16 +28,25 @@ scans for it to resync after any corruption. | | opcode | | opcode | |---|---|---|---| -| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` `ZERO 0x09` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` | +| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` `ZERO 0x09` `SIZE 0x0a` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` `SIZE 0x89` | | **Reserved** | `AUTH 0x07` `HELLO 0x08` | | `AUTH_RESULT 0x87` `HELLO 0x88` | -The `READY` payload carries a protocol version byte (currently 1) for -forward compatibility. `DATA` is `size:u32 + content + crc:u32` (CRC32-IEEE); +The `READY` payload is `version:u8 + size:u64` (total device bytes, measured +at open); the protocol version is currently 1. `DATA` is +`size:u32 + content + crc:u32` (CRC32-IEEE); `READ` is `len:u64` (reads from the current offset, advances it, capped at 1 MiB) and returns `READ_DATA` with the bytes. `ZERO` is `len:u64` — writes that many zeros from the current offset and advances it, sending only the 8 length bytes over the wire (for filling regions without transferring bytes). +`SIZE` has no request payload and returns `SIZE 0x89` with `size:u64` (the +same total device bytes measured at open), without changing the cursor. If +the size could not be determined, it returns `ERR` with the original errno; +the startup `READY` uses zero for unknown size. +Before writing an image, the client can use `READY` or `SIZE` to check +`image_size <= device_size - write_offset` after checking that +`write_offset <= device_size`. For sparse images, compare the expanded size. + ## Build Binaries are named `flswh--` (matching the `fls` artifact scheme): diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 6d97dd4..7ca59c0 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -27,20 +27,21 @@ * it to resync after any corruption. * * Requests: DATA(0x01) SKIP(0x02) SEEK(0x03) SYNC(0x04) QUIT(0x05) - * READ(0x06) ZERO(0x09) [implemented] + * READ(0x06) ZERO(0x09) SIZE(0x0a) [implemented] * AUTH(0x07) HELLO(0x08) [reserved, spec only] * Responses: READY(0x81) OK(0x82) PROG(0x83) ERR(0x84) DONE(0x85) - * READ_DATA(0x86) [implemented] + * READ_DATA(0x86) SIZE(0x89) [implemented] * AUTH_RESULT(0x87) HELLO(0x88) [reserved] * * DATA payload: size:u32 + content[size] + crc:u32 (CRC32-IEEE of content) * SKIP payload: len:u64 * SEEK payload: offset:u64 - * SYNC/QUIT: (no payload) + * SYNC/QUIT/SIZE: (no payload) * READ payload: len:u64 (read len bytes from the current offset, advance it; * len must be <= 1 MiB) * * READY payload: version:u8 + size:u64 + * SIZE payload: size:u64 (total bytes measured at open; cursor unchanged) * OK payload: op:1 + offset:u64 + bytes:u64 * PROG payload: offset:u64 + bytes_done:u64 (every 1 MiB of a DATA) * ERR payload: code:1 + op:1 + errno:u32 + offset:u64 @@ -58,8 +59,10 @@ enum { OP_DATA = 0x01, OP_SKIP = 0x02, OP_SEEK = 0x03, OP_SYNC = 0x04, OP_QUIT = 0x05, OP_READ = 0x06, OP_AUTH = 0x07, OP_HELLO = 0x08, OP_ZERO = 0x09, + OP_SIZE = 0x0a, R_READY = 0x81, R_ACK = 0x82, R_PROG = 0x83, R_ERR = 0x84, R_DONE = 0x85, - R_READ_DATA = 0x86, R_AUTH_RESULT = 0x87, R_HELLO = 0x88 /* reserved */ + R_READ_DATA = 0x86, R_SIZE = 0x89, + R_AUTH_RESULT = 0x87, R_HELLO = 0x88 /* reserved */ }; enum { ERR_DEV = 0, ERR_CRC = 1, ERR_NI = 2 }; /* NI = not implemented */ @@ -263,6 +266,7 @@ int main(int argc, char **argv) if (fd < 0) { err_rec(ERR_DEV, 0, (unsigned)errno, 0); return 1; } long long sz = lseek(fd, 0, SEEK_END); + int size_errno = sz < 0 ? errno : 0; lseek(fd, 0, SEEK_SET); /* the size query left the offset at the end */ unsigned char rp[9]; rp[0] = PROTO_VER; wr_u64(rp + 1, (unsigned long long)(sz < 0 ? 0 : sz)); @@ -283,6 +287,14 @@ int main(int argc, char **argv) if (op == OP_QUIT) break; + else if (op == OP_SIZE) { + if (plen != 0) { skip(plen); err_rec(ERR_NI, OP_SIZE, 0, cur); continue; } + if (sz < 0) { err_rec(ERR_DEV, OP_SIZE, (unsigned)size_errno, cur); continue; } + unsigned char p[8]; + wr_u64(p, (unsigned long long)sz); + emit(R_SIZE, p, sizeof p); + } + else if (op == OP_DATA) { if (plen < 8) { skip(plen); err_rec(ERR_NI, OP_DATA, 0, cur); continue; } unsigned char szf[4]; diff --git a/remote/test.py b/remote/test.py index fae5a25..a8def22 100644 --- a/remote/test.py +++ b/remote/test.py @@ -18,6 +18,7 @@ MAGIC = b"FLSW" OP_DATA, OP_SKIP, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ, OP_ZERO = 1, 2, 3, 4, 5, 6, 9 R_READY, R_ACK, R_PROG, R_ERR, R_DONE, R_READ_DATA = 0x81, 0x82, 0x83, 0x84, 0x85, 0x86 +OP_SIZE, R_SIZE = 0x0A, 0x89 def frame(op, payload): @@ -67,8 +68,7 @@ def main(): with open(blk, "wb") as f: f.truncate(1 << 20) - # 3. build the command script: SEEK 0, DATA 16, SEEK 0, READ 16, SKIP 8, - # ZERO 256, SYNC, QUIT + # SIZE between SKIP and ZERO must leave the logical and device offsets intact. content = bytes(range(16)) crc = zlib.crc32(content) & 0xFFFFFFFF script = b"" @@ -78,7 +78,9 @@ def main(): script += frame(OP_SEEK, struct.pack("= 0) errno = EINVAL; return result; @@ -164,7 +170,7 @@ def data(content): seek0 = frame(OP_SEEK, struct.pack(" Date: Tue, 13 Oct 2026 12:02:18 +0200 Subject: [PATCH 07/10] Drop the unused SKIP opcode SKIP is not used by the fls client and is covered by SEEK (seek to cur + len). Remove it from the protocol, the write head, and the native test (the offset-advance coverage is kept via SEEK). Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- remote/README.md | 2 +- remote/fls-wh.c | 17 +++-------------- remote/test.py | 10 +++++----- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/remote/README.md b/remote/README.md index 499d086..6ab1d81 100644 --- a/remote/README.md +++ b/remote/README.md @@ -28,7 +28,7 @@ scans for it to resync after any corruption. | | opcode | | opcode | |---|---|---|---| -| **Requests** | `DATA 0x01` `SKIP 0x02` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` `ZERO 0x09` `SIZE 0x0a` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` `SIZE 0x89` | +| **Requests** | `DATA 0x01` `SEEK 0x03` `SYNC 0x04` `QUIT 0x05` `READ 0x06` `ZERO 0x09` `SIZE 0x0a` | **Responses** | `READY 0x81` `OK 0x82` `PROG 0x83` `ERR 0x84` `DONE 0x85` `READ_DATA 0x86` `SIZE 0x89` | | **Reserved** | `AUTH 0x07` `HELLO 0x08` | | `AUTH_RESULT 0x87` `HELLO 0x88` | The `READY` payload is `version:u8 + size:u64` (total device bytes, measured diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 7ca59c0..e125d44 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -26,15 +26,14 @@ * Little-endian throughout. The magic is a sync header: the reader scans for * it to resync after any corruption. * - * Requests: DATA(0x01) SKIP(0x02) SEEK(0x03) SYNC(0x04) QUIT(0x05) + * Requests: DATA(0x01) SEEK(0x03) SYNC(0x04) QUIT(0x05) * READ(0x06) ZERO(0x09) SIZE(0x0a) [implemented] - * AUTH(0x07) HELLO(0x08) [reserved, spec only] + * 0x02 AUTH(0x07) HELLO(0x08) [reserved, spec only] * Responses: READY(0x81) OK(0x82) PROG(0x83) ERR(0x84) DONE(0x85) * READ_DATA(0x86) SIZE(0x89) [implemented] * AUTH_RESULT(0x87) HELLO(0x88) [reserved] * * DATA payload: size:u32 + content[size] + crc:u32 (CRC32-IEEE of content) - * SKIP payload: len:u64 * SEEK payload: offset:u64 * SYNC/QUIT/SIZE: (no payload) * READ payload: len:u64 (read len bytes from the current offset, advance it; @@ -56,7 +55,7 @@ */ enum { - OP_DATA = 0x01, OP_SKIP = 0x02, OP_SEEK = 0x03, OP_SYNC = 0x04, + OP_DATA = 0x01, OP_SEEK = 0x03, OP_SYNC = 0x04, OP_QUIT = 0x05, OP_READ = 0x06, OP_AUTH = 0x07, OP_HELLO = 0x08, OP_ZERO = 0x09, OP_SIZE = 0x0a, @@ -351,16 +350,6 @@ int main(int argc, char **argv) } } - else if (op == OP_SKIP) { - if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_SKIP, 0, cur); continue; } - unsigned char lf[8]; - if (!rreadn(lf, 8)) break; - unsigned long long len = rd_u64(lf); - if (lseek(fd, cur + len, SEEK_SET) < 0) - err_rec(ERR_DEV, OP_SKIP, (unsigned)errno, cur); - else { ok_rec(OP_SKIP, cur, len); cur += len; } - } - else if (op == OP_ZERO) { if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_ZERO, 0, cur); continue; } unsigned char lf[8]; diff --git a/remote/test.py b/remote/test.py index a8def22..8cbc111 100644 --- a/remote/test.py +++ b/remote/test.py @@ -16,7 +16,7 @@ import zlib MAGIC = b"FLSW" -OP_DATA, OP_SKIP, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ, OP_ZERO = 1, 2, 3, 4, 5, 6, 9 +OP_DATA, OP_SEEK, OP_SYNC, OP_QUIT, OP_READ, OP_ZERO = 1, 3, 4, 5, 6, 9 R_READY, R_ACK, R_PROG, R_ERR, R_DONE, R_READ_DATA = 0x81, 0x82, 0x83, 0x84, 0x85, 0x86 OP_SIZE, R_SIZE = 0x0A, 0x89 @@ -68,7 +68,7 @@ def main(): with open(blk, "wb") as f: f.truncate(1 << 20) - # SIZE between SKIP and ZERO must leave the logical and device offsets intact. + # SIZE between SEEK and ZERO must leave the logical and device offsets intact. content = bytes(range(16)) crc = zlib.crc32(content) & 0xFFFFFFFF script = b"" @@ -77,7 +77,7 @@ def main(): + struct.pack(" Date: Tue, 13 Oct 2026 12:05:56 +0200 Subject: [PATCH 08/10] Exit on the first ERR instead of recovering The fls client treats every ERR as fatal, so the write head's post-error recovery (draining the frame, re-seeking the fd back to the logical cursor) is dead weight. Report the error and exit: - DATA: drop the drain and re-seek after a failed write - ZERO / READ: drop the re-seek - malformed frames: drop the drain - skip() has no callers left; delete it The native fault-injection test now runs each fault in a fresh process and checks the exact READY/ERR/DONE sequence. Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- remote/README.md | 3 +- remote/fls-wh.c | 92 +++++++++++++++++------------------------------- remote/test.py | 60 ++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 81 deletions(-) diff --git a/remote/README.md b/remote/README.md index 6ab1d81..86aab2b 100644 --- a/remote/README.md +++ b/remote/README.md @@ -18,7 +18,8 @@ Opens `` (`O_RDWR`), reports its size (`READY`), then reads framed commands from stdin and writes them to the device, emitting framed records to stdout. Intended to be driven by a remote client (e.g. over SSH). QUIT or EOF exits. -Device I/O errors are reported (`ERR`) and processing continues. +Device I/O errors are reported (`ERR`), after which the write head exits; +the client treats any `ERR` as fatal. ## Protocol diff --git a/remote/fls-wh.c b/remote/fls-wh.c index e125d44..397597e 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -232,17 +232,6 @@ static int read_magic(void) w[0] = w[1]; w[1] = w[2]; w[2] = b; } } -/* skip n bytes from the stream (drop a malformed frame's remainder); 0 on EOF */ -static int skip(unsigned long long n) -{ - unsigned char b[1 << 16]; - while (n > 0) { - size_t take = (n > sizeof b) ? sizeof b : (size_t)n; - if (!rreadn(b, take)) return 0; - n -= take; - } - return 1; -} /* read all, retrying on partial reads; -1 on error (errno set), else the bytes * read (which may be < len near end of device) */ @@ -287,33 +276,31 @@ int main(int argc, char **argv) break; else if (op == OP_SIZE) { - if (plen != 0) { skip(plen); err_rec(ERR_NI, OP_SIZE, 0, cur); continue; } - if (sz < 0) { err_rec(ERR_DEV, OP_SIZE, (unsigned)size_errno, cur); continue; } + if (plen != 0) { err_rec(ERR_NI, OP_SIZE, 0, cur); break; } + if (sz < 0) { err_rec(ERR_DEV, OP_SIZE, (unsigned)size_errno, cur); break; } unsigned char p[8]; wr_u64(p, (unsigned long long)sz); emit(R_SIZE, p, sizeof p); } else if (op == OP_DATA) { - if (plen < 8) { skip(plen); err_rec(ERR_NI, OP_DATA, 0, cur); continue; } + if (plen < 8) { err_rec(ERR_NI, OP_DATA, 0, cur); break; } unsigned char szf[4]; if (!rreadn(szf, 4)) break; unsigned int size = rd_u32(szf); if (plen != 8 + (unsigned long long)size) { /* malformed frame */ - skip(plen - 4); err_rec(ERR_NI, OP_DATA, 0, cur); - continue; + break; } unsigned long long start = cur; crc_reset(); - unsigned long long done = 0, rd = 0, next_prog = PROG_STEP; /* rd = bytes read from the stream */ + unsigned long long done = 0, next_prog = PROG_STEP; int status = 0; /* 0=ok, 1=eof, 2=write error */ static unsigned char chunk[WRITE_CHUNK_SIZE]; while (done < size) { size_t want = (size_t)(size - done); if (want > sizeof chunk) want = sizeof chunk; if (!rreadn(chunk, want)) { status = 1; break; } - rd += want; crc_update(chunk, want); if (write_all(fd, chunk, want) < 0) { status = 2; break; } done += want; @@ -326,32 +313,22 @@ int main(int argc, char **argv) } cur = start + done; total += done; - if (status == 1) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } - if (status == 2) { - int e = errno; - if (!skip((unsigned long long)(size - rd) + 4)) { /* unread content + CRC; failed chunk already consumed */ - err_rec(ERR_DEV, OP_DATA, (unsigned)e, start); - break; - } - if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { - err_rec(ERR_DEV, OP_DATA, (unsigned)errno, cur); - break; - } + if (status) { + int e = status == 2 ? errno : 0; err_rec(ERR_DEV, OP_DATA, (unsigned)e, start); + break; } - else if (done < size) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } - else { - unsigned char crcf[4]; - if (!rreadn(crcf, 4)) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } - if (crc_final() != rd_u32(crcf)) - err_rec(ERR_CRC, OP_DATA, 0, start); - else - ok_rec(OP_DATA, start, done); + unsigned char crcf[4]; + if (!rreadn(crcf, 4)) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + if (crc_final() != rd_u32(crcf)) { + err_rec(ERR_CRC, OP_DATA, 0, start); + break; } + ok_rec(OP_DATA, start, done); } else if (op == OP_ZERO) { - if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_ZERO, 0, cur); continue; } + if (plen != 8) { err_rec(ERR_NI, OP_ZERO, 0, cur); break; } unsigned char lf[8]; if (!rreadn(lf, 8)) break; unsigned long long len = rd_u64(lf); @@ -373,56 +350,53 @@ int main(int argc, char **argv) cur = start + done; total += done; if (status) { - if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { - err_rec(ERR_DEV, OP_ZERO, (unsigned)errno, cur); - break; - } - err_rec(ERR_DEV, OP_ZERO, (unsigned)status, start); /* no drain: nothing left in the stream */ + err_rec(ERR_DEV, OP_ZERO, (unsigned)status, start); + break; } - else ok_rec(OP_ZERO, start, done); + ok_rec(OP_ZERO, start, done); } else if (op == OP_SEEK) { - if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_SEEK, 0, cur); continue; } + if (plen != 8) { err_rec(ERR_NI, OP_SEEK, 0, cur); break; } unsigned char of[8]; if (!rreadn(of, 8)) break; unsigned long long off = rd_u64(of); - if (lseek(fd, off, SEEK_SET) < 0) + if (lseek(fd, off, SEEK_SET) < 0) { err_rec(ERR_DEV, OP_SEEK, (unsigned)errno, off); - else { ok_rec(OP_SEEK, off, 0); cur = off; } + break; + } + ok_rec(OP_SEEK, off, 0); + cur = off; } else if (op == OP_READ) { - if (plen != 8) { skip(plen); err_rec(ERR_NI, OP_READ, 0, cur); continue; } + if (plen != 8) { err_rec(ERR_NI, OP_READ, 0, cur); break; } unsigned char lf[8]; if (!rreadn(lf, 8)) break; unsigned long long len = rd_u64(lf); - if (len > MAX_READ) { err_rec(ERR_DEV, OP_READ, 22, cur); continue; } /* 22 = EINVAL */ + if (len > MAX_READ) { err_rec(ERR_DEV, OP_READ, 22, cur); break; } /* 22 = EINVAL */ ssize_t n = read_all(fd, dbuf, (size_t)len); if (n < 0) { int e = errno; - if (lseek(fd, (off_t)cur, SEEK_SET) < 0) { - err_rec(ERR_DEV, OP_READ, (unsigned)errno, cur); - break; - } err_rec(ERR_DEV, OP_READ, (unsigned)e, cur); - continue; + break; } emit(R_READ_DATA, dbuf, (size_t)n); /* the bytes read (may be empty) */ cur += (unsigned long long)n; } else if (op == OP_SYNC) { - if (plen != 0) { skip(plen); err_rec(ERR_NI, OP_SYNC, 0, cur); continue; } - if (sync_file(fd) < 0) + if (plen != 0) { err_rec(ERR_NI, OP_SYNC, 0, cur); break; } + if (sync_file(fd) < 0) { err_rec(ERR_DEV, OP_SYNC, (unsigned)errno, cur); /* durability not achieved; don't claim it */ - else - ok_rec(OP_SYNC, cur, 0); + break; + } + ok_rec(OP_SYNC, cur, 0); } else { /* reserved (AUTH/HELLO) or unknown opcode */ - skip(plen); err_rec(ERR_NI, op, 0, cur); + break; } } diff --git a/remote/test.py b/remote/test.py index 8cbc111..96d1044 100644 --- a/remote/test.py +++ b/remote/test.py @@ -169,27 +169,46 @@ def data(content): + struct.pack(" 8 partial write, then EIO on the retry. + fault_run(seek0 + data(bytes(range(16))), (0, OP_DATA, errno.EIO, 0), 0) + # DATA: a zero-progress write is a hard error. + fault_run(seek0 + data(bytes(32)), (0, OP_DATA, errno.EIO, 0), 0) + # READ: 16 -> 8 partial read, then EIO on the retry. + fault_run(seek0 + frame(OP_READ, struct.pack(" 8 partial write, then EIO on the retry. + fault_run(seek0 + frame(OP_ZERO, struct.pack(" Date: Tue, 13 Oct 2026 12:13:51 +0200 Subject: [PATCH 09/10] Validate DATA CRC before writing to the device Stage the complete payload in the existing 8 MiB buffer, validate the trailing CRC, and only then write to the device: a corrupt chunk no longer leaves bytes behind. Cap a single DATA at the staging buffer size (8 MiB, matching the client's cap) so the cap check is also a memory-exhaustion guard. Error-record behavior is unchanged: malformed frames, read failures, CRC mismatches, and write failures all report ERR and exit. The native test now covers a CRC mismatch (device left untouched) and an oversized DATA (rejected before staging). Assisted-by: OpenCode Co-Authored-By: Qwen 3.8 27B --- remote/README.md | 4 +++- remote/fls-wh.c | 44 +++++++++++++++++++++++++++----------------- remote/test.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/remote/README.md b/remote/README.md index 86aab2b..ef975cc 100644 --- a/remote/README.md +++ b/remote/README.md @@ -34,7 +34,9 @@ scans for it to resync after any corruption. The `READY` payload is `version:u8 + size:u64` (total device bytes, measured at open); the protocol version is currently 1. `DATA` is -`size:u32 + content + crc:u32` (CRC32-IEEE); +`size:u32 + content + crc:u32` (CRC32-IEEE, capped at 8 MiB); the payload is +staged and CRC-validated before the device is written, so a corrupt chunk +leaves no bytes behind. `READ` is `len:u64` (reads from the current offset, advances it, capped at 1 MiB) and returns `READ_DATA` with the bytes. `ZERO` is `len:u64` — writes that many zeros from the current offset and advances it, sending only the 8 diff --git a/remote/fls-wh.c b/remote/fls-wh.c index 397597e..cb98431 100644 --- a/remote/fls-wh.c +++ b/remote/fls-wh.c @@ -34,6 +34,8 @@ * AUTH_RESULT(0x87) HELLO(0x88) [reserved] * * DATA payload: size:u32 + content[size] + crc:u32 (CRC32-IEEE of content) + * (size must be <= 8 MiB; the payload is staged and CRC- + * validated before any device write) * SEEK payload: offset:u64 * SYNC/QUIT/SIZE: (no payload) * READ payload: len:u64 (read len bytes from the current offset, advance it; @@ -73,6 +75,7 @@ static const unsigned char PROTO_VER = 1; static const size_t PROG_STEP = WRITE_CHUNK_SIZE; /* emit PROG every chunk */ #define MAX_READ (1u << 20) /* cap a single READ at 1 MiB */ static unsigned char dbuf[MAX_READ]; /* device read buffer */ +#define MAX_DATA WRITE_CHUNK_SIZE /* cap a single DATA at the staging buffer size */ /* --- little-endian packing --- */ static unsigned int rd_u32(const unsigned char *p) @@ -292,17 +295,34 @@ int main(int argc, char **argv) err_rec(ERR_NI, OP_DATA, 0, cur); break; } + if (size > MAX_DATA) { err_rec(ERR_NI, OP_DATA, 0, cur); break; } + /* Stage the whole payload, validate the CRC, and only then touch + * the device: a corrupt chunk must not leave bytes behind. */ + static unsigned char buf[WRITE_CHUNK_SIZE]; unsigned long long start = cur; crc_reset(); + unsigned long long rd = 0; + int fail = 0; + while (rd < size) { + size_t want = (size_t)(size - rd); + if (want > sizeof buf) want = sizeof buf; + if (!rreadn(buf + rd, want)) { fail = 1; break; } + crc_update(buf + rd, want); + rd += want; + } + if (fail) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + unsigned char crcf[4]; + if (!rreadn(crcf, 4)) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } + if (crc_final() != rd_u32(crcf)) { + err_rec(ERR_CRC, OP_DATA, 0, start); + break; + } unsigned long long done = 0, next_prog = PROG_STEP; - int status = 0; /* 0=ok, 1=eof, 2=write error */ - static unsigned char chunk[WRITE_CHUNK_SIZE]; + int werr = 0; while (done < size) { size_t want = (size_t)(size - done); - if (want > sizeof chunk) want = sizeof chunk; - if (!rreadn(chunk, want)) { status = 1; break; } - crc_update(chunk, want); - if (write_all(fd, chunk, want) < 0) { status = 2; break; } + if (want > sizeof buf) want = sizeof buf; + if (write_all(fd, buf + done, want) < 0) { werr = errno; break; } done += want; while (done >= next_prog) { unsigned char pp[16]; @@ -313,17 +333,7 @@ int main(int argc, char **argv) } cur = start + done; total += done; - if (status) { - int e = status == 2 ? errno : 0; - err_rec(ERR_DEV, OP_DATA, (unsigned)e, start); - break; - } - unsigned char crcf[4]; - if (!rreadn(crcf, 4)) { err_rec(ERR_DEV, OP_DATA, 0, start); break; } - if (crc_final() != rd_u32(crcf)) { - err_rec(ERR_CRC, OP_DATA, 0, start); - break; - } + if (werr) { err_rec(ERR_DEV, OP_DATA, (unsigned)werr, start); break; } ok_rec(OP_DATA, start, done); } diff --git a/remote/test.py b/remote/test.py index 96d1044..bf992b3 100644 --- a/remote/test.py +++ b/remote/test.py @@ -126,6 +126,36 @@ def main(): assert head == content, f"file bytes mismatch: {head.hex()} != {content.hex()}" assert zeros == b"\0" * 256, f"ZERO region not zeroed: {zeros[:16].hex()}..." + # A CRC mismatch is fatal and must not touch the device: the (distinct) + # payload must not land at offset 0. + bad = b"X" * 16 + r = subprocess.run([binp, blk], + input=frame(OP_SEEK, struct.pack(" Date: Tue, 13 Oct 2026 12:32:29 +0200 Subject: [PATCH 10/10] Test that the write head reports ERR and exits immediately Queue a valid seek and DATA write after each injected I/O fault while keeping stdin open. Assert the exact ERR/DONE response, process exit, and an untouched follow-up write target. Cover SEEK errors and verify a CRC failure prevents a subsequent SIZE query. Assisted-by: OpenCode Co-Authored-By: GPT-6.1 Sol --- remote/test.py | 59 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/remote/test.py b/remote/test.py index bf992b3..c423cba 100644 --- a/remote/test.py +++ b/remote/test.py @@ -133,6 +133,7 @@ def main(): input=frame(OP_SEEK, struct.pack("= 0) errno = EINVAL; return result; @@ -201,41 +203,52 @@ def data(content): seek0 = frame(OP_SEEK, struct.pack(" 8 partial write, then EIO on the retry. - fault_run(seek0 + data(bytes(range(16))), (0, OP_DATA, errno.EIO, 0), 0) + fault_run(data(bytes(range(16))), (0, OP_DATA, errno.EIO, 0)) # DATA: a zero-progress write is a hard error. - fault_run(seek0 + data(bytes(32)), (0, OP_DATA, errno.EIO, 0), 0) + fault_run(data(bytes(32)), (0, OP_DATA, errno.EIO, 0)) # READ: 16 -> 8 partial read, then EIO on the retry. - fault_run(seek0 + frame(OP_READ, struct.pack(" 8 partial write, then EIO on the retry. - fault_run(seek0 + frame(OP_ZERO, struct.pack("