Regression test harness

.github/workflows/ci.yml

@@ -40,6 +40,34 @@ jobs:
       - name: FetchWebsite
         run: arduino-cli --config-file ${{ matrix.config }} --library ./src/ compile --fqbn esp32:esp32:esp32 ./examples/FetchWebsite/FetchWebsite.ino
 
+  native-tests:
+    name: "pio:native"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Cache PlatformIO
+        uses: actions/cache@v6
+        with:
+          key: ${{ runner.os }}-pio-native
+          path: |
+            ~/.cache/pip
+            ~/.platformio
+
+      - name: Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade platformio
+
+      - run: python3 test/check_registration.py
+      - run: pio test -e native -e native-asan -e native-no-ipv6 -e native-no-core-locking
+
   platformio:
     name: "pio:${{ matrix.env }}:${{ matrix.board }}"
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -33,6 +33,16 @@ repos:
       - id: codespell
         exclude: ^.*\.(svd|SVD)$
 
+  - repo: local
+    hooks:
+      # Unity runs what RUN_TEST lists, so an unregistered test is silently dead
+      - id: tests-registered
+        name: every test is registered to run
+        entry: python3 test/check_registration.py
+        language: system
+        files: ^test/(check_registration\.py|test_native/(test_main\.cpp|tests/.*\.cpp))$
+        pass_filenames: false
+
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: "v18.1.3"
     hooks:

README.md

@@ -53,3 +53,13 @@ I personally use the following configuration in my projects:
 
 - ESP32
 - Arduino Core 2.x and 3.x
+
+## Testing
+
+`test/test_native` runs the library on a desktop, against mock lwIP and FreeRTOS:
+
+```
+pio test -e native
+```
+
+See [test/test_native/README.md](test/test_native/README.md) for how to run it and read the results.

platformio.ini

@@ -4,11 +4,15 @@ default_envs = arduino-2, arduino-3, arduino-libretiny
 ; src_dir = examples/FetchWebsite
 src_dir = examples/AsyncSend
 
-[env]
+; Settings shared by every build that targets real hardware.  This cannot be [env],
+; because [env] also applies to [env:native], which has no board and no framework.
+[target]
 framework = arduino
 ; Build the repository as the library it is, so library.json drives the build
 ; rather than lib_dir treating the repository root as a folder of libraries.
 lib_deps = symlink://.
+; The host suite is for [env:native]; there is nothing to flash to a board.
+test_ignore = test_native
 build_flags = 
   -Wall -Wextra
   -D CONFIG_ASYNC_TCP_MAX_ACK_TIME=5000
@@ -27,17 +31,20 @@ monitor_filters = esp32_exception_decoder, log2file
 board = esp32dev
 
 [env:arduino-2]
+extends = target
 platform = espressif32@6.12.0
 
 [env:arduino-3]
+extends = target
 platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip
 
 [env:arduino-libretiny]
+extends = target
 platform = libretiny
 board = generic-bk7231n-qfn32-tuya
 ; board = generic-rtl8710bn-2mb-788k
 build_flags =
-  ${env.build_flags}
+  ${target.build_flags}
   -Wno-unused-parameter
   -Wno-unused-variable
   -Wno-missing-field-initializers
@@ -48,34 +55,99 @@ custom_versions.freertos = 9.0.0
 ;  CI
 
 [env:ci-arduino-2]
+extends = target
 platform = espressif32@6.12.0
 board = ${sysenv.PIO_BOARD}
 
 [env:ci-arduino-3]
+extends = target
 platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip
 board = ${sysenv.PIO_BOARD}
 
 [env:ci-arduino-2-esp-idf_log]
+extends = target
 platform = espressif32@6.12.0
 board = ${sysenv.PIO_BOARD}
 build_flags =
-  ${env.build_flags}
+  ${target.build_flags}
   -D USE_ESP_IDF_LOG=1
   -D TAG=\"core\"
 
 [env:ci-arduino-3-esp-idf_log]
+extends = target
 platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.311/platform-espressif32.zip
 board = ${sysenv.PIO_BOARD}
 build_flags =
-  ${env.build_flags}
+  ${target.build_flags}
   -D USE_ESP_IDF_LOG=1
 
 [env:ci-arduino-libretiny]
+extends = target
 platform = libretiny
 board = ${sysenv.PIO_BOARD}
 build_flags =
-  ${env.build_flags}
+  ${target.build_flags}
   -Wno-unused-parameter
   -Wno-unused-variable
   -Wno-missing-field-initializers
 custom_versions.freertos = 9.0.0
+
+;  Host-native tests -- see test/test_native/README.md
+;
+;  pio test -e native
+;  pio test -e native-asan
+;  pio test -e native-no-ipv6
+;  pio test -e native-no-core-locking
+
+[env:native]
+platform = native
+test_framework = unity
+; Exceptions must stay on, for C as well as C++: the mock FreeRTOS unwinds out of the
+; library's task loop to end a pump, and a failed assertion unwinds out of Unity to end
+; the test (see test/test_native/unity_config.h).
+; Locals start out filled with 0xfe, as fresh heap memory does (see test_main.cpp), so a
+; value read before it was written is the same on every run.
+build_flags =
+  -std=gnu++17
+  -g -O0 -fexceptions
+  -ftrivial-auto-var-init=pattern
+  -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -Wno-format
+  -I src
+  -I test/test_native/mocks/include
+  -D ASYNCTCP_NATIVE_TEST=1
+  -D ARDUINO=10812
+  -D CONFIG_LWIP_TCPIP_CORE_LOCKING=1
+  -D CONFIG_ASYNC_TCP_RUNNING_CORE=-1
+  -D CONFIG_ASYNC_TCP_USE_WDT=1
+  -D CONFIG_ASYNC_TCP_TASK_STACK_SIZE=8192
+  -D CONFIG_ASYNC_TCP_TASK_PRIORITY=3
+  -D CONFIG_ASYNC_TCP_MAX_ACK_TIME=5000
+  -D CONFIG_ASYNC_TCP_PRIORITY=10
+  -D CONFIG_ASYNC_TCP_QUEUE_SIZE=64
+  -D CONFIG_ASYNC_TCP_STACK_SIZE=8192
+
+; Leak detection earns its keep here: ownership of connections and buffers passes
+; between the application, the library and LwIP, and a dropped owner surfaces as a leak
+; rather than as a crash.  Each test is forked, so a leak is reported against the test
+; that caused it.  Undefined behavior is fatal, so it fails the test rather than
+; scrolling past.
+[env:native-asan]
+extends = env:native
+build_flags =
+  ${env:native.build_flags}
+  -fsanitize=address,undefined
+  -fno-sanitize-recover=undefined
+  -fno-omit-frame-pointer
+
+; LwIP built for IPv4 only, as LibreTiny builds it: ip_addr_t is then plain ip4_addr_t.
+[env:native-no-ipv6]
+extends = env:native
+build_flags =
+  ${env:native.build_flags}
+  -D LWIP_IPV6=0
+
+; LwIP without core locking, as ESP-IDF builds it by default and LibreTiny always
+; does: raw tcp_* calls are then safe only on the LwIP thread.
+[env:native-no-core-locking]
+extends = env:native
+build_unflags = -D CONFIG_LWIP_TCPIP_CORE_LOCKING=1

src/AsyncTCP.cpp

@@ -1611,7 +1611,7 @@ int8_t AsyncClientImpl::_poll(tcp_pcb *pcb) {
   }
 
   if (pcb != _pcb) {
-    async_tcp_log_d("0x%08" PRIx32 " != 0x%08" PRIx32, (uint32_t)pcb, (uint32_t)_pcb);
+    async_tcp_log_d("0x%08" PRIx32 " != 0x%08" PRIx32, (uint32_t)(uintptr_t)pcb, (uint32_t)(uintptr_t)_pcb);
     return ERR_OK;
   }
 

test/check_registration.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Check that every test function is actually registered to run.
+
+Unity is told what to run by explicit RUN_TEST lines, so a test that is written but
+never listed is silently dead. This fails the build instead.
+
+It also checks that each test is named for its file: tests in test_foo.cpp are
+test_foo_*.
+
+    python3 test/check_registration.py
+"""
+
+import pathlib
+import re
+import sys
+
+NATIVE = pathlib.Path(__file__).parent / "test_native"
+TESTS = NATIVE / "tests"
+MAIN = NATIVE / "test_main.cpp"
+
+DEFINED = re.compile(r"^\s*static\s+void\s+(test_\w+)\s*\(\s*void\s*\)", re.M)
+REGISTERED = re.compile(r"^\s*RUN_TEST\s*\(\s*(test_\w+)\s*\)", re.M)
+ENTRY = re.compile(r"^\s*void\s+(run_\w+_tests)\s*\(\s*void\s*\)\s*\{", re.M)
+CALLED = re.compile(r"^\s*(run_\w+_tests)\s*\(\s*\)\s*;", re.M)
+
+
+def main() -> int:
+    problems = []
+    total = 0
+
+    for path in sorted(TESTS.glob("*.cpp")):
+        text = path.read_text()
+        defined = set(DEFINED.findall(text))
+        registered = REGISTERED.findall(text)
+        total += len(defined)
+
+        for name in sorted(defined - set(registered)):
+            problems.append(f"{path}: {name} is never run -- add RUN_TEST({name})")
+        for name in sorted(set(registered) - defined):
+            problems.append(f"{path}: RUN_TEST({name}) has no such test")
+        for name in sorted({n for n in registered if registered.count(n) > 1}):
+            problems.append(f"{path}: {name} is run more than once")
+
+        prefix = path.stem + "_"
+        for name in sorted(defined):
+            if not name.startswith(prefix):
+                problems.append(f"{path}: {name} should be named {prefix}*")
+
+    # A file can be fully self-consistent and still never be reached, if main() does
+    # not call its entry point.
+    main_text = MAIN.read_text()
+    called = set(CALLED.findall(main_text))
+    for path in sorted(TESTS.glob("*.cpp")):
+        entries = ENTRY.findall(path.read_text())
+        if not entries:
+            problems.append(f"{path}: no run_*_tests() entry point")
+        for entry in entries:
+            if entry not in called:
+                problems.append(f"{path}: {entry}() is never called from {MAIN.name}")
+
+    if problems:
+        print("\n".join(problems), file=sys.stderr)
+        return 1
+
+    print(f"{total} tests, all registered")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())