Improve Windows mirroring reliability, latency and window controls

.github/workflows/build.yml

@@ -20,7 +20,12 @@ jobs:
         run: choco install openssl innosetup -y --no-progress
 
       - name: Build
-        run: powershell -ExecutionPolicy Bypass -File scripts\build.ps1
+        shell: pwsh
+        run: ./scripts/build.ps1
+
+      - name: Test
+        shell: pwsh
+        run: ctest --test-dir build -C Release --output-on-failure
 
       - name: Build installer
         shell: pwsh

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(AirScreen VERSION 1.0.0 LANGUAGES C CXX)
+project(AirScreen VERSION 1.2.0 LANGUAGES C CXX)
 
 set(CMAKE_C_STANDARD 11)
 set(CMAKE_C_STANDARD_REQUIRED ON)
@@ -111,9 +111,9 @@ target_link_libraries(airplay PUBLIC
     bcrypt
 )
 if (MSVC)
-    target_compile_options(airplay PRIVATE /wd4244 /wd4267 /wd4018 /wd4996 /FI"${COMPAT_DIR}/airscreen_types.h")
-    target_compile_options(plist PRIVATE /wd4244 /wd4267 /wd4996 /wd4018 /FI"${COMPAT_DIR}/airscreen_types.h")
-    target_compile_options(dnssd PRIVATE /wd4244 /wd4267 /FI"${COMPAT_DIR}/airscreen_types.h")
+    target_compile_options(airplay PRIVATE /wd4244 /wd4267 /wd4018 /wd4996 "/FI${COMPAT_DIR}/airscreen_types.h")
+    target_compile_options(plist PRIVATE /wd4244 /wd4267 /wd4996 /wd4018 "/FI${COMPAT_DIR}/airscreen_types.h")
+    target_compile_options(dnssd PRIVATE /wd4244 /wd4267 "/FI${COMPAT_DIR}/airscreen_types.h")
 endif()
 
 if (MSVC)
@@ -190,3 +190,14 @@ install(TARGETS AirScreen RUNTIME DESTINATION .)
 if (FFMPEG_BIN_DIR)
     install(DIRECTORY "${FFMPEG_BIN_DIR}/" DESTINATION . FILES_MATCHING PATTERN "*.dll")
 endif()
+
+include(CTest)
+if (BUILD_TESTING)
+    add_executable(MediaTimingTests tests/media_timing_test.cpp)
+    target_include_directories(MediaTimingTests PRIVATE src)
+    add_test(NAME MediaTiming COMMAND MediaTimingTests)
+
+    add_executable(WindowSizeTests tests/window_size_test.cpp)
+    target_include_directories(WindowSizeTests PRIVATE src)
+    add_test(NAME WindowSize COMMAND WindowSizeTests)
+endif()

README.md

@@ -27,6 +27,7 @@ AirScreen turns a Windows PC into a wireless AirPlay receiver. Your iPhone lists
 - Audio plays through your default Windows output device
 - Rename the receiver, require a pairing PIN, keep the window on top, fill the screen or letterbox
 - Light and dark idle screen
+- Native-resolution window sizes (0.5x, 1x, 1.5x, 2x), optional fixed aspect ratio, and automatic portrait/landscape sizing
 - Closing the window quits AirScreen; settings are also in the tray icon menu
 - Optional Start with Windows: waits in the system tray, ready to mirror to
 
@@ -52,16 +53,23 @@ The installer isn't code-signed yet, so Windows SmartScreen may show "Windows pr
 2. On your iPhone, open **Control Center → Screen Mirroring** and pick **AirScreen**.
 3. If a PIN appears on the PC, enter it on the iPhone.
 
-To stop, end mirroring on the iPhone. Close the window to quit AirScreen. Right-click the window or the tray icon for settings.
+To stop mirroring, end it on the iPhone or choose **Stop mirroring** above **Exit** in the window or tray menu. AirScreen stays available for another connection. Close the window to quit AirScreen. Right-click the window or the tray icon for settings.
+
+**Streaming quality** defaults to 1080p at 30 FPS and also offers 1080p at 60 or 24 FPS, and 720p at 30 FPS (Smooth). Saved quality settings are preserved. Lower settings request a smaller stream from the iPhone, reducing network and decoding load. Changing quality restarts the receiver; reconnect the iPhone afterward. The iPhone chooses the actual stream dimensions and frame rate. For persistent playback delay, try 720p at 30 FPS before reducing audio buffering.
+
+**Size** sets the picture area to **0.5x**, **1x**, **1.5x**, or **2x** the resolution reported by the mirrored device. Use **Ctrl+1**, **Ctrl+2**, **Ctrl+3**, or **Ctrl+4**, respectively. A size selected before connecting applies when the stream starts. Dragging an edge returns to a custom size. Enable **Fixed Ratio** to keep the original picture proportions while resizing. **Auto Rotate Window** is enabled by default and adjusts the window when the mirrored device switches between portrait and landscape; in full screen, it updates the window size restored on exit. Both options are saved.
+
+In full screen, portrait video fits the screen height with its original proportions. **Fill screen (landscape only)** controls whether landscape video fills the display or keeps the complete picture visible.
 
 ## Troubleshooting
 
 **AirScreen doesn't appear on the iPhone**
+- On Windows, discovery uses a connected physical Ethernet or Wi-Fi adapter and its current IPv4 address. VPN and Hyper-V virtual adapters are excluded.
 - Make sure both devices are on the same Wi-Fi network (not a guest network).
 - Turn off any VPN on the PC or the iPhone.
 - Right-click the tray icon → **Allow through firewall**.
 
-**Stuttering or delay.** Use 5 GHz Wi-Fi or connect the PC by Ethernet. Other devices streaming on the same network can add delay.
+**Stuttering or delay.** Use 5 GHz Wi-Fi or connect the PC by Ethernet. Other devices streaming on the same network can add delay. Decoding and presentation run independently; overdue decoded frames are skipped to follow audio playback. With no audio, the newest decoded frame is shown immediately. The log includes the decoder format and periodic video timing statistics.
 
 **No sound.** Check the default playback device in Windows sound settings. Apps that take exclusive control of the device can block AirScreen.
 
@@ -76,7 +84,7 @@ Requires Visual Studio 2022 with the *Desktop development with C++* workload.
 ```powershell
 winget install Kitware.CMake
 winget install ShiningLight.OpenSSL.Dev
-powershell -ExecutionPolicy Bypass -File scripts\build.ps1
+pwsh -File scripts\build.ps1
 ```
 
 `build.ps1` downloads FFmpeg into `third_party\prebuilt`, configures CMake and builds `build\Release\AirScreen.exe` with the required DLLs next to it. OpenSSL is found through `OPENSSL_ROOT_DIR`, the Shining Light install location, or vcpkg.

scripts/airscreen.iss

@@ -2,7 +2,7 @@
 ; iscc /DCrtDir=<VS>\VC\Redist\MSVC\<ver>\x64\Microsoft.VC143.CRT scripts\airscreen.iss
 #define MyAppName "AirScreen"
 #ifndef MyAppVersion
-  #define MyAppVersion "1.0.0"
+  #define MyAppVersion "1.2.0"
 #endif
 #define MyAppPublisher "AirScreen"
 #define MyAppURL "https://github.com/donerfolk/airscreen"

src/app/resource.h

@@ -15,3 +15,16 @@
 #define IDM_TRAY_FILL       40009
 #define IDC_SETTINGS        40010
 #define IDM_TRAY_STARTUP    40011
+
+#define IDM_QUALITY_1080_60 40012
+#define IDM_QUALITY_1080_30 40013
+#define IDM_QUALITY_1080_24 40014
+#define IDM_QUALITY_720_30  40015
+
+#define IDM_SIZE_HALF      40016
+#define IDM_SIZE_NATIVE    40017
+#define IDM_SIZE_ONE_HALF  40018
+#define IDM_SIZE_DOUBLE    40019
+#define IDM_FIXED_RATIO    40020
+#define IDM_AUTO_ROTATE    40021
+#define IDM_TRAY_STOP      40022

src/audio.cpp

@@ -88,6 +88,17 @@ size_t ms_to_samples(int rate, int ch, int ms) {
     return (size_t) rate * (size_t) ch * (size_t) ms / 1000;
 }
 
+int64_t monotonic_ns() {
+    static const int64_t frequency = [] {
+        LARGE_INTEGER value;
+        QueryPerformanceFrequency(&value);
+        return value.QuadPart;
+    }();
+    LARGE_INTEGER value;
+    QueryPerformanceCounter(&value);
+    return value.QuadPart / frequency * 1000000000 + value.QuadPart % frequency * 1000000000 / frequency;
+}
+
 DWORD WINAPI audio_thread_proc(LPVOID p) {
     auto *self = static_cast<AudioPipeline *>(p);
     self->render_thread();
@@ -185,18 +196,26 @@ void AudioPipeline::stop_locked() {
     buffering_ = true;
     ring_.clear();
     rpos_ = wpos_ = count_ = 0;
+    timeline_.clear();
 }
 
 void AudioPipeline::flush() {
     std::lock_guard<std::mutex> dlock(dec_mu_);
     std::lock_guard<std::mutex> lock(mu_);
     rpos_ = wpos_ = count_ = 0;
+    timeline_.clear();
     buffering_ = true;
     if (av_codec_) {
         avcodec_flush_buffers((AVCodecContext *) av_codec_);
     }
 }
 
+int64_t AudioPipeline::playback_time_ns() {
+    std::lock_guard<std::mutex> lock(mu_);
+    return running_ && out_rate_ > 0 && out_ch_ > 0
+        ? timeline_.position(monotonic_ns(), out_rate_ * out_ch_) : 0;
+}
+
 void AudioPipeline::set_volume(float linear) {
     if (linear < 0) {
         linear = 0;
@@ -229,6 +248,7 @@ bool AudioPipeline::init_decoder(unsigned char codec_type) {
     av_channel_layout_default(&ctx->ch_layout, src_ch_);
     ctx->request_sample_fmt = AV_SAMPLE_FMT_S16;
     ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
+    ctx->pkt_timebase = AVRational{1, 1000000000};
     if (id == AV_CODEC_ID_ALAC) {
         set_extra(ctx, kAlacMagic, (int) sizeof(kAlacMagic));
     } else if (codec_type == 4) {
@@ -380,6 +400,7 @@ bool AudioPipeline::init_wasapi() {
     std::lock_guard<std::mutex> lock(mu_);
     ring_.assign(cap, 0);
     rpos_ = wpos_ = count_ = 0;
+    timeline_.clear();
     return true;
 }
 
@@ -429,7 +450,7 @@ bool AudioPipeline::ensure_swr(int in_fmt, int in_rate, int in_ch) {
     return true;
 }
 
-void AudioPipeline::convert_and_push(const uint8_t **src, int in_samples) {
+void AudioPipeline::convert_and_push(const uint8_t **src, int in_samples, int64_t timestamp_ns) {
     if (!swr_ || in_samples <= 0) {
         return;
     }
@@ -439,19 +460,26 @@ void AudioPipeline::convert_and_push(const uint8_t **src, int in_samples) {
     }
     std::vector<int16_t> pcm((size_t) out_samples * (size_t) out_ch_ + 32);
     uint8_t *dst = (uint8_t *) pcm.data();
+    int64_t delay_ns = swr_get_delay((SwrContext *) swr_, 1000000000);
     int got = swr_convert((SwrContext *) swr_, &dst, out_samples, src, in_samples);
     if (got > 0) {
-        push_pcm(pcm.data(), got, out_ch_);
+        push_pcm(pcm.data(), got, out_ch_, timestamp_ns > 0 ? timestamp_ns - delay_ns : 0);
     }
 }
 
-void AudioPipeline::push_pcm(const int16_t *pcm, int frames, int channels) {
+void AudioPipeline::push_pcm(const int16_t *pcm, int frames, int channels, int64_t timestamp_ns) {
     std::lock_guard<std::mutex> lock(mu_);
     if (ring_.empty() || frames <= 0 || channels <= 0) {
         return;
     }
     size_t samples = (size_t) frames * (size_t) channels;
     size_t cap = ring_.size();
+    if (samples > cap) {
+        size_t trim = samples - cap;
+        pcm += trim;
+        samples = cap;
+        timestamp_ns += AudioTimeline::duration_ns(trim, out_rate_ * out_ch_);
+    }
     size_t max_keep = ms_to_samples(out_rate_, out_ch_, kMaxMs);
     if (max_keep < samples) {
         max_keep = samples;
@@ -468,7 +496,9 @@ void AudioPipeline::push_pcm(const int16_t *pcm, int frames, int channels) {
         rpos_ = (rpos_ + drop) % cap;
         count_ -= drop;
         dropped_ += drop;
+        timeline_.discard(drop, out_rate_ * out_ch_);
     }
+    timeline_.append(timestamp_ns, samples);
     float vol = volume_.load();
     for (size_t i = 0; i < samples; ++i) {
         int v = (int) (pcm[i] * vol);
@@ -489,7 +519,6 @@ void AudioPipeline::push_pcm(const int16_t *pcm, int frames, int channels) {
 }
 
 void AudioPipeline::submit(const uint8_t *data, int len, uint64_t ntp_ns) {
-    (void) ntp_ns;
     if (!data || len <= 0) {
         return;
     }
@@ -504,20 +533,24 @@ void AudioPipeline::submit(const uint8_t *data, int len, uint64_t ntp_ns) {
             return;
         }
         const uint8_t *src = data;
-        convert_and_push(&src, frames);
+        convert_and_push(&src, frames, (int64_t) ntp_ns);
         return;
     }
     if (!av_codec_ || !av_pkt_ || !av_frame_) {
         return;
     }
     auto *ctx = (AVCodecContext *) av_codec_;
     auto *pkt = (AVPacket *) av_pkt_;
-    auto *frame = (AVFrame *) av_frame_;
     if (av_new_packet(pkt, len) < 0) {
         return;
     }
     memcpy(pkt->data, data, (size_t) len);
+    pkt->pts = ntp_ns ? (int64_t) ntp_ns : AV_NOPTS_VALUE;
     int serr = avcodec_send_packet(ctx, pkt);
+    if (serr == AVERROR(EAGAIN)) {
+        receive_frames();
+        serr = avcodec_send_packet(ctx, pkt);
+    }
     av_packet_unref(pkt);
     if (serr < 0 && serr != AVERROR(EAGAIN)) {
         static int decode_errs = 0;
@@ -529,14 +562,21 @@ void AudioPipeline::submit(const uint8_t *data, int len, uint64_t ntp_ns) {
         }
         return;
     }
+    receive_frames();
+}
+
+void AudioPipeline::receive_frames() {
+    auto *ctx = (AVCodecContext *) av_codec_;
+    auto *frame = (AVFrame *) av_frame_;
     while (avcodec_receive_frame(ctx, frame) == 0) {
         int in_rate = frame->sample_rate > 0 ? frame->sample_rate : src_rate_;
         int in_ch = frame->ch_layout.nb_channels > 0 ? frame->ch_layout.nb_channels : src_ch_;
         if (!ensure_swr(frame->format, in_rate, in_ch)) {
             av_frame_unref(frame);
             continue;
         }
-        convert_and_push((const uint8_t **) frame->extended_data, frame->nb_samples);
+        convert_and_push((const uint8_t **) frame->extended_data, frame->nb_samples,
+                         frame->pts == AV_NOPTS_VALUE ? 0 : frame->pts);
         av_frame_unref(frame);
     }
 }
@@ -565,6 +605,7 @@ void AudioPipeline::render_thread() {
         if (!running_) {
             break;
         }
+        int64_t output_now = monotonic_ns();
         UINT32 padding = 0;
         if (FAILED(client->GetCurrentPadding(&padding))) {
             continue;
@@ -590,6 +631,7 @@ void AudioPipeline::render_thread() {
                 rpos_ = (rpos_ + drop) % cap;
                 count_ -= drop;
                 dropped_ += drop;
+                timeline_.discard(drop, out_rate_ * out_ch_);
                 buffering_ = false;
             }
             if (!buffering_) {
@@ -608,21 +650,27 @@ void AudioPipeline::render_thread() {
             if (n < need) {
                 memset(tmp.data() + n, 0, (need - n) * sizeof(int16_t));
             }
-        }
-        if (out_float_) {
-            float *out = (float *) dest;
-            for (size_t i = 0; i < need; ++i) {
-                out[i] = tmp[i] * (1.0f / 32768.0f);
+            if (out_float_) {
+                float *out = (float *) dest;
+                for (size_t i = 0; i < need; ++i) {
+                    out[i] = tmp[i] * (1.0f / 32768.0f);
+                }
+            } else if (out_bits_ >= 24) {
+                int32_t *out = (int32_t *) dest;
+                for (size_t i = 0; i < need; ++i) {
+                    out[i] = (int32_t) tmp[i] << 16;
+                }
+            } else {
+                memcpy(dest, tmp.data(), need * sizeof(int16_t));
             }
-        } else if (out_bits_ >= 24) {
-            int32_t *out = (int32_t *) dest;
-            for (size_t i = 0; i < need; ++i) {
-                out[i] = (int32_t) tmp[i] << 16;
+            HRESULT released = render->ReleaseBuffer(avail, 0);
+            if (SUCCEEDED(released)) {
+                timeline_.schedule(n, output_now + AudioTimeline::duration_ns(
+                    (size_t) padding * out_ch_, out_rate_ * out_ch_), out_rate_ * out_ch_, output_now);
+            } else {
+                timeline_.discard(n, out_rate_ * out_ch_);
             }
-        } else {
-            memcpy(dest, tmp.data(), need * sizeof(int16_t));
         }
-        render->ReleaseBuffer(avail, 0);
     }
     if (mmcss) {
         AvRevertMmThreadCharacteristics(mmcss);

src/audio.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "media_timing.h"
+
 #include <cstdint>
 #include <mutex>
 #include <vector>
@@ -19,15 +21,17 @@ class AudioPipeline {
     void submit(const uint8_t *data, int len, uint64_t ntp_ns);
     bool is_running() const { return running_.load(); }
     void render_thread();
+    int64_t playback_time_ns();
 
 private:
     bool init_decoder(unsigned char codec_type);
     bool init_wasapi();
+    void receive_frames();
     void teardown_wasapi();
     void stop_locked();
-    void push_pcm(const int16_t *pcm, int frames, int channels);
+    void push_pcm(const int16_t *pcm, int frames, int channels, int64_t timestamp_ns);
     bool ensure_swr(int in_fmt, int in_rate, int in_ch);
-    void convert_and_push(const uint8_t **src, int in_samples);
+    void convert_and_push(const uint8_t **src, int in_samples, int64_t timestamp_ns);
 
     void *av_codec_ = nullptr;
     void *av_frame_ = nullptr;
@@ -58,6 +62,7 @@ class AudioPipeline {
     size_t rpos_ = 0;
     size_t wpos_ = 0;
     size_t count_ = 0;
+    AudioTimeline timeline_;
     bool buffering_ = true; // refilling cushion before playing; guarded by mu_
     unsigned rebuffers_ = 0;
     size_t dropped_ = 0;