Inventory Trusted Types violations in system tests

.github/workflows/test.yml

@@ -71,3 +71,11 @@ jobs:
 
       - name: Run system tests
         run: bin/rails test:system
+
+      - name: Upload Trusted Types inventory
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: trusted-types-violations-${{ inputs.saas && 'saas' || 'oss' }}-${{ matrix.db_adapter }}
+          path: tmp/trusted_types_violations.json
+          if-no-files-found: ignore

config/environments/test.rb

@@ -70,4 +70,8 @@
 
   # Enable multi-tenant mode for tests
   config.x.multi_tenant.enabled = true
+
+  # Serve the Trusted Types report-only policy so system tests can inventory
+  # the DOM sinks that would violate it. See TrustedTypesViolationsTestHelper.
+  config.x.content_security_policy.trusted_types_report_only = true
 end

config/initializers/content_security_policy.rb

@@ -15,6 +15,48 @@
 # config.x.content_security_policy.* (string, space-separated string, or array):
 #   script_src, style_src, connect_src, frame_src, img_src, font_src, media_src,
 #   worker_src, frame_ancestors, form_action, report_uri, report_only
+#
+# config.x.content_security_policy.trusted_types_report_only (boolean): adds
+# the Trusted Types report-only policy below. On in the test environment only.
+
+# Trusted Types report-only policy.
+#
+# Marks every raw-string write to a DOM sink (innerHTML, outerHTML,
+# insertAdjacentHTML, DOMParser#parseFromString, document.write, ...) as a
+# violation without blocking it, so we can inventory the sinks fizzy and its
+# libraries write to before designing enforcement. Chromium, Safari and Firefox
+# all support Trusted Types.
+#
+# It rides a separate Content-Security-Policy-Report-Only header, so the policy
+# below is untouched whether it enforces or reports. Report-only can only
+# report, never block.
+#
+# Only `require-trusted-types-for 'script'` is emitted. A `trusted-types`
+# allowlist restricts which policy names may be created, which would only add
+# policy-creation noise (lexxy creates its own DOMPurify policy) to a pilot that
+# registers no policy of its own.
+class TrustedTypesReportOnly
+  DIRECTIVE = "require-trusted-types-for 'script'"
+  HEADER = ActionDispatch::Constants::CONTENT_SECURITY_POLICY_REPORT_ONLY.downcase # Rack 3 header names are lowercase
+
+  def initialize(app, report_uri = nil)
+    @app = app
+    @policy = report_uri ? "#{DIRECTIVE}; report-uri #{report_uri}" : DIRECTIVE
+  end
+
+  def call(env)
+    status, headers, body = @app.call(env)
+
+    # Only documents run script, so only they can write to a sink. When the app
+    # policy is itself report-only, both go out as separate header fields (a
+    # Rack 3 array), each reporting independently.
+    if headers[Rack::CONTENT_TYPE].to_s.start_with?("text/html")
+      headers[HEADER] = headers[HEADER] ? [ *headers[HEADER], @policy ] : @policy
+    end
+
+    [ status, headers, body ]
+  end
+end
 
 Rails.application.configure do
   # Helper to get additional CSP sources from ENV or config.x.
@@ -76,6 +118,12 @@
   # Report violations without enforcing the policy.
   config.content_security_policy_report_only = report_only
 
+  # Inserted outside the Rails CSP middleware so it runs after that header is
+  # written, stacking beside an app policy that is itself report-only.
+  if config.x.content_security_policy.trusted_types_report_only
+    config.middleware.insert_before ActionDispatch::ContentSecurityPolicy::Middleware, TrustedTypesReportOnly, report_uri
+  end
+
   # Locked-down policy for the static pages served from public/ (error pages).
   # They carry no script or inline styles at all, so everything falls back to
   # default-src 'none'.

test/application_system_test_case.rb

@@ -1,6 +1,8 @@
 require "test_helper"
 
 class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
+  include TrustedTypesViolationsTestHelper
+
   browser_options = Selenium::WebDriver::Chrome::Options.new.tap do |opts|
     opts.add_argument("--window-size=1200,800")
     opts.add_argument("--disable-extensions")

test/integration/static_csp_test.rb

@@ -35,6 +35,14 @@ class StaticCspTest < ActionDispatch::IntegrationTest
     assert_not_equal STATIC_POLICY, csp
   end
 
+  test "the Trusted Types policy rides a report-only header and never the app policy" do
+    sign_in_as identities(:david)
+    get root_url
+
+    assert_not_includes response.headers[ActionDispatch::Constants::CONTENT_SECURITY_POLICY].to_s, "trusted-types"
+    assert_equal "require-trusted-types-for 'script'", response.headers[ActionDispatch::Constants::CONTENT_SECURITY_POLICY_REPORT_ONLY]
+  end
+
   private
     # Suppress the diagnostics page the test environment would otherwise
     # render, so the request falls through to the exceptions app like in

test/middleware/trusted_types_report_only_test.rb

@@ -0,0 +1,35 @@
+require "test_helper"
+require "rack/lint"
+
+class TrustedTypesReportOnlyTest < ActiveSupport::TestCase
+  HEADER = "content-security-policy-report-only"
+  DIRECTIVE = "require-trusted-types-for 'script'"
+
+  test "adds the directive as the only report-only policy when the app sets none" do
+    headers = call_with "text/html; charset=utf-8"
+
+    assert_equal DIRECTIVE, headers[HEADER]
+  end
+
+  test "stacks a second header field beside an existing report-only policy" do
+    headers = call_with "text/html", HEADER => "default-src 'self'"
+
+    assert_equal [ "default-src 'self'", DIRECTIVE ], headers[HEADER]
+  end
+
+  test "leaves non-document responses alone" do
+    headers = call_with "application/json"
+
+    assert_nil headers[HEADER]
+  end
+
+  private
+    def call_with(content_type, extra_headers = {})
+      inner = ->(env) { [ 200, { "content-type" => content_type, **extra_headers }, [ "" ] ] }
+      app = Rack::Lint.new(TrustedTypesReportOnly.new(inner))
+
+      _status, headers, body = app.call(Rack::MockRequest.env_for("/"))
+      body.close
+      headers
+    end
+end

test/test_helpers/trusted_types_violations_test_helper.rb

@@ -0,0 +1,130 @@
+# Inventories the Trusted Types violations system tests trip, without failing
+# any test over them.
+#
+# The test environment serves the report-only Trusted Types policy (see
+# config/initializers/content_security_policy.rb). Before any page script runs,
+# a listener injected over CDP records each `securitypolicyviolation` for that
+# policy into sessionStorage, which survives full page loads. Teardown drains it
+# into tmp/trusted_types_violations.jsonl, one line per violation, and the run
+# ends by grouping those by sink and source location into
+# tmp/trusted_types_violations.json.
+#
+# Only the default Capybara session is observed.
+module TrustedTypesViolationsTestHelper
+  extend ActiveSupport::Concern
+
+  LOG = Rails.root.join("tmp/trusted_types_violations.jsonl")
+  INVENTORY = Rails.root.join("tmp/trusted_types_violations.json")
+  STORAGE_KEY = "trusted-types-violations"
+
+  LISTENER = <<~JS
+    window.addEventListener("securitypolicyviolation", (event) => {
+      if (!/trusted-types/.test(event.effectiveDirective)) return
+
+      try {
+        const violations = JSON.parse(sessionStorage.getItem("#{STORAGE_KEY}") || "[]")
+        violations.push({
+          directive: event.effectiveDirective,
+          sample: event.sample,
+          source: event.sourceFile,
+          line: event.lineNumber,
+          column: event.columnNumber,
+          page: location.pathname
+        })
+        sessionStorage.setItem("#{STORAGE_KEY}", JSON.stringify(violations))
+      } catch {}
+    }, true)
+  JS
+
+  DRAIN = <<~JS
+    try {
+      const violations = sessionStorage.getItem("#{STORAGE_KEY}") || "[]"
+      sessionStorage.removeItem("#{STORAGE_KEY}")
+      return violations
+    } catch {
+      return "[]"
+    }
+  JS
+
+  class << self
+    def reset
+      FileUtils.rm_f [ LOG, INVENTORY ]
+    end
+
+    def record(test, violations)
+      if violations.any?
+        lines = violations.map { |violation| violation.merge("test" => test).to_json + "\n" }
+        File.open(LOG, "a") { |file| file.write(lines.join) }
+      end
+    end
+
+    def write_inventory
+      if LOG.exist?
+        violations = LOG.each_line.map { |line| JSON.parse(line) }
+        INVENTORY.write JSON.pretty_generate(inventory_of(violations))
+      end
+    end
+
+    private
+      def inventory_of(violations)
+        groups = violations.group_by { |violation| [ sink_of(violation), location_of(violation) ] }
+
+        { "violations" => violations.size,
+          "by_sink" => violations.group_by { |violation| sink_of(violation) }.transform_values(&:size).sort_by { -it.last }.to_h,
+          "sites" => groups.map { |(sink, location), group| site(sink, location, group) }.sort_by { -it["count"] } }
+      end
+
+      def site(sink, location, violations)
+        { "sink" => sink,
+          "source" => location,
+          "count" => violations.size,
+          "pages" => violations.map { it["page"] }.uniq.sort,
+          "tests" => violations.map { it["test"] }.uniq.sort,
+          "example" => violations.first["sample"] }
+      end
+
+      # Chromium samples read "<sink>|<first 40 characters written>".
+      def sink_of(violation)
+        violation["sample"].to_s.split("|", 2).first.presence || violation["directive"]
+      end
+
+      def location_of(violation)
+        [ script_of(violation["source"]), violation["line"], violation["column"] ].join(":")
+      end
+
+      # Drop the origin, which changes with the test server's port, and the
+      # asset digest, which changes with every build.
+      def script_of(source)
+        path = URI.parse(source.to_s).path.presence || source.presence || "(inline)"
+        path.sub(/-\h{8,}(?=\.js\z)/, "")
+      rescue URI::InvalidURIError
+        source
+      end
+  end
+
+  included do
+    TrustedTypesViolationsTestHelper.reset
+    Minitest.after_run { TrustedTypesViolationsTestHelper.write_inventory }
+
+    setup do
+      @trusted_types_listener = page.driver.browser.execute_cdp("Page.addScriptToEvaluateOnNewDocument", source: LISTENER)["identifier"]
+    end
+  end
+
+  # Teardown callbacks run after Capybara has reset the session to a blank
+  # page, so drain before then.
+  def before_teardown
+    TrustedTypesViolationsTestHelper.record "#{self.class.name}##{name}", drain_trusted_types_violations
+    page.driver.browser.execute_cdp("Page.removeScriptToEvaluateOnNewDocument", identifier: @trusted_types_listener)
+  ensure
+    super
+  end
+
+  private
+    def drain_trusted_types_violations
+      JSON.parse(page.execute_script(DRAIN))
+    rescue Selenium::WebDriver::Error::WebDriverError => error
+      warn "Trusted Types inventory lost #{name}'s violations: #{error.class}"
+      []
+    end
+end