feat(devices): name the crawlers production reads generically

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
       - uses: actions/checkout@v7
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
-      # Every one of the 38 030 fixture cases, one test each, read off disk. Twice: once against
+      # Every one of the 38 301 fixture cases, one test each, read off disk. Twice: once against
       # the database as it ships, embedded, and once against the files, which is how it is run
       # while working.
       - run: cargo test --release --test corpus

AGENTS.md

@@ -8,11 +8,17 @@ agent needs to know before touching anything.
 
 ## The specification is the fixtures, and only the fixtures
 
-`tests/matomo-device-detector/fixtures/*.yml` says what every one of 38 030 user agents means.
+`tests/matomo-device-detector/fixtures/*.yml` says what every one of 38 301 user agents means.
 That corpus, pinned to one revision of matomo-org/device-detector in `castor.php`, is the whole
 specification. A change is right when the tests pass and wrong when they do not; there is no other
 authority to appeal to.
 
+With one exception, kept as short as it can be: `tests/corrections.yml` overrules the category
+the fixtures give a bot, where the operator's own page or the evidence says otherwise -- matomo's
+Search bot for a crawler that feeds no search engine. Each correction carries its source, applies
+to every case naming that bot, and fails the corpus once no case names it, so it goes when matomo
+agrees. A category is the only thing it may correct.
+
 Matomo's own regexes are not vendored, but they may be read: this library is LGPL-3.0-or-later,
 the same family as matomo's, so there is nothing to keep out. Read them for what they say about
 *which* tokens carry meaning -- that is the question the fixtures answer only by example, one
@@ -130,7 +136,7 @@ feature does rebuild, so stay on one side of it while working.
 
 `cargo bench` is criterion, so it keeps the last run and tells you what moved; `--save-baseline`
 and `--baseline` name one to keep. It detects over a stride across the corpus rather than all
-38 030 agents, because at a budget of zero a lookup costs about six milliseconds against half a
+38 301 agents, because at a budget of zero a lookup costs about six milliseconds against half a
 millisecond cached, and ten samples of the whole corpus at that rate is most of an hour. That
 makes the default good for comparing two runs and not for quoting on its own; `DETECT_SAMPLE`
 raises it when a number has to stand by itself. Criterion sizes a run by the measurement time it

README.md

@@ -15,7 +15,7 @@ assert_eq!(found.os.unwrap().version, "10");
 ```
 
 This library is based on the [matomo device detector library](https://github.com/matomo-org/device-detector):
-its test fixtures, 38 030 user agents and what each one means, are the specification this database
+its test fixtures, 38 301 user agents and what each one means, are the specification this database
 is written against. Matomo's own regexes are not vendored and none of them is copied here.
 
 The main change is how the parsing is done, with a specific structure to ensure better speed with

benches/detection.rs

@@ -2,7 +2,7 @@
 //!
 //!     cargo bench
 //!     cargo bench -- detect/20000        # one group of it
-//!     DETECT_SAMPLE=38030 cargo bench    # the whole corpus, and a long wait
+//!     DETECT_SAMPLE=38301 cargo bench    # the whole corpus, and a long wait
 //!
 //! The memory side of the same question is `cargo run --release --example memory`. Criterion
 //! cannot answer it: resident size is a property of the process rather than of an operation, and

castor.php

@@ -14,7 +14,7 @@
  * library answers. How matomo goes about answering it is its own business, and its regexes are
  * under a licence we do not want to carry.
  */
-const MATOMO_REVISION = '5a6f5d0184b5a867c1db9c0733f6b89686c5c47b';
+const MATOMO_REVISION = 'e83ce7a97f05d8ee3e1eee7ea77a73b3bdfebd9a';
 
 #[AsTask(description: 'Syncs the matomo device-detector test fixtures', name: 'sync', namespace: 'matomo')]
 function sync()

examples/all/fixtures.rs

@@ -1,6 +1,6 @@
 //! The fixtures, read as they are rather than compiled into a test.
 //!
-//! `tests/matomo-device-detector/fixtures/*.yml` is the specification: 38 030 user agents and
+//! `tests/matomo-device-detector/fixtures/*.yml` is the specification: 38 301 user agents and
 //! what each one means. Generating a `#[test]` per case out of them cost forty megabytes of rust
 //! and two minutes of build for every change to an entry; reading them at startup costs the
 //! second the yaml takes to parse.
@@ -14,6 +14,39 @@ use ua_detector::Detection;
 
 pub const FIXTURES: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/matomo-device-detector/fixtures");
 
+pub const CORRECTIONS: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/corrections.yml");
+
+/// A category this library gives a bot against what the fixtures say, from `tests/corrections.yml`.
+#[derive(serde::Deserialize)]
+pub struct Correction {
+    pub bot: String,
+    pub category: String,
+    pub source: String,
+    pub reason: String,
+}
+
+pub fn corrections() -> &'static [Correction] {
+    static CORRECTIONS_READ: std::sync::OnceLock<Vec<Correction>> = std::sync::OnceLock::new();
+
+    CORRECTIONS_READ.get_or_init(|| {
+        let content = std::fs::read_to_string(CORRECTIONS)
+            .unwrap_or_else(|error| panic!("cannot read {CORRECTIONS}: {error}"));
+
+        serde_yaml::from_str(&content).unwrap_or_else(|error| panic!("cannot parse {CORRECTIONS}: {error}"))
+    })
+}
+
+/// The corrections no fixture case names a bot for, which have nothing left to correct.
+pub fn unused_corrections() -> Vec<&'static str> {
+    let named: std::collections::HashSet<String> = files()
+        .iter()
+        .flat_map(|path| read(path))
+        .filter_map(|case| case.expected.get("bot")?.get("name")?.as_str().map(str::to_owned))
+        .collect();
+
+    corrections().iter().filter(|correction| !named.contains(&correction.bot)).map(|correction| correction.bot.as_str()).collect()
+}
+
 #[derive(serde::Deserialize)]
 pub struct Case {
     pub user_agent: String,
@@ -78,6 +111,14 @@ fn expected_of(value: &serde_yaml::Value) -> Detection {
     if let serde_yaml::Value::Mapping(mapping) = &mut value {
         mapping.remove("headers");
 
+        if let Some(serde_yaml::Value::Mapping(bot)) = mapping.get_mut("bot") {
+            let name = bot.get("name").and_then(|name| name.as_str()).unwrap_or_default();
+
+            if let Some(correction) = corrections().iter().find(|correction| correction.bot == name) {
+                bot.insert("category".into(), correction.category.clone().into());
+            }
+        }
+
         // A fixture writes an absent result as null, or as an empty list.
         for key in ["os", "client", "device"] {
             if mapping.get(key).is_some_and(|value| value.is_null() || value.is_sequence()) {

examples/report.rs

@@ -3,38 +3,24 @@
 //!     cargo run --example report -- smart_display          # only what fails
 //!     cargo run --example report -- smart_display --shapes # failures grouped by shape
 
-use std::collections::BTreeMap;
-
-use ua_detector::Detection;
+#[path = "all/fixtures.rs"]
+mod fixtures;
 
-#[derive(serde::Deserialize)]
-struct Case {
-    user_agent: String,
-    #[serde(default)]
-    headers: BTreeMap<String, serde_yaml::Value>,
-    #[serde(flatten)]
-    expected: serde_yaml::Value,
-}
+use std::collections::BTreeMap;
 
 fn main() {
     let mut args = std::env::args().skip(1);
     let name = args.next().expect("usage: report <fixture> [--shapes]");
     let shapes = args.any(|arg| arg == "--shapes");
 
-    let path = format!("tests/matomo-device-detector/fixtures/{name}.yml");
-    let content = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"));
-    let cases: Vec<Case> = serde_yaml::from_str(&content).expect("a valid fixture");
+    let cases = fixtures::read(std::path::Path::new(&format!("{}/{name}.yml", fixtures::FIXTURES)));
 
     let detector = ua_detector::shared();
     let mut failures = Vec::new();
 
     for case in &cases {
-        let expected: Detection = expected_of(&case.expected);
-        let headers: Vec<(String, String)> = case
-            .headers
-            .iter()
-            .map(|(name, value)| (name.clone(), header_of(value)))
-            .collect();
+        let expected = case.expected();
+        let headers = case.headers();
         let headers: Vec<(&str, &str)> =
             headers.iter().map(|(name, value)| (name.as_str(), value.as_str())).collect();
         let detected = detector.detect_with_headers(&case.user_agent, &headers);
@@ -73,42 +59,6 @@ fn main() {
     println!("{}/{} case(s) failing", failures.len(), cases.len());
 }
 
-/// A few fixtures give a brand list as a structure rather than as the header text a browser
-/// actually sends. Render it back to the wire form.
-fn header_of(value: &serde_yaml::Value) -> String {
-    match value {
-        serde_yaml::Value::String(text) => text.clone(),
-        serde_yaml::Value::Sequence(brands) => brands
-            .iter()
-            .map(|brand| {
-                let text = |key| brand.get(key).and_then(|v| v.as_str()).unwrap_or_default();
-
-                format!("\"{}\";v=\"{}\"", text("brand"), text("version"))
-            })
-            .collect::<Vec<String>>()
-            .join(", "),
-        _ => String::new(),
-    }
-}
-
-/// A fixture spells an undetected operating system as an empty list and an undetected client
-/// as null, neither of which our entry format uses.
-fn expected_of(value: &serde_yaml::Value) -> Detection {
-    let mut value = value.clone();
-
-    if let serde_yaml::Value::Mapping(mapping) = &mut value {
-        mapping.remove("headers");
-
-        for key in ["os", "client", "device"] {
-            if mapping.get(key).is_some_and(|value| value.is_null() || value.is_sequence()) {
-                mapping.remove(key);
-            }
-        }
-    }
-
-    serde_yaml::from_value(value).expect("a fixture our types can hold")
-}
-
 /// Strips the parts of a user agent that vary between otherwise identical cases, so failures
 /// can be counted per shape rather than one by one.
 fn shape_of(user_agent: &str) -> String {