feat(decisions): annotate, annotate_provider and Rerank.decide(provider=)

integration/test_collection_decisions.py

@@ -51,7 +51,7 @@ def test_decisions_config_round_trip(collection_factory: CollectionFactory) -> N
     assert config.reranker_config.reranker == "reranker-dummy"
 
 
-def test_decide_with_every_kind_of_question(collection_factory: CollectionFactory) -> None:
+def test_annotate_with_every_kind_of_question(collection_factory: CollectionFactory) -> None:
     collection = _collection(collection_factory)
     collection.data.insert_many(
         [{"text": "the customer is angry about billing"}, {"text": "all good, thanks"}]
@@ -68,32 +68,32 @@ def test_decide_with_every_kind_of_question(collection_factory: CollectionFactor
     ]
 
     for query in (
-        lambda: collection.query.bm25("billing good", decide=questions),
-        lambda: collection.query.fetch_objects(decide=questions),
-        lambda: collection.query.hybrid("billing good", alpha=0, decide=questions),
+        lambda: collection.query.bm25("billing good", annotate=questions),
+        lambda: collection.query.fetch_objects(annotate=questions),
+        lambda: collection.query.hybrid("billing good", alpha=0, annotate=questions),
     ):
         objs = sorted(query().objects, key=lambda o: str(o.properties["text"]))
         assert len(objs) == 2
         angry, fine = objs[1], objs[0]
-        assert angry.decisions is not None and fine.decisions is not None
-        assert angry.decisions["angry"].probability == 1.0
-        assert fine.decisions["angry"].probability == 0.0
-        assert angry.decisions["team"].choice == "billing"
-        assert angry.decisions["team"].probabilities == {"billing": 1.0, "other": 0.0}
-        assert angry.decisions["team"].confidence == 1.0
-        assert fine.decisions["team"].choice == "billing"
-        assert angry.decisions["length"].score == float(
+        assert angry.annotations is not None and fine.annotations is not None
+        assert angry.annotations["angry"].probability == 1.0
+        assert fine.annotations["angry"].probability == 0.0
+        assert angry.annotations["team"].choice == "billing"
+        assert angry.annotations["team"].probabilities == {"billing": 1.0, "other": 0.0}
+        assert angry.annotations["team"].confidence == 1.0
+        assert fine.annotations["team"].choice == "billing"
+        assert angry.annotations["length"].score == float(
             len("the customer is angry about billing") % 3
         )
-        assert fine.decisions["length"].score == float(len("all good, thanks") % 3)
-        assert angry.decisions["length"].probabilities is not None
-        assert set(angry.decisions["length"].probabilities) == {"short", "medium", "long"}
-        assert angry.decisions["no"].refused
-        assert angry.decisions["no"].probability is None
+        assert fine.annotations["length"].score == float(len("all good, thanks") % 3)
+        assert angry.annotations["length"].probabilities is not None
+        assert set(angry.annotations["length"].probabilities) == {"short", "medium", "long"}
+        assert angry.annotations["no"].refused
+        assert angry.annotations["no"].probability is None
 
-    objs = collection.query.bm25("billing", decide=questions[0]).objects
-    assert objs[0].decisions is not None
-    assert list(objs[0].decisions) == ["angry"]
+    objs = collection.query.bm25("billing", annotate=questions[0]).objects
+    assert objs[0].annotations is not None
+    assert list(objs[0].annotations) == ["angry"]
 
 
 ANGRY = "the customer is angry about billing"
@@ -205,16 +205,16 @@ def test_a_reranker_and_a_decisions_module_on_one_collection(
         pytest.skip("the server has no decisions-dummy module")
     collection.data.insert_many([{"text": ANGRY}, {"text": CALM}])
 
-    # A plain rerank goes to the reranker, which scores by length; decide goes to the
+    # A plain rerank goes to the reranker, which scores by length; annotate goes to the
     # decisions module.
     objs = collection.query.bm25(
         "billing good",
         rerank=wvc.query.Rerank(prop="text", query="customer angry"),
-        decide=wvc.query.Decision.predicate("angry", prop="text", instructions="customer angry"),
+        annotate=wvc.query.Decision.predicate("angry", prop="text", instructions="customer angry"),
     ).objects
     assert _texts(objs) == [ANGRY, CALM]
     assert [o.metadata.rerank_score for o in objs] == [float(len(ANGRY)), float(len(CALM))]
-    assert [o.decisions["angry"].probability for o in objs if o.decisions] == [1.0, 0.0]
+    assert [o.annotations["angry"].probability for o in objs if o.annotations] == [1.0, 0.0]
 
     # A decision goes to the decisions module: the probability is the score.
     objs = collection.query.bm25(
@@ -225,7 +225,7 @@ def test_a_reranker_and_a_decisions_module_on_one_collection(
     assert objs[0].metadata.rerank_score == 1.0
 
 
-def test_decide_is_rejected_with_group_by(collection_factory: CollectionFactory) -> None:
+def test_annotate_is_rejected_with_group_by(collection_factory: CollectionFactory) -> None:
     collection = _collection(collection_factory)
     collection.data.insert({"text": "x"})
 
@@ -234,11 +234,11 @@ def test_decide_is_rejected_with_group_by(collection_factory: CollectionFactory)
         collection.query.bm25(
             "x",
             group_by=wvc.query.GroupBy(prop="text", objects_per_group=1, number_of_groups=1),
-            decide=wvc.query.Decision.predicate("q", prop="text", instructions="i"),
+            annotate=wvc.query.Decision.predicate("q", prop="text", instructions="i"),
         )
 
 
-def test_decide_without_a_decisions_module(collection_factory: CollectionFactory) -> None:
+def test_annotate_without_a_decisions_module(collection_factory: CollectionFactory) -> None:
     collection = collection_factory(
         vectorizer_config=wvc.config.Configure.Vectorizer.none(),
         properties=[wvc.config.Property(name="text", data_type=wvc.config.DataType.TEXT)],
@@ -249,15 +249,15 @@ def test_decide_without_a_decisions_module(collection_factory: CollectionFactory
 
     with pytest.raises(WeaviateQueryError):
         collection.query.bm25(
-            "x", decide=wvc.query.Decision.predicate("q", prop="text", instructions="i")
+            "x", annotate=wvc.query.Decision.predicate("q", prop="text", instructions="i")
         )
 
 
 # A 1x1 PNG, as a blob property holds it.
 PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNgYGAAAAAEAAH2FzhVAAAAAElFTkSuQmCC"
 
 
-def test_decide_and_rerank_about_an_image_property(collection_factory: CollectionFactory) -> None:
+def test_annotate_and_rerank_about_an_image_property(collection_factory: CollectionFactory) -> None:
     # decisions-dummy judges a blob as the text of its base64, so the answer is a
     # probability of 0 for an object with a photo and a refusal for an object without
     # one. A refused candidate stays in a rerank with a score of 0.
@@ -275,10 +275,10 @@ def test_decide_and_rerank_about_an_image_property(collection_factory: Collectio
 
     objs = collection.query.bm25(
         "here",
-        decide=wvc.query.Decision.predicate("damaged", prop="photo", instructions="damaged"),
+        annotate=wvc.query.Decision.predicate("damaged", prop="photo", instructions="damaged"),
     ).objects
 
-    by_text = {o.properties["text"]: o.decisions["damaged"] for o in objs if o.decisions}
+    by_text = {o.properties["text"]: o.annotations["damaged"] for o in objs if o.annotations}
     assert by_text["photo here"].probability == 0.0
     assert by_text["nothing here"].refused
 
@@ -289,3 +289,59 @@ def test_decide_and_rerank_about_an_image_property(collection_factory: Collectio
         "photo here": 0.0,
         "nothing here": 0.0,
     }
+
+
+def test_a_provider_named_by_the_query(collection_factory: CollectionFactory) -> None:
+    # The collection has no decisions module; the query names one, with a setting for
+    # this query only.
+    collection = collection_factory(
+        vectorizer_config=wvc.config.Configure.Vectorizer.none(),
+        properties=[wvc.config.Property(name="text", data_type=wvc.config.DataType.TEXT)],
+    )
+    if "decisions-dummy" not in collection._connection.get_meta()["modules"]:
+        pytest.skip("the server has no decisions-dummy module")
+    collection.data.insert_many([{"text": "the customer is angry"}, {"text": "all good"}])
+    dummy = wvc.query.DecisionsConfig.custom("decisions-dummy")
+
+    objs = collection.query.bm25(
+        "customer good",
+        annotate=wvc.query.Decision.predicate("angry", prop="text", instructions="customer angry"),
+        annotate_provider=dummy,
+    ).objects
+    assert {
+        o.properties["text"]: o.annotations["angry"].probability for o in objs if o.annotations
+    } == {
+        "the customer is angry": 1.0,
+        "all good": 0.0,
+    }
+
+    objs = collection.query.bm25(
+        "customer good",
+        rerank=wvc.query.Rerank.decide(
+            prop="text", query="customer angry", min_probability=0.6, provider=dummy
+        ),
+    ).objects
+    assert [o.properties["text"] for o in objs] == ["the customer is angry"]
+
+    # The request's maxDocuments bounds the page.
+    with pytest.raises(WeaviateQueryError, match="maxDocuments 1"):
+        collection.query.bm25(
+            "customer good",
+            limit=2,
+            annotate=wvc.query.Decision.predicate(
+                "angry", prop="text", instructions="customer angry"
+            ),
+            annotate_provider=wvc.query.DecisionsConfig.custom(
+                "decisions-dummy", {"maxDocuments": 1}
+            ),
+        )
+
+    # A module that is not enabled fails before the search.
+    with pytest.raises(WeaviateQueryError, match="not enabled"):
+        collection.query.bm25(
+            "customer good",
+            annotate=wvc.query.Decision.predicate(
+                "angry", prop="text", instructions="customer angry"
+            ),
+            annotate_provider=wvc.query.DecisionsConfig.custom("decisions-nope"),
+        )

integration/test_collection_decisions_openai.py

@@ -39,7 +39,7 @@ def test_decisions_openai(collection_factory: CollectionFactory) -> None:
     objs = collection.query.bm25(
         "refund arrived",
         rerank=wvc.query.Rerank(prop="text", query="The customer is unhappy"),
-        decide=[
+        annotate=[
             wvc.query.Decision.predicate(
                 "angry", prop="text", instructions="Is the customer angry?"
             ),
@@ -65,18 +65,18 @@ def test_decisions_openai(collection_factory: CollectionFactory) -> None:
     assert unhappy.metadata.rerank_score > happy.metadata.rerank_score
 
     for obj in objs:
-        assert obj.decisions is not None
-        assert set(obj.decisions) == {"angry", "team", "urgency"}
-        assert obj.decisions["team"].probabilities is not None
-        assert set(obj.decisions["team"].probabilities) == {"billing", "none"}
-        assert obj.decisions["urgency"].probabilities is not None
-        assert set(obj.decisions["urgency"].probabilities) == {"low", "medium", "high"}
-    assert unhappy.decisions is not None and happy.decisions is not None
-    assert unhappy.decisions["angry"].probability is not None
-    assert happy.decisions["angry"].probability is not None
-    assert unhappy.decisions["angry"].probability > happy.decisions["angry"].probability
-    assert unhappy.decisions["team"].choice == "billing"
-    assert happy.decisions["team"].choice == "none"
-    assert unhappy.decisions["urgency"].score is not None
-    assert happy.decisions["urgency"].score is not None
-    assert unhappy.decisions["urgency"].score > happy.decisions["urgency"].score
+        assert obj.annotations is not None
+        assert set(obj.annotations) == {"angry", "team", "urgency"}
+        assert obj.annotations["team"].probabilities is not None
+        assert set(obj.annotations["team"].probabilities) == {"billing", "none"}
+        assert obj.annotations["urgency"].probabilities is not None
+        assert set(obj.annotations["urgency"].probabilities) == {"low", "medium", "high"}
+    assert unhappy.annotations is not None and happy.annotations is not None
+    assert unhappy.annotations["angry"].probability is not None
+    assert happy.annotations["angry"].probability is not None
+    assert unhappy.annotations["angry"].probability > happy.annotations["angry"].probability
+    assert unhappy.annotations["team"].choice == "billing"
+    assert happy.annotations["team"].choice == "none"
+    assert unhappy.annotations["urgency"].score is not None
+    assert happy.annotations["urgency"].score is not None
+    assert unhappy.annotations["urgency"].score > happy.annotations["urgency"].score

mock_tests/conftest.py

@@ -258,7 +258,7 @@ def Search(
                     properties=search_get_pb2.PropertiesResult(
                         non_ref_props=properties_pb2.Properties(fields={"text": text})
                     ),
-                    decisions=decisions_pb2.DecisionResult(
+                    annotations=decisions_pb2.DecisionResult(
                         answers=[
                             decisions_pb2.DecisionAnswer(
                                 name="angry",