diff --git a/backend/infrahub/core/node/__init__.py b/backend/infrahub/core/node/__init__.py index 239983cd2ea..61182fbda2c 100644 --- a/backend/infrahub/core/node/__init__.py +++ b/backend/infrahub/core/node/__init__.py @@ -1106,10 +1106,14 @@ async def new_from_fields( profiles_mandatory_field_getter: ProfilesMandatoryFieldGetter, process_pools: bool = True, user_id: str = SYSTEM_USER_ID, + generated_id: str | None = None, ) -> Self: """Initialize the node from field values keyed by name, where an `id` key, when set, is the node's uuid. Keeping the field values in a dict lets a schema name a field like any parameter of this method. + Without an `id` key the node takes `generated_id`, a uuid the caller generated for it, which is not + checked against the database; without either, the node generates its uuid. `generated_id` is a parameter + and never a key of `fields`, so that no payload can set a uuid that is not checked. Raises: ValidationError: When the `id` is not a valid UUID or is already in use. @@ -1124,7 +1128,7 @@ async def new_from_fields( if await query.count(db=db): raise ValidationError({"id": f"{node_id} is already in use"}) - self.id = node_id or str(UUIDT()) + self.id = node_id or generated_id or str(UUIDT()) await self._process_fields( db=db, diff --git a/backend/infrahub/core/node/create.py b/backend/infrahub/core/node/create.py index 1b2d5e2ad1c..ed27484efa5 100644 --- a/backend/infrahub/core/node/create.py +++ b/backend/infrahub/core/node/create.py @@ -3,6 +3,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast, overload +from infrahub_sdk.uuidt import UUIDT + from infrahub import lock from infrahub.core import registry from infrahub.core.attribute import PayloadPresence @@ -18,7 +20,7 @@ from infrahub.core.constraint.node.runner import NodeConstraintRunner from infrahub.core.mutation_context import NodeMutationContext from infrahub.core.node import Node -from infrahub.core.node.lock_utils import get_lock_names_on_object_mutation +from infrahub.core.node.lock_utils import get_lock_names_on_object_mutation, get_pool_lock_names, ip_pool_lock_name from infrahub.core.protocols_base import CoreNode from infrahub.core.query.resource_manager import TrackingPool from infrahub.core.relationship.model import PeerWithRelationshipMetadata @@ -26,6 +28,7 @@ from infrahub.dependencies.registry import get_component_registry from infrahub.lock import InfrahubMultiLock from infrahub.pools.attribute_pool_applier_factory import build_attribute_pool_applier +from infrahub.pools.number_pool_locker import picking_lock_name from infrahub.pools.scope import AllocationScope from infrahub.profiles.data_reader import ProfileDataReader from infrahub.profiles.mandatory_fields_checker import ProfilesMandatoryFieldGetter @@ -33,7 +36,7 @@ from infrahub.templates.node_applier import get_relationship_names_to_read if TYPE_CHECKING: - from collections.abc import Iterable, Mapping + from collections.abc import Collection, Iterable, Mapping from infrahub.core.branch import Branch from infrahub.core.node import SchemaProtocol @@ -50,12 +53,14 @@ class TemplateComponent: """A subtemplate to materialize, the schema of the object it describes, and what it holds. - `components` is filled in once the level below this subtemplate has been read. + `components` is filled in once the level below this subtemplate has been read. `object_id` is the uuid of the + object to create, set before the object exists so that the locks of the create can name it. """ subtemplate: CoreObjectTemplate schema: NonGenericSchemaTypes components: list[TemplateComponent] = field(default_factory=list) + object_id: str = field(default_factory=lambda: str(UUIDT())) @dataclass @@ -307,6 +312,34 @@ async def allocate_from_resource_pools( ) +async def _new_component_node( + db: InfrahubDatabase, + branch: Branch, + component: TemplateComponent, + fields: Mapping[str, Any], + process_pools: bool, + at: Timestamp | None = None, +) -> Node: + """Build the unsaved object a subtemplate describes, with the uuid its component holds. + + Raises: + ValidationError: When the fields are not valid for the kind of the component. + + """ + node = await Node.init(schema=component.schema, db=db, branch=branch, at=at) + write_components = _build_node_write_components( + db=db, branch=branch, schema_branch=db.schema.get_schema_branch(name=branch.name) + ) + return await node.new_from_fields( + db=db, + fields=dict(fields), + pool_applier=write_components.pool_applier, + profiles_mandatory_field_getter=write_components.profiles_mandatory_field_getter, + process_pools=process_pools, + generated_id=component.object_id, + ) + + async def create_component( db: InfrahubDatabase, branch: Branch, @@ -327,8 +360,9 @@ async def create_component( current_template=parent_template, ) - obj_peer = await Node.init(schema=component.schema, db=db, branch=branch, at=at) - await obj_peer.new(db=db, **obj_peer_data) + obj_peer = await _new_component_node( + db=db, branch=branch, component=component, fields=obj_peer_data, process_pools=True, at=at + ) await constraint_runner.check( node=obj_peer, field_filters=_with_pooled_attributes(node=obj_peer, fields=list(obj_peer_data)) ) @@ -399,42 +433,170 @@ async def create_components( ) -async def handle_template_relationships( +def _has_number_pool_attribute(schema: MainSchemaTypes) -> bool: + return any(attribute.kind == "NumberPool" for attribute in schema.attributes) + + +async def _template_pools( + db: InfrahubDatabase, template: CoreObjectTemplate, skipped_fields: Collection[str] +) -> dict[str, Node]: + """Return the pool the template names for each field of the object, except for the fields in `skipped_fields`. + + The template was read with these relationships and their peers, so naming the pools costs no query. + """ + pools: dict[str, Node] = {} + for rel_name in template.get_schema().relationship_names: + field_name = rel_name.removesuffix(RESOURCE_POOL_REL_SUFFIX) + if not rel_name.endswith(RESOURCE_POOL_REL_SUFFIX) or field_name in skipped_fields: + continue + pool = await template.get_relationship(name=rel_name).get_peer(db=db) + if pool is not None: + pools[field_name] = pool + return pools + + +async def _template_pool_lock_names( + node: Node, pools: Mapping[str, Node], pool_applier: AttributePoolApplierInterface, user_id: str +) -> set[str]: + """Return the locks the unsaved `node` takes to draw from `pools`, the pool its template names for each field. + + Raises: + ValidationError: When the schema of the branch does not define an element of a pool's scope. + NodeNotFoundError: When a relationship element of a pool's scope names a peer that does not exist. + + """ + schema = node.get_schema() + lock_names: set[str] = set() + for field_name, pool in pools.items(): + if field_name in schema.relationship_names: + lock_names.add(ip_pool_lock_name(pool_id=pool.get_id())) + continue + if field_name not in schema.attribute_names: + continue + attribute = node.get_attribute(name=field_name) + if attribute.from_pool: + # An object names a scoped pool on the attribute itself, so its own lock names hold this one. + continue + if AllocationScope.from_stored(value=pool.get_attribute("allocation_scope").value, pool=pool.get_id()).is_empty: + lock_names.add(picking_lock_name(pool_id=pool.get_id(), division_key=None)) + continue + attribute.from_pool = {"id": pool.get_id()} + attribute.from_pool_presence = PayloadPresence.SET + await pool_applier.apply(node=node, attribute=attribute, allocate=False, user_id=user_id) + lock_names.add(picking_lock_name(pool_id=pool.get_id(), division_key=attribute.from_pool.get("division"))) + return lock_names + + +@dataclass +class _ComponentPreview: + """An unsaved object built from a subtemplate as the component will be, and the pools the subtemplate names.""" + + node: Node + pools: dict[str, Node] + + +async def _component_previews( + db: InfrahubDatabase, + branch: Branch, + parent_obj: Node, + parent_template: CoreObjectTemplate, + components: list[TemplateComponent], + pooled_subtemplate_ids: set[str], +) -> list[_ComponentPreview]: + """Return a preview of each component that draws from a pool, and of each component holding one that does. + + A preview draws nothing: it holds the fields its division of a scoped number pool is read from, and it stands + for the parent of the components below it. + """ + previews: list[_ComponentPreview] = [] + for component in components: + if component.subtemplate.get_id() not in pooled_subtemplate_ids: + continue + obj_peer_data = await extract_peer_data( + db=db, + template_peer=component.subtemplate, + obj_peer_schema=component.schema, + parent_obj=parent_obj, + current_template=parent_template, + ) + node = await _new_component_node( + db=db, branch=branch, component=component, fields=obj_peer_data, process_pools=False + ) + previews.append( + _ComponentPreview( + node=node, + pools=await _template_pools(db=db, template=component.subtemplate, skipped_fields=()), + ) + ) + previews.extend( + await _component_previews( + db=db, + branch=branch, + parent_obj=node, + parent_template=component.subtemplate, + components=component.components, + pooled_subtemplate_ids=pooled_subtemplate_ids, + ) + ) + return previews + + +async def _pooled_subtemplate_ids(db: InfrahubDatabase, components: list[TemplateComponent]) -> set[str]: + """Return the id of each subtemplate whose component draws from a pool, or holds a component that does.""" + pooled: set[str] = set() + for component in components: + below = await _pooled_subtemplate_ids(db=db, components=component.components) + if ( + below + or _has_number_pool_attribute(schema=component.schema) + or await _template_pools(db=db, template=component.subtemplate, skipped_fields=()) + ): + pooled.add(component.subtemplate.get_id()) + pooled |= below + return pooled + + +async def get_template_pool_lock_names( db: InfrahubDatabase, branch: Branch, obj: Node, template: CoreObjectTemplate, - fields: list, + components: list[TemplateComponent], + user_fields: Collection[str], pool_applier: AttributePoolApplierInterface, - at: Timestamp | None = None, user_id: str = SYSTEM_USER_ID, -) -> None: - """Create the objects the components of a template describe. +) -> set[str]: + """Return the pool locks a create from a template asks for beyond those of the object's own fields. + + `obj` is the unsaved preview of the object created from `template`, and `user_fields` names the fields the + payload sets, which the template leaves alone. The locks are those of the pools the template names for the + object without a scope, and of every pool its components draw from. + + Raises: + ValidationError: When the schema of the branch does not define an element of a pool's scope. + NodeNotFoundError: When a relationship element of a pool's scope names a peer that does not exist. - The template is read a level at a time and the objects are created depth first. Those are two - different walks on purpose: reading a whole level at once costs one read per level rather than - one per parent, while creating a component together with the components it holds keeps the - order in which a resource pool shared by several levels is drawn from. """ - components = await read_template_components( - db=db, branch=branch, schema=obj.get_schema(), template=template, fields=fields + lock_names = await _template_pool_lock_names( + node=obj, + pools=await _template_pools(db=db, template=template, skipped_fields=user_fields), + pool_applier=pool_applier, + user_id=user_id, ) - if not components: - return - - component_registry = get_component_registry() - constraint_runner = await component_registry.get_component(NodeConstraintRunner, db=db, branch=branch) - await create_components( + previews = await _component_previews( db=db, branch=branch, - constraint_runner=constraint_runner, parent_obj=obj, parent_template=template, components=components, - pool_applier=pool_applier, - at=at, - user_id=user_id, + pooled_subtemplate_ids=await _pooled_subtemplate_ids(db=db, components=components), ) + for preview in previews: + lock_names |= await _template_pool_lock_names( + node=preview.node, pools=preview.pools, pool_applier=pool_applier, user_id=user_id + ) + lock_names |= get_pool_lock_names(node=preview.node) + return lock_names async def get_profile_ids(db: InfrahubDatabase, obj: Node | CoreObjectTemplate) -> set[str]: @@ -510,9 +672,11 @@ async def _do_create_node( branch: Branch, fields_to_validate: list[str], data: dict[str, Any], + object_template: CoreObjectTemplate | None, + template_components: list[TemplateComponent], + generated_id: str | None, at: Timestamp | None = None, user_id: str = SYSTEM_USER_ID, - object_template: CoreObjectTemplate | None = None, ) -> Node: with mutation_context: obj = await node_class.init(db=db, schema=schema, branch=branch) @@ -523,6 +687,7 @@ async def _do_create_node( pool_applier=pool_applier, profiles_mandatory_field_getter=profiles_mandatory_field_getter, user_id=user_id, + generated_id=generated_id, ) await node_constraint_runner.check( node=obj, field_filters=_with_pooled_attributes(node=obj, fields=fields_to_validate) @@ -530,12 +695,13 @@ async def _do_create_node( await obj.save(db=db, at=at, user_id=user_id) if object_template: - await handle_template_relationships( + await create_components( db=db, branch=branch, - template=object_template, - obj=obj, - fields=fields_to_validate, + constraint_runner=node_constraint_runner, + parent_obj=obj, + parent_template=object_template, + components=template_components, pool_applier=pool_applier, at=at, user_id=user_id, @@ -603,6 +769,8 @@ class that is not a node schema protocol is passed. # gets components of its own. schema_branch = db.schema.get_schema_branch(name=branch.name) components = _build_node_write_components(db=db, branch=branch, schema_branch=schema_branch) + # The preview and the node created under the lock share one uuid, so the lock names hold the uuid of the node. + generated_id = None if data.get("id") else str(UUIDT()) preview_obj = await node_class.init(db=db, schema=schema, branch=branch) await preview_obj.new_from_fields( db=db, @@ -610,11 +778,30 @@ class that is not a node schema protocol is passed. pool_applier=components.pool_applier, profiles_mandatory_field_getter=components.profiles_mandatory_field_getter, process_pools=False, + generated_id=generated_id, ) lock_names = get_lock_names_on_object_mutation(node=preview_obj, schema_branch=schema_branch) # The preview read the object template to work out the lock names; the node created under the # lock is built from that same template rather than reading it again. object_template = preview_obj._object_template + template_components: list[TemplateComponent] = [] + if object_template is not None: + template_components = await read_template_components( + db=db, branch=branch, schema=schema, template=object_template, fields=fields_to_validate + ) + # Waiting for a pool's lock inside the transaction can leave this write and another allocation from the + # pool waiting on each other. + component_lock_names = await get_template_pool_lock_names( + db=db, + branch=branch, + obj=preview_obj, + template=object_template, + components=template_components, + user_fields=fields_to_validate, + pool_applier=components.pool_applier, + user_id=user_id, + ) + lock_names = sorted({*lock_names, *component_lock_names}) obj: Node mutation_context = NodeMutationContext() @@ -636,6 +823,8 @@ class that is not a node schema protocol is passed. at=at, user_id=user_id, object_template=object_template, + template_components=template_components, + generated_id=generated_id, ) else: async with db.start_transaction() as dbt: @@ -660,6 +849,8 @@ class that is not a node schema protocol is passed. at=at, user_id=user_id, object_template=object_template, + template_components=template_components, + generated_id=generated_id, ) # The node was written from its in-memory state, so the profiles it is linked to are the ones diff --git a/backend/infrahub/core/node/lock_utils.py b/backend/infrahub/core/node/lock_utils.py index 7a680f73410..1466755cd3b 100644 --- a/backend/infrahub/core/node/lock_utils.py +++ b/backend/infrahub/core/node/lock_utils.py @@ -99,27 +99,38 @@ def _attribute_pool_lock_name(from_pool: dict[str, Any]) -> str: return picking_lock_name(pool_id=from_pool["id"], division_key=from_pool.get("division")) +def get_pool_lock_names(node: Node) -> set[str]: + """Return the locks the node takes to draw from the resource pools its attributes and relationships name.""" + lock_names: set[str] = set() + for attr_name in node.get_schema().attribute_names: + attribute = getattr(node, attr_name, None) + if attribute is not None and getattr(attribute, "from_pool", None) and "id" in attribute.from_pool: + lock_names.add(_attribute_pool_lock_name(from_pool=attribute.from_pool)) + + for rel_name in node._relationships: + for rel in node.get_relationship(rel_name)._relationships: + if rel.from_pool and "id" in rel.from_pool: + lock_names.add(ip_pool_lock_name(pool_id=rel.from_pool["id"])) + return lock_names + + +def ip_pool_lock_name(pool_id: str) -> str: + """Return the name of the lock that allocations from the same IP address or IP prefix pool take.""" + return f"{RESOURCE_POOL_LOCK_NAMESPACE}.{pool_id}" + + def get_lock_names_on_object_mutation(node: Node, schema_branch: SchemaBranch) -> list[str]: """Return lock names for object on which we want to avoid concurrent mutation (create/update). Lock names include kind, some generic kinds, resource pool ids, peer ids for cardinality one relationships, and values of attributes of corresponding uniqueness constraints. """ - lock_names: set[str] = set() - - # Check if node is using resource manager allocation via attributes - for attr_name in node.get_schema().attribute_names: - attribute = getattr(node, attr_name, None) - if attribute is not None and getattr(attribute, "from_pool", None) and "id" in attribute.from_pool: - lock_names.add(_attribute_pool_lock_name(from_pool=attribute.from_pool)) + lock_names = get_pool_lock_names(node=node) - # Check if relationships allocate resources or have cardinality one constraint + # Check if relationships have cardinality one constraint for rel_name in node._relationships: rel_manager = node.get_relationship(rel_name) for rel in rel_manager._relationships: - if rel.from_pool and "id" in rel.from_pool: - lock_names.add(f"{RESOURCE_POOL_LOCK_NAMESPACE}.{rel.from_pool['id']}") - peer_id = rel.peer_id if not peer_id or not rel.schema.identifier: continue diff --git a/backend/tests/component/core/node/test_template_pool_lock_order.py b/backend/tests/component/core/node/test_template_pool_lock_order.py new file mode 100644 index 00000000000..0cce1543b74 --- /dev/null +++ b/backend/tests/component/core/node/test_template_pool_lock_order.py @@ -0,0 +1,515 @@ +from __future__ import annotations + +import asyncio +from copy import deepcopy +from dataclasses import dataclass +from enum import StrEnum +from typing import TYPE_CHECKING, Any + +import pytest +from infrahub_sdk.uuidt import UUIDT + +from infrahub import lock +from infrahub.core import registry +from infrahub.core.constants import InfrahubKind, RelationshipCardinality, RelationshipKind +from infrahub.core.manager import NodeManager +from infrahub.core.node import Node +from infrahub.core.node.create import create_node +from infrahub.core.node.resource_manager.ip_address_pool import CoreIPAddressPool +from infrahub.core.node.resource_manager.ip_prefix_pool import CoreIPPrefixPool +from infrahub.core.node.resource_manager.number_pool import CoreNumberPool +from infrahub.core.query.node import NodeCheckIDQuery +from infrahub.core.schema import AttributeSchema, NodeSchema, RelationshipSchema, SchemaRoot +from infrahub.core.schema.attribute_parameters import NumberPoolParameters +from tests.adapters.lock.timeline import LockAction +from tests.helpers.agnostic_edges import IsReservedEdge, is_reserved_edge_on +from tests.helpers.constants import LOCK_WAIT_TIMEOUT_SECONDS +from tests.helpers.db_query_counter import CountingInfrahubDatabase +from tests.helpers.db_transaction_tracker import TransactionTrackingInfrahubDatabase +from tests.helpers.ip_pool import create_namespace, create_prefix +from tests.helpers.lock import wait_until_waiting +from tests.helpers.number_pool import ( + SCOPED_CARD, + SCOPED_CHASSIS, + SCOPED_SITE, + add_pool_range, + pool_lock_events, + provision_schema_number_pools, + save_schema_kinds, + scoped_site, + stored_scope, +) + +if TYPE_CHECKING: + from infrahub.core.branch import Branch + from infrahub.core.schema.schema_branch import SchemaBranch + from infrahub.database import InfrahubDatabase + from tests.adapters.lock import RecordingLockRegistry + +MODULE_KIND = "ScopeModule" +ADDRESS_KIND = "IpamIPAddress" +PREFIX_KIND = "IpamIPPrefix" +CHASSIS_SCOPED_POOL = "card-scoped-by-chassis" + + +class Level(StrEnum): + OBJECT = "object" + COMPONENT = "component" + NESTED_COMPONENT = "nested-component" + + +LEVEL_KINDS = { + Level.OBJECT: SCOPED_CHASSIS.kind, + Level.COMPONENT: SCOPED_CARD.kind, + Level.NESTED_COMPONENT: MODULE_KIND, +} + + +class PoolSource(StrEnum): + TEMPLATE = "template" + """The template of the level names the pool with a `_from_resource_pool` relationship.""" + SCHEMA = "schema" + """The schema declares the pool with a NumberPool attribute.""" + + +class PoolType(StrEnum): + NUMBER = "number" + SCOPED_NUMBER = "scoped-number" + ADDRESS = "address" + PREFIX = "prefix" + + +POOLED_FIELDS = { + (PoolSource.TEMPLATE, PoolType.NUMBER): "vlan_id", + (PoolSource.TEMPLATE, PoolType.SCOPED_NUMBER): "vlan_id", + (PoolSource.SCHEMA, PoolType.NUMBER): "serial", + (PoolSource.SCHEMA, PoolType.SCOPED_NUMBER): "site_serial", + (PoolSource.TEMPLATE, PoolType.ADDRESS): "address", + (PoolSource.TEMPLATE, PoolType.PREFIX): "prefix", +} + +EXPECTED_IP_VALUES = { + (Level.OBJECT, PoolType.ADDRESS): "10.1.0.1/24", + (Level.COMPONENT, PoolType.ADDRESS): "10.2.0.1/24", + (Level.OBJECT, PoolType.PREFIX): "10.3.0.0/28", + (Level.COMPONENT, PoolType.PREFIX): "10.4.0.0/28", +} +IP_POOL_RESOURCES = { + (Level.OBJECT, PoolType.ADDRESS): "10.1.0.0/24", + (Level.COMPONENT, PoolType.ADDRESS): "10.2.0.0/24", + (Level.OBJECT, PoolType.PREFIX): "10.3.0.0/24", + (Level.COMPONENT, PoolType.PREFIX): "10.4.0.0/24", +} + + +@dataclass(frozen=True) +class PoolLockCase: + """A pool drawn from by the object created from a template, or by one of its components.""" + + level: Level + source: PoolSource + pool_type: PoolType + + @property + def name(self) -> str: + return f"{self.level}-{self.source}-{self.pool_type}" + + @property + def field(self) -> str: + return POOLED_FIELDS[self.source, self.pool_type] + + +POOL_LOCK_CASES = [ + PoolLockCase(level=level, source=source, pool_type=pool_type) + for level in (Level.OBJECT, Level.COMPONENT) + for source, pool_type in POOLED_FIELDS +] + [PoolLockCase(level=Level.NESTED_COMPONENT, source=PoolSource.TEMPLATE, pool_type=PoolType.SCOPED_NUMBER)] + + +@dataclass +class PoolLockDataset: + """A chassis holding cards holding modules, each in a site, and one pool for each case. + + Every kind has the fields of `POOLED_FIELDS` except the module, which has only `vlan_id`. + """ + + branch: Branch + site: Node + pools: dict[str, Node] + + +def _pooled_fields() -> tuple[list[AttributeSchema], list[RelationshipSchema]]: + attributes = [ + AttributeSchema( + name="serial", + kind="NumberPool", + optional=False, + read_only=True, + parameters=NumberPoolParameters(start_range=1, end_range=100), + ), + AttributeSchema( + name="site_serial", + kind="NumberPool", + optional=False, + read_only=True, + parameters=NumberPoolParameters(start_range=1, end_range=100, allocation_scope=["site"]), + ), + ] + relationships = [ + RelationshipSchema( + name="address", + peer=ADDRESS_KIND, + kind=RelationshipKind.ATTRIBUTE, + cardinality=RelationshipCardinality.ONE, + optional=True, + ), + RelationshipSchema( + name="prefix", + peer=PREFIX_KIND, + kind=RelationshipKind.ATTRIBUTE, + cardinality=RelationshipCardinality.ONE, + optional=True, + ), + ] + return attributes, relationships + + +def _site_relationship(identifier: str) -> RelationshipSchema: + return RelationshipSchema( + name="site", + peer=SCOPED_SITE.kind, + identifier=identifier, + kind=RelationshipKind.ATTRIBUTE, + cardinality=RelationshipCardinality.ONE, + optional=False, + ) + + +def pool_lock_schema() -> SchemaRoot: + """The chassis and card kinds with every pooled field, a site on each, and a module held by the card.""" + chassis = deepcopy(SCOPED_CHASSIS) + card = deepcopy(SCOPED_CARD) + chassis.attributes.append(AttributeSchema(name="vlan_id", kind="Number", optional=True)) + for kind, site_identifier in ((chassis, "scope_chassis__site"), (card, "scope_card__site")): + attributes, relationships = _pooled_fields() + kind.attributes.extend(attributes) + kind.relationships.extend([*relationships, _site_relationship(identifier=site_identifier)]) + card.relationships.append( + RelationshipSchema( + name="modules", + peer=MODULE_KIND, + identifier="scope_card__modules", + kind=RelationshipKind.COMPONENT, + cardinality=RelationshipCardinality.MANY, + optional=True, + ) + ) + module = NodeSchema( + name="Module", + namespace="Scope", + generate_template=True, + attributes=[ + AttributeSchema(name="name", kind="Text"), + AttributeSchema(name="vlan_id", kind="Number", optional=True), + ], + relationships=[ + RelationshipSchema( + name="card", + peer=SCOPED_CARD.kind, + identifier="scope_card__modules", + kind=RelationshipKind.PARENT, + cardinality=RelationshipCardinality.ONE, + optional=False, + ), + _site_relationship(identifier="scope_module__site"), + ], + ) + return SchemaRoot(nodes=[deepcopy(SCOPED_SITE), chassis, card, module]) + + +async def _number_pool(db: InfrahubDatabase, kind: str, scope: str | None) -> Node: + pool = await CoreNumberPool.init(db=db, schema=InfrahubKind.NUMBERPOOL) + await pool.new( + db=db, + name=f"{kind}-{scope or 'unscoped'}", + node=kind, + node_attribute="vlan_id", + start_range=1, + end_range=100, + allocation_scope=stored_scope(names=[scope], kind=kind) if scope else None, + ) + await pool.save(db=db) + await add_pool_range(db=db, pool=pool, start=1, end=100) + return pool + + +async def _ip_pool(db: InfrahubDatabase, namespace: Node, name: str, pool_type: PoolType, resource: str) -> Node: + prefix = await create_prefix(db=db, namespace=namespace, value=resource) + pool: CoreIPAddressPool | CoreIPPrefixPool + if pool_type is PoolType.ADDRESS: + pool = await CoreIPAddressPool.init(db=db, schema=InfrahubKind.IPADDRESSPOOL) + await pool.new(db=db, name=name, resources=[prefix], ip_namespace=namespace, default_address_type=ADDRESS_KIND) + else: + pool = await CoreIPPrefixPool.init(db=db, schema=InfrahubKind.IPPREFIXPOOL) + await pool.new( + db=db, + name=name, + resources=[prefix], + ip_namespace=namespace, + default_prefix_type=PREFIX_KIND, + default_prefix_length=28, + ) + await pool.save(db=db) + return pool + + +async def _schema_pool(db: InfrahubDatabase, branch: Branch, kind: str, attribute_name: str) -> Node: + parameters = registry.schema.get_node_schema(name=kind, branch=branch).get_attribute(name=attribute_name).parameters + assert isinstance(parameters, NumberPoolParameters) + assert parameters.number_pool_id is not None + return await NodeManager.get_one(db=db, id=parameters.number_pool_id, branch=branch, raise_on_error=True) + + +async def _hold(name: str, held: asyncio.Event, release: asyncio.Event) -> None: + async with lock.registry.get(name=name): + held.set() + await release.wait() + + +class TestPoolLocksOfACreateFromATemplate: + """Each test creates a chassis from a template while another task holds the lock of one pool it draws from.""" + + @pytest.fixture(scope="class") + async def dataset( + self, + db: InfrahubDatabase, + default_branch_scope_class: Branch, + register_core_models_schema_scope_class: SchemaBranch, + register_ipam_schema_scope_class: SchemaBranch, + init_nodes_registry_scope_class: None, + ) -> PoolLockDataset: + branch = default_branch_scope_class + await save_schema_kinds(db=db, schema=pool_lock_schema()) + await provision_schema_number_pools(db=db) + site = await scoped_site(db=db, branch=branch, name="site-a") + namespace = await create_namespace(db=db) + pools: dict[str, Node] = {} + for case in POOL_LOCK_CASES: + kind = LEVEL_KINDS[case.level] + if case.source is PoolSource.SCHEMA: + pools[case.name] = await _schema_pool(db=db, branch=branch, kind=kind, attribute_name=case.field) + elif case.pool_type in {PoolType.NUMBER, PoolType.SCOPED_NUMBER}: + pools[case.name] = await _number_pool( + db=db, kind=kind, scope="site" if case.pool_type is PoolType.SCOPED_NUMBER else None + ) + else: + pools[case.name] = await _ip_pool( + db=db, + namespace=namespace, + name=case.name, + pool_type=case.pool_type, + resource=IP_POOL_RESOURCES[case.level, case.pool_type], + ) + pools[CHASSIS_SCOPED_POOL] = await _number_pool(db=db, kind=SCOPED_CARD.kind, scope="chassis") + return PoolLockDataset(branch=branch, site=site, pools=pools) + + async def _template(self, db: InfrahubDatabase, dataset: PoolLockDataset, case: PoolLockCase) -> Node: + """A chassis template with a card and a module below it, where the level of the case names its pool.""" + levels = [Level.OBJECT, Level.COMPONENT, Level.NESTED_COMPONENT] + parent_fields = {Level.COMPONENT: "chassis", Level.NESTED_COMPONENT: "card"} + templates: dict[Level, Node] = {} + for index, level in enumerate(levels[: levels.index(case.level) + 1]): + fields: dict[str, Any] = {"template_name": f"{level}-of-{case.name}", "site": dataset.site} + if level is not Level.OBJECT: + fields["name"] = f"{level}-of-{case.name}" + fields[parent_fields[level]] = templates[levels[index - 1]].id + if level is case.level and case.source is PoolSource.TEMPLATE: + fields[f"{case.field}_from_resource_pool"] = {"id": dataset.pools[case.name].id} + template = await Node.init(db=db, schema=f"Template{LEVEL_KINDS[level]}", branch=dataset.branch) + await template.new(db=db, **fields) + await template.save(db=db) + templates[level] = template + return templates[Level.OBJECT] + + def _lock_name(self, dataset: PoolLockDataset, case: PoolLockCase) -> str: + pool_id = dataset.pools[case.name].id + if case.pool_type is PoolType.NUMBER: + return f"resource_pool.{pool_id}.all" + if case.pool_type is PoolType.SCOPED_NUMBER: + return f"resource_pool.{pool_id}.{dataset.site.id}" + return f"resource_pool.{pool_id}" + + async def _created_at_level( + self, db: InfrahubDatabase, dataset: PoolLockDataset, chassis: Node, level: Level + ) -> Node: + if level is Level.OBJECT: + return chassis + cards = await NodeManager.query( + db=db, schema=SCOPED_CARD.kind, branch=dataset.branch, filters={"chassis__ids": [chassis.id]} + ) + assert len(cards) == 1 + if level is Level.COMPONENT: + return cards[0] + modules = await NodeManager.query( + db=db, schema=MODULE_KIND, branch=dataset.branch, filters={"card__ids": [cards[0].id]} + ) + assert len(modules) == 1 + return modules[0] + + @pytest.mark.parametrize("case", POOL_LOCK_CASES, ids=[case.name for case in POOL_LOCK_CASES]) + async def test_the_create_waits_for_the_lock_of_the_pool_before_its_transaction( + self, + db: InfrahubDatabase, + dataset: PoolLockDataset, + recording_lock_registry: RecordingLockRegistry, + case: PoolLockCase, + ) -> None: + template = await self._template(db=db, dataset=dataset, case=case) + lock_name = self._lock_name(dataset=dataset, case=case) + tracking_db = TransactionTrackingInfrahubDatabase.from_db(db=db) + held = asyncio.Event() + release = asyncio.Event() + holder = asyncio.create_task(_hold(name=lock_name, held=held, release=release)) + await asyncio.wait_for(held.wait(), timeout=LOCK_WAIT_TIMEOUT_SECONDS) + + async with tracking_db.start_session() as session_db: + write = asyncio.create_task( + create_node( + data={"name": {"value": f"chassis-of-{case.name}"}, "object_template": {"id": template.id}}, + db=session_db, + branch=dataset.branch, + schema=session_db.schema.get(name=SCOPED_CHASSIS.kind, branch=dataset.branch, duplicate=False), + ) + ) + try: + await wait_until_waiting(timeline=recording_lock_registry.timeline, name=lock_name) + open_transactions_while_waiting = tracking_db.transaction_usage.open_transactions + finally: + release.set() + await asyncio.wait_for(holder, timeout=LOCK_WAIT_TIMEOUT_SECONDS) + chassis = await asyncio.wait_for(write, timeout=LOCK_WAIT_TIMEOUT_SECONDS) + + assert open_transactions_while_waiting == 0 + created = await self._created_at_level(db=db, dataset=dataset, chassis=chassis, level=case.level) + if case.pool_type in {PoolType.ADDRESS, PoolType.PREFIX}: + peer = await created.get_relationship(case.field).get_peer(db=db, raise_on_error=True) + assert peer.get_attribute(case.field).value == EXPECTED_IP_VALUES[case.level, case.pool_type] + else: + assert ( + await is_reserved_edge_on( + db=db, pool_id=dataset.pools[case.name].id, node_id=created.id, attribute_name=case.field + ) + == IsReservedEdge.OPEN + ) + + async def test_a_division_holding_an_object_whose_id_the_payload_sets_is_locked_before_the_transaction( + self, db: InfrahubDatabase, dataset: PoolLockDataset, recording_lock_registry: RecordingLockRegistry + ) -> None: + """Another write can name the chassis by the id the payload sets, so the division of its card is contended.""" + pool = dataset.pools[CHASSIS_SCOPED_POOL] + template = await Node.init(db=db, schema=f"Template{SCOPED_CHASSIS.kind}", branch=dataset.branch) + await template.new(db=db, template_name="chassis-with-an-id", site=dataset.site) + await template.save(db=db) + subtemplate = await Node.init(db=db, schema=f"Template{SCOPED_CARD.kind}", branch=dataset.branch) + await subtemplate.new( + db=db, + template_name="card-of-chassis-with-an-id", + name="card-of-chassis-with-an-id", + chassis=template.id, + site=dataset.site, + vlan_id_from_resource_pool={"id": pool.id}, + ) + await subtemplate.save(db=db) + chassis_id = str(UUIDT()) + lock_name = f"resource_pool.{pool.id}.{chassis_id}" + tracking_db = TransactionTrackingInfrahubDatabase.from_db(db=db) + held = asyncio.Event() + release = asyncio.Event() + holder = asyncio.create_task(_hold(name=lock_name, held=held, release=release)) + await asyncio.wait_for(held.wait(), timeout=LOCK_WAIT_TIMEOUT_SECONDS) + + async with tracking_db.start_session() as session_db: + write = asyncio.create_task( + create_node( + data={ + "id": chassis_id, + "name": {"value": "chassis-with-an-id"}, + "object_template": {"id": template.id}, + }, + db=session_db, + branch=dataset.branch, + schema=session_db.schema.get(name=SCOPED_CHASSIS.kind, branch=dataset.branch, duplicate=False), + ) + ) + try: + await wait_until_waiting(timeline=recording_lock_registry.timeline, name=lock_name) + open_transactions_while_waiting = tracking_db.transaction_usage.open_transactions + finally: + release.set() + await asyncio.wait_for(holder, timeout=LOCK_WAIT_TIMEOUT_SECONDS) + chassis = await asyncio.wait_for(write, timeout=LOCK_WAIT_TIMEOUT_SECONDS) + + assert open_transactions_while_waiting == 0 + assert chassis.id == chassis_id + card = await self._created_at_level(db=db, dataset=dataset, chassis=chassis, level=Level.COMPONENT) + assert ( + await is_reserved_edge_on(db=db, pool_id=pool.id, node_id=card.id, attribute_name="vlan_id") + == IsReservedEdge.OPEN + ) + + async def test_a_division_holding_the_object_created_is_locked_once_for_all_its_components( + self, db: InfrahubDatabase, dataset: PoolLockDataset, recording_lock_registry: RecordingLockRegistry + ) -> None: + """Two cards draw in the division of their new chassis, whose lock the create takes once and keeps.""" + pool = dataset.pools[CHASSIS_SCOPED_POOL] + template = await Node.init(db=db, schema=f"Template{SCOPED_CHASSIS.kind}", branch=dataset.branch) + await template.new(db=db, template_name="chassis-with-two-cards", site=dataset.site) + await template.save(db=db) + for index in (1, 2): + subtemplate = await Node.init(db=db, schema=f"Template{SCOPED_CARD.kind}", branch=dataset.branch) + await subtemplate.new( + db=db, + template_name=f"card-{index}-of-chassis-with-two-cards", + name=f"card-{index}", + chassis=template.id, + site=dataset.site, + vlan_id_from_resource_pool={"id": pool.id}, + ) + await subtemplate.save(db=db) + timeline = recording_lock_registry.timeline + start = timeline.checkpoint("write") + + chassis = await create_node( + data={"name": {"value": "chassis-with-two-cards"}, "object_template": {"id": template.id}}, + db=db, + branch=dataset.branch, + schema=db.schema.get(name=SCOPED_CHASSIS.kind, branch=dataset.branch, duplicate=False), + ) + + lock_name = f"resource_pool.{pool.id}.{chassis.id}" + assert pool_lock_events(timeline=timeline, pool=pool, after=start) == [ + (lock_name, LockAction.ACQUIRE), + (lock_name, LockAction.RELEASE), + ] + + async def test_the_locks_of_a_create_name_the_object_created_without_checking_its_id( + self, db: InfrahubDatabase, dataset: PoolLockDataset, recording_lock_registry: RecordingLockRegistry + ) -> None: + template = await Node.init(db=db, schema=f"Template{SCOPED_CHASSIS.kind}", branch=dataset.branch) + await template.new(db=db, template_name="chassis-without-an-id", site=dataset.site) + await template.save(db=db) + counting_db = CountingInfrahubDatabase.from_db(db=db) + + async with counting_db.start_session() as session_db: + chassis = await create_node( + data={"name": {"value": "chassis-without-an-id"}, "object_template": {"id": template.id}}, + db=session_db, + branch=dataset.branch, + schema=session_db.schema.get(name=SCOPED_CHASSIS.kind, branch=dataset.branch, duplicate=False), + ) + + site_lock_prefix = "relationship_count.scope_chassis__site." + assert recording_lock_registry.timeline.acquire_sequence(prefix=site_lock_prefix) == [ + f"{site_lock_prefix}{chassis.id}" + ] + assert counting_db.count_for(NodeCheckIDQuery.name) == 0 diff --git a/backend/tests/component/core/resource_manager/test_ipaddress_pool.py b/backend/tests/component/core/resource_manager/test_ipaddress_pool.py index bbfb1dbfafd..d9c84cab247 100644 --- a/backend/tests/component/core/resource_manager/test_ipaddress_pool.py +++ b/backend/tests/component/core/resource_manager/test_ipaddress_pool.py @@ -215,6 +215,20 @@ async def test_get_resource_address_type_overrides_pool_default( assert node.get_kind() == "TestIPAddress" +async def test_get_resource_data_cannot_set_the_id_of_the_address_without_a_check( + db: InfrahubDatabase, default_branch: Branch, kind_override_address_pool: CoreIPAddressPool +) -> None: + """Only `id` in the data names the uuid of the new address, and that one is checked against the database.""" + node = await kind_override_address_pool.get_resource( + db=db, + branch=default_branch, + peer_kind=InfrahubKind.IPADDRESS, + data={"generated_id": kind_override_address_pool.id}, + ) + + assert node.id != kind_override_address_pool.id + + async def test_get_resource_address_type_falls_back_to_pool_default( db: InfrahubDatabase, default_branch: Branch, kind_override_address_pool: CoreIPAddressPool ) -> None: diff --git a/backend/tests/component/core/resource_manager/test_number_pool_scope_allocation.py b/backend/tests/component/core/resource_manager/test_number_pool_scope_allocation.py index 66c02b714f7..525f82762b0 100644 --- a/backend/tests/component/core/resource_manager/test_number_pool_scope_allocation.py +++ b/backend/tests/component/core/resource_manager/test_number_pool_scope_allocation.py @@ -22,6 +22,7 @@ from tests.adapters.lock.timeline import LockAction from tests.helpers.agnostic_edges import IsReservedEdge, is_reserved_edge_on from tests.helpers.constants import LOCK_WAIT_TIMEOUT_SECONDS +from tests.helpers.lock import wait_until_waiting from tests.helpers.number_pool import ( SCOPED_ATTRIBUTE_NAME, SCOPED_CARD, @@ -65,17 +66,21 @@ def device_payload(name: str, site: Node, pool: CoreNumberPool, role: str | None return payload -async def create_device_in_own_session(db: InfrahubDatabase, branch: Branch, payload: dict[str, Any]) -> Node: +async def create_in_own_session(db: InfrahubDatabase, branch: Branch, kind: str, payload: dict[str, Any]) -> Node: # A session holds a single connection, which cannot serve two racing coroutines. async with db.start_session() as session_db: return await create_node( data=deepcopy(payload), db=session_db, branch=branch, - schema=session_db.schema.get(name=SCOPED_DEVICE.kind, branch=branch, duplicate=False), + schema=session_db.schema.get(name=kind, branch=branch, duplicate=False), ) +async def create_device_in_own_session(db: InfrahubDatabase, branch: Branch, payload: dict[str, Any]) -> Node: + return await create_in_own_session(db=db, branch=branch, kind=SCOPED_DEVICE.kind, payload=payload) + + async def save_templated_scoped_schema(db: InfrahubDatabase) -> None: """Save the scoped pool test schema with an object template generated for the device.""" schema = deepcopy(SCOPED_POOL_SCHEMA) @@ -879,3 +884,75 @@ async def test_the_component_allocates_in_its_division_of_the_pool_its_subtempla (expected_lock_name, LockAction.ACQUIRE), (expected_lock_name, LockAction.RELEASE), ] + + +async def test_a_template_create_and_a_create_in_the_division_of_its_second_component_both_complete( + db: InfrahubDatabase, + default_branch: Branch, + register_core_models_schema: SchemaBranch, + recording_lock_registry: RecordingLockRegistry, +) -> None: + """A card created in site-b holds the site's lock while the chassis template reaches its card of site-b.""" + await save_sited_component_pool_schema(db=db) + pool = await scoped_pool(db=db, name="vlan-per-site", allocation_scope=["site"], kind=SCOPED_CARD.kind) + site_a = await scoped_site(db=db, branch=default_branch, name="site-a") + site_b = await scoped_site(db=db, branch=default_branch, name="site-b") + chassis_a = await Node.init(db=db, schema=SCOPED_CHASSIS.kind, branch=default_branch) + await chassis_a.new(db=db, name="chassis-a") + await chassis_a.save(db=db) + template = await Node.init(db=db, schema=f"Template{SCOPED_CHASSIS.kind}", branch=default_branch) + await template.new(db=db, template_name="chassis-template") + await template.save(db=db) + for site in (site_a, site_b): + subtemplate = await Node.init(db=db, schema=f"Template{SCOPED_CARD.kind}", branch=default_branch) + await subtemplate.new( + db=db, + template_name=f"card-template-{site.get_attribute('name').value}", + name=f"card-{site.get_attribute('name').value}", + chassis=template.id, + site=site, + vlan_id_from_resource_pool={"id": pool.id}, + ) + await subtemplate.save(db=db) + site_b_lock_name = f"resource_pool.{pool.id}.{site_b.id}" + card_gate = recording_lock_registry.gate(f"resource_pool.{pool.id}:shared") + card_write = asyncio.create_task( + create_in_own_session( + db=db, + branch=default_branch, + kind=SCOPED_CARD.kind, + payload={ + "name": {"value": "card-b1"}, + "chassis": {"id": chassis_a.id}, + "site": {"id": site_b.id}, + "vlan_id": {"from_pool": {"id": pool.id}}, + }, + ) + ) + await card_gate.wait_reached() + chassis_write = asyncio.create_task( + create_in_own_session( + db=db, + branch=default_branch, + kind=SCOPED_CHASSIS.kind, + payload={"name": {"value": "chassis-b"}, "object_template": {"id": template.id}}, + ) + ) + await wait_until_waiting(timeline=recording_lock_registry.timeline, name=site_b_lock_name) + + card_gate.release() + _, blocked = await asyncio.wait([card_write, chassis_write], timeout=LOCK_WAIT_TIMEOUT_SECONDS) + for write in blocked: + write.cancel() + await asyncio.gather(*blocked, return_exceptions=True) + + assert len(blocked) == 0 + chassis = chassis_write.result() + cards = await NodeManager.query( + db=db, schema=SCOPED_CARD.kind, branch=default_branch, filters={"chassis__ids": [chassis.id]} + ) + card_sites = [await card.get_relationship("site").get_peer(db=db) for card in cards] + assert sorted((site.id, vlan_id(card)) for site, card in zip(card_sites, cards, strict=True)) == sorted( + [(site_a.id, 1), (site_b.id, 2)] + ) + assert vlan_id(card_write.result()) == 1 diff --git a/backend/tests/component/graphql/resource_manager/number_pools/test_pool_range_changes_and_writes.py b/backend/tests/component/graphql/resource_manager/number_pools/test_pool_range_changes_and_writes.py index c29cbfc87ab..4646b1bdefb 100644 --- a/backend/tests/component/graphql/resource_manager/number_pools/test_pool_range_changes_and_writes.py +++ b/backend/tests/component/graphql/resource_manager/number_pools/test_pool_range_changes_and_writes.py @@ -16,6 +16,7 @@ from infrahub.pools.number_pool_locker import picking_lock_name from infrahub.pools.scope import Division, DivisionElementPath from tests.helpers.constants import LOCK_WAIT_TIMEOUT_SECONDS +from tests.helpers.lock import wait_until_waiting from tests.helpers.number_pool import ( SCOPED_ATTRIBUTE_NAME, SCOPED_DEVICE, @@ -181,14 +182,6 @@ async def _device_numbers(db: InfrahubDatabase, branch: Branch) -> list[int]: return [vlan_id(device) for device in devices] -async def _wait_until_waiting(timeline: LockTimeline, name: str) -> None: - async def _poll() -> None: - while timeline.waiting(name) != 1: # noqa: ASYNC110 - await asyncio.sleep(0.01) - - await asyncio.wait_for(_poll(), timeout=LOCK_WAIT_TIMEOUT_SECONDS) - - async def _race( registry: RecordingLockRegistry, first: Coroutine[Any, Any, ExecutionResult], @@ -201,7 +194,7 @@ async def _race( first_task = asyncio.create_task(first) await gate.wait_reached() second_task = asyncio.create_task(second) - await _wait_until_waiting(timeline=registry.timeline, name=waits_at) + await wait_until_waiting(timeline=registry.timeline, name=waits_at) gate.release() first_result = await asyncio.wait_for(first_task, timeout=LOCK_WAIT_TIMEOUT_SECONDS) @@ -371,7 +364,7 @@ async def allocate() -> int: holder = asyncio.create_task(hold_picking()) await asyncio.wait_for(picking_held.wait(), timeout=LOCK_WAIT_TIMEOUT_SECONDS) allocation = asyncio.create_task(allocate()) - await _wait_until_waiting(timeline=recording_lock_registry.timeline, name=picking_lock) + await wait_until_waiting(timeline=recording_lock_registry.timeline, name=picking_lock) result = await asyncio.wait_for( execute_in_own_session( diff --git a/backend/tests/helpers/db_transaction_tracker.py b/backend/tests/helpers/db_transaction_tracker.py new file mode 100644 index 00000000000..4b4519bbc6c --- /dev/null +++ b/backend/tests/helpers/db_transaction_tracker.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Self + +from infrahub.database import InfrahubDatabase, InfrahubDatabaseMode + +if TYPE_CHECKING: + from types import TracebackType + + +class TransactionUsage: + """How many database transactions are open right now.""" + + def __init__(self) -> None: + self.open_transactions = 0 + + +class TransactionTrackingInfrahubDatabase(InfrahubDatabase): + """Database that records how many of its transactions are open.""" + + def __init__(self, transaction_usage: TransactionUsage | None = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + # shared by reference so the sessions and transactions derived from this database report into the same tally + self.transaction_usage = transaction_usage if transaction_usage is not None else TransactionUsage() + + @classmethod + def from_db(cls, db: InfrahubDatabase) -> TransactionTrackingInfrahubDatabase: + """Build a transaction-tracking database on the driver of an existing one.""" + return cls( + mode=InfrahubDatabaseMode.DRIVER, + driver=db._driver, + db_type=db.db_type, + default_neo4j_runtime=db.default_neo4j_runtime, + queries_names_to_config=db.queries_names_to_config, + ) + + def get_context(self) -> dict[str, Any]: + ctx = super().get_context() + ctx["transaction_usage"] = self.transaction_usage + return ctx + + async def __aenter__(self) -> Self: + await super().__aenter__() + if self.is_transaction: + self.transaction_usage.open_transactions += 1 + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await super().__aexit__(exc_type, exc_value, traceback) + if self.is_transaction: + self.transaction_usage.open_transactions -= 1 diff --git a/backend/tests/helpers/lock.py b/backend/tests/helpers/lock.py new file mode 100644 index 00000000000..ad9cc223e75 --- /dev/null +++ b/backend/tests/helpers/lock.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from tests.helpers.constants import LOCK_WAIT_TIMEOUT_SECONDS + +if TYPE_CHECKING: + from tests.adapters.lock import LockTimeline + + +async def wait_until_waiting(timeline: LockTimeline, name: str) -> None: + """Return once exactly one caller waits for the lock `name`. + + Raises: + TimeoutError: When no caller waits for the lock within the lock wait limit of the tests. + + """ + + async def _poll() -> None: + while timeline.waiting(name) != 1: # noqa: ASYNC110 + await asyncio.sleep(0.01) + + await asyncio.wait_for(_poll(), timeout=LOCK_WAIT_TIMEOUT_SECONDS) diff --git a/changelog/+template-component-number-pool-lock-order.fixed.md b/changelog/+template-component-number-pool-lock-order.fixed.md new file mode 100644 index 00000000000..0f68bcfa8c1 --- /dev/null +++ b/changelog/+template-component-number-pool-lock-order.fixed.md @@ -0,0 +1 @@ +Creating an object from a template now takes the locks of the resource pools that the template, its component templates and the `NumberPool` attributes of its components draw from before it starts writing. Before, the create could block when another request took a number from the same number pool at the same time, and both requests waited for each other until the client gave up. diff --git a/dev/knowledge/backend/mutations.md b/dev/knowledge/backend/mutations.md index 4fcea035dc0..9c73c560416 100644 --- a/dev/knowledge/backend/mutations.md +++ b/dev/knowledge/backend/mutations.md @@ -22,7 +22,9 @@ GraphQL mutations for creating, updating, upserting, and deleting nodes. All mut ``` mutate_create() -> create_node() # create.py - -> preview node (process_pools=False) for lock calculation + -> preview node (process_pools=False) for lock calculation, with the id of the node to create + -> read_template_components() # template only: the subtemplates, read a level at a time + -> get_template_pool_lock_names() # template only: the pool locks of the object and its components -> acquire InfrahubMultiLock -> _do_create_node() -> NodeMutationContext # records the other nodes the mutation writes, for events @@ -36,7 +38,7 @@ mutate_create() -> add_human_friendly_id() -> add_display_label() -> NodeCreateAllQuery # bulk Neo4j insert - -> handle_template_relationships() # template instantiation, read a level at a time + -> create_components() # template instantiation, created depth first -> apply profiles if applicable -> emit NodeCreatedEvent ``` @@ -224,8 +226,10 @@ A peer that must keep its relationship, such as a component under a mandatory pa ## Locking Strategy -1. A preview node is created with `process_pools=False` (no side effects) -2. Lock names are computed from uniqueness constraints and resource pool fields +1. A preview node is created with `process_pools=False` (no side effects); a create gives it the id of + the node it creates +2. Lock names are computed from uniqueness constraints and resource pool fields; for a create from a + template they also include the pool locks of its components 3. `InfrahubMultiLock` acquired before the transaction 4. Actual mutation runs inside the lock @@ -272,16 +276,16 @@ the new pool before its transaction commits, so no write can hold it. - **A hold lasts until the commit.** Inside a transaction, `holding` and `changing` hand their release to `InfrahubDatabase.hold_until_transaction_end`, because a number or a range only becomes visible to other writes once it is committed. This also covers allocations that run inside a transaction outside - the mutation lock, such as template components and migrations. + the mutation lock, such as migrations. - **An exclusive request waits for the shared holders.** It takes the mutex, so new shared requests queue behind it, then waits until no entry remains. After `INFRAHUB_CACHE_EXCLUSIVE_LOCK_WAIT_SECS` it releases the mutex and the range change fails with a retry message. The lock error, which names the setting, is logged as a warning, so that the API message does not name a server setting. - **The division's lock comes before the pool's shared hold.** A write that holds the pool until its commit and then waits for a division's lock can wait on an allocation that is itself queued behind a - range change, so allocations take `picking` first. Template components are the exception: they - allocate inside the template's transaction without taking the division's lock first, so the cycle can - still form until the waiting range change reaches its wait limit and fails. + range change, so allocations take `picking` first. A create from a template takes it before its + transaction, with the exceptions listed in + [Pool locks of a create from a template](#pool-locks-of-a-create-from-a-template). - **A task that holds the pool exclusively skips `picking`.** Every other allocation picks inside its shared hold, so none can pick while the pool is held exclusively, and skipping the division's lock keeps the exclusive holder from waiting on an allocation that is itself queued behind it. @@ -304,13 +308,65 @@ the new pool before its transaction commits, so no write can hold it. - **Not covered:** the IP prefix and IP address pool `GetResource` mutations run without a transaction, so a number drawn by the IP node they create is released before its write. +### Pool locks of a create from a template + +A create from a template draws from pools inside its transaction: the pools the template names for the +object, and the pools its components draw from. The query that creates a node with a number that a +number pool tracks locks that pool's vertex in Neo4j until the commit, for every division. A write that +waits for a division's lock inside its transaction, after it wrote such a number, can therefore wait on +an allocation that holds that lock and itself waits for the vertex. Neither wait has a time limit, and +Neo4j does not detect the cycle. So `create_node()` adds the division locks and the IP pool locks of the +create to the mutation's lock names, which it takes before it opens its transaction; the allocations +inside the transaction take them again without waiting. The pool's shared hold is still taken inside the +transaction. The exceptions are listed below. + +`get_template_pool_lock_names()` in `core/node/create.py` returns the names the preview of the object +does not already hold: + +| Draws from | Object created from the template | Component created from a subtemplate | +|------------|----------------------------------|--------------------------------------| +| Number pool the template names, with a scope | preview of the object | `resource_pool..` | +| Number pool the template names, without a scope | `resource_pool..all` | `resource_pool..all` | +| `NumberPool` attribute of the schema | preview of the object | `resource_pool..`, or `.all` without a scope | +| IP address or IP prefix pool the template names | `resource_pool.` | `resource_pool.` | + +- **Each pooled component is built twice.** `create_node()` builds an unsaved preview of each component + that draws from a pool, and of each component above one, to read its division. A component draws from a + pool when its subtemplate names one or its kind has a `NumberPool` attribute. A preview of a component in + a scoped number pool reads the pool once more, and the peers or the profiles that hold a scope element + when the preview does not hold them. A template whose components draw from no pool builds no preview. +- **The preview and the object created share one id.** `create_node()` generates the id of the object + before it builds the preview, unless the payload sets one, and each `TemplateComponent` holds the id of + the component it describes. So a lock name that holds the id of a new object, such as the division of a + component whose scope names its new parent, or `relationship_count..`, names the + object created. An id that `create_node()` generated is not checked against the database; an id the + payload sets is. +- **The subtemplates are read before the locks, as the template is.** A subtemplate that another write + changes or deletes while the create waits for a lock is applied as it was read. +- **A validation error of a component that draws from a pool, or holds one that does, is raised before the + object's own checks**, because its preview is built before the locks. Other components are validated + after them. +- **Not covered: a scope element that a pool fills.** When a pool fills a field that another pool's scope + names, on the object or on a component, the preview reads that element as empty. The create takes the + lock of that empty division (`.all` when the scope has one element), and takes the lock of the real + division inside the transaction. +- **Not covered: an IP pool whose address or prefix kind has a `NumberPool` attribute.** The IP node is + created inside the transaction, where it takes the lock of that number pool. A create that names the IP + pool with `from_pool` on a relationship does the same. +- **Not covered: a caller that passes a database with a transaction already open.** `create_node()` then + takes the locks inside that transaction. + +`backend/tests/component/core/node/test_template_pool_lock_order.py` covers every cell of the table, a +nested component, a payload that sets the `id`, and a division that holds the object created. + ## Key Files | File | What | |------|------| | `graphql/mutations/main.py` | Mutation dispatcher, create/update/upsert/delete | | `core/node/__init__.py` | `Node.new()`, `load()`, `from_graphql()`, `_create()`, `_update()`, `save()` | -| `core/node/create.py` | `create_node()`, `_do_create_node()`, template relationship handling | +| `core/node/create.py` | `create_node()`, `_do_create_node()`, `read_template_components()`, `get_template_pool_lock_names()`, `create_components()` | +| `core/node/lock_utils.py` | `get_lock_names_on_object_mutation()`, `get_pool_lock_names()`, `ip_pool_lock_name()` | | `core/relationship/model.py` | `RelationshipManager.update()`, `resolve()`, `save()` | | `core/constraint/node/runner.py` | `NodeConstraintRunner` for uniqueness/constraint checks | | `core/query/node.py` | `NodeCreateAllQuery` and other DB queries | diff --git a/dev/knowledge/backend/templates.md b/dev/knowledge/backend/templates.md index 4b2db2feb25..e724d30778c 100644 --- a/dev/knowledge/backend/templates.md +++ b/dev/knowledge/backend/templates.md @@ -13,7 +13,7 @@ Object Templates are user-defined "shape" objects: a template node holds default | `backend/infrahub/core/schema/definitions/core/template.py` | `core_object_template` and `core_object_component_template` generics — the core base kinds every template inherits from. | | `backend/infrahub/core/schema/schema_branch.py` | Generation of per-kind template schemas from `generate_template`-enabled nodes (`manage_object_template_schemas`, `add_relationships_to_template`, etc.). | | `backend/infrahub/core/relationship/constraints/template_resource_pool_exclusive.py` | Constraint preventing both a fixed value and a pool reference being set on the same template field. | -| `backend/infrahub/core/node/create.py` | `handle_template_relationships()` reads the template's component peers a level at a time, then materializes them as real instances depth first. | +| `backend/infrahub/core/node/create.py` | `create_node()` reads the template's component peers a level at a time with `read_template_components()`, before it takes the mutation's locks. `create_components()` then materializes them as real instances depth first. | | `backend/infrahub/core/node/__init__.py` | `Node.handle_object_template()` is the call site that constructs and invokes `NodeTemplateApplier` during node field processing. | ## Core Kinds @@ -62,7 +62,7 @@ When a user creates an object with `object_template={id: ...}`, `Node.handle_obj 4. Merges only previously-absent keys back into `fields`, preserving user input. 5. Returns the set of `pool_pending_fields` — fields whose pool allocation was deferred (e.g., because the chosen allocator is the no-op variant). -After save, `handle_template_relationships()` (`core/node/create.py`) walks the new node's `COMPONENT` relationships and materializes any subtemplate peers as their own real objects, then walks their components in turn. It reads the subtemplates a level at a time and creates the objects depth first — see [What applying a template reads](#what-applying-a-template-reads) for why those are two different walks. +After save, `create_components()` (`core/node/create.py`) materializes the subtemplates that the template holds on the kind's `COMPONENT` relationships, except the relationships the payload sets, as their own real objects, then their components in turn. `create_node()` reads the subtemplates before it takes the mutation's locks, because the lock names include the locks of the pools the components draw from (see [Pool locks of a create from a template](mutations.md#pool-locks-of-a-create-from-a-template)); the objects are created depth first — see [What applying a template reads](#what-applying-a-template-reads) for why those are two different walks. ### What applying a template reads @@ -80,7 +80,7 @@ The peers of the template's component relationships come back with that read, so of the first level are already in memory; each component is then created from the ids and kinds those relationships carry. -`handle_template_relationships()` reads the subtemplates a level at a time rather than a parent at +`read_template_components()` reads the subtemplates a level at a time rather than a parent at a time. The subtemplates a level names are all known before any of them is read, so the whole level is read at once — one relationship read covering every subtemplate of the level, which brings back the peers naming the level below. A level therefore costs the same read whether it hangs from one @@ -100,7 +100,9 @@ in hand. A create pays one of those reads for the template and one for each leve under it. The rest of its cost is the 2 queries reading the template node itself, the node's own uniqueness check and write, and 2 queries per component — its uniqueness check and its write — plus a count constraint for any mandatory parent that component declares. None of it grows with the -number of objects created from the template before. +number of objects created from the template before. A component that draws from a pool, or holds one +that does, is also built once as an unsaved preview before the locks; see +[Pool locks of a create from a template](mutations.md#pool-locks-of-a-create-from-a-template). On the test schema's device -> interface -> SFP, a device created from a template carrying ten interfaces costs 30 queries, and one carrying five interfaces each holding an SFP costs 38. @@ -134,6 +136,7 @@ Templates can reference resource pools instead of fixed values. The integration - When iterating template relationships, names ending in `RESOURCE_POOL_REL_SUFFIX` are routed to `_handle_pool_relationship`. - The applier asks the `PoolAllocator` to allocate either an attribute value (number) or a relationship peer (IP), then writes the result under the original (non-suffixed) name. For a number it writes only the value and records the pool in `TemplatePoolFields.allocated`; the pool's `IS_RESERVED` edge is what the attribute reports as its GraphQL `source`, and the stored source is never set to the pool. For an IP peer it sets the relationship's `source` to the pool's id. - If allocation cannot proceed (e.g., `NoOpPoolAllocator` is in use), the field is added to `pool_pending_fields` so the caller can decide. +- Components draw from the pools their subtemplates name in `allocate_from_resource_pools()` (`core/node/create.py`). `create_node()` takes the locks of these pools before its transaction; see [Pool locks of a create from a template](mutations.md#pool-locks-of-a-create-from-a-template). **Constraint** (`TemplateResourcePoolExclusiveConstraint`): @@ -156,6 +159,7 @@ Templates can reference resource pools instead of fixed values. The integration |------|----------| | `backend/tests/component/templates/test_template_applier.py` | End-to-end behaviour of `NodeTemplateApplier` (attributes, relationships, resource pools, `*_for_instances` group propagation). | | `backend/tests/component/core/node/test_template_pool_allocation.py` | Resource pools on templates, including the order a pool shared by two component levels is drawn from. | +| `backend/tests/component/core/node/test_template_pool_lock_order.py` | The pool locks a create from a template takes before its transaction, for the object and its components. | | `backend/tests/component/core/schema_manager/test_template_resource_pool_relationships.py` | Schema-level emission of `_from_resource_pool` rels for IP and Number cases. | | `backend/tests/component/core/schema_manager/test_template_group_relationships.py` | Schema-level emission of `member_of_groups_for_instances` / `subscriber_of_groups_for_instances` on templates. | | `backend/tests/integration/templates/` | Lifecycle tests (template attribute updates, resource pool wiring). |