Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 79 additions & 7 deletions core/src/subgraph/runner/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,30 @@ where
Ok(self)
}

/// Only a started block stream registers the cancel guard that unassigning removes.
fn can_observe_unassign(&self) -> bool {
self.cancel_handle.is_some()
}

async fn back_off_before_restart(
&mut self,
err: Error,
reason: RestartReason,
) -> Result<RunnerState<C>, SubgraphRunnerError> {
if self.is_canceled() {
return self.cancel();
}
self.metrics.subgraph.deployment_status.failed();
error!(self.logger, "Subgraph failed to restart, retrying: {:#}", err;
"attempt" => self.state.backoff.attempt,
"retry_delay_s" => self.state.backoff.delay().as_secs());
self.state.backoff.sleep_async().await;
if self.is_canceled() {
return self.cancel();
}
Ok(RunnerState::Restarting { reason })
}

pub async fn run(self) -> Result<(), SubgraphRunnerError> {
self.run_inner(false).await.map(|_| ())
}
Expand Down Expand Up @@ -499,7 +523,12 @@ where
reason: StopReason::Canceled,
}
} else {
self.restart(reason).await?
match self.restart(reason).await {
Err(SubgraphRunnerError::Unknown(e)) if self.can_observe_unassign() => {
self.back_off_before_restart(e, reason).await?
}
res => res?,
}
}
}

Expand Down Expand Up @@ -563,7 +592,7 @@ where
if block.trigger_count() == 0
&& self.state.skip_ptr_updates_timer.elapsed() <= SKIP_PTR_UPDATES_THRESHOLD
&& !self.inputs.store.is_deployment_synced()
&& !close_to_chain_head(&block_ptr, &self.inputs.chain.chain_head_ptr().await?, 1000)
&& !self.is_close_to_chain_head(&block_ptr).await
&& !max_end_block_reached
{
// Skip this block and continue with the same stream
Expand Down Expand Up @@ -604,6 +633,18 @@ where
}
}

/// An unreadable chain head counts as close, so the block is processed rather than skipped.
async fn is_close_to_chain_head(&self, block_ptr: &BlockPtr) -> bool {
match self.inputs.chain.chain_head_ptr().await {
Ok(chain_head_ptr) => close_to_chain_head(block_ptr, &chain_head_ptr, 1000),
Err(e) => {
warn!(self.logger, "Failed to read the chain head, processing the block";
"error" => format!("{:#}", e));
true
}
}
}

/// Handle a revert event and determine the next state.
///
/// This is the state machine wrapper around `handle_revert` that handles
Expand Down Expand Up @@ -1295,11 +1336,19 @@ where
if self.state.should_try_unfail_non_deterministic {
// If the deployment head advanced, we can unfail
// the non-deterministic error (if there's any).
let outcome = self
let outcome = match self
.inputs
.store
.unfail_non_deterministic_error(&block_ptr)
.await?;
.await
{
Ok(outcome) => outcome,
Err(e) => {
warn!(self.logger, "Failed to unfail subgraph, trying again on the next block";
"error" => format!("{:#}", e));
UnfailOutcome::BehindErrorBlock
}
};

// Stop trying unless we're still behind the error block.
if outcome != UnfailOutcome::BehindErrorBlock {
Expand Down Expand Up @@ -1381,7 +1430,19 @@ where
let message = format!("{:#}", e).replace('\n', "\t");
let err = anyhow!("{}, code: {}", message, LogCode::SubgraphSyncingFailure);

self.fail_subgraph(message, Some(block_ptr), true).await?;
// The block's PoI is already written, so a restart would resume past it.
while let Err(e) = self
.fail_subgraph(message.clone(), Some(block_ptr.clone()), true)
.await
{
if self.is_canceled() {
return Ok(Action::Stop);
}
error!(self.logger, "Failed to record deterministic error, retrying: {:#}", e;
"attempt" => self.state.backoff.attempt,
"retry_delay_s" => self.state.backoff.delay().as_secs());
self.state.backoff.sleep_async().await;
}

Err(err)
}
Expand All @@ -1408,8 +1469,19 @@ where
//
// If we don't do this check we would keep adding the same error to the
// database.
if !self.inputs.store.health().await?.is_failed() {
self.fail_subgraph(message, Some(block_ptr), false).await?;
let already_failed = match self.inputs.store.health().await {
Ok(health) => health.is_failed(),
Err(e) => {
warn!(self.logger, "Failed to read subgraph health";
"error" => format!("{:#}", e));
false
}
};
if !already_failed
&& let Err(e) = self.fail_subgraph(message, Some(block_ptr), false).await
{
warn!(self.logger, "Failed to record non-deterministic error";
"error" => format!("{:#}", e));
}

// Sleep before restarting.
Expand Down
Loading
Loading