diff --git a/bin/dipper-service/src/log_text.rs b/bin/dipper-service/src/log_text.rs index 417c6bce..3c6a4288 100644 --- a/bin/dipper-service/src/log_text.rs +++ b/bin/dipper-service/src/log_text.rs @@ -29,10 +29,46 @@ pub(crate) fn shortened(error: &impl std::fmt::Display) -> String { } } +/// `err` and each error behind it, so a log says why a request failed and not only that it did: an +/// HTTP client's own text for a dropped connection is just "error sending request". +pub(crate) fn with_causes(err: &(dyn std::error::Error + 'static)) -> String { + std::iter::successors(Some(err), |err| err.source()) + .map(ToString::to_string) + .collect::>() + .join(": ") +} + #[cfg(test)] mod tests { use super::*; + /// An error with its text and the error behind it, if any. + #[derive(Debug)] + struct Failed(&'static str, Option>); + + impl std::fmt::Display for Failed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0) + } + } + + impl std::error::Error for Failed { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + self.1.as_deref().map(|cause| cause as _) + } + } + + #[test] + fn an_error_is_logged_with_each_cause_behind_it() { + let closed = Failed("connection closed before message completed", None); + let err = Failed("error sending request", Some(Box::new(closed))); + + assert_eq!( + with_causes(&err), + "error sending request: connection closed before message completed" + ); + } + #[test] fn a_short_text_is_quoted_whole() { assert_eq!(untrusted("price too low"), r#""price too low""#); diff --git a/bin/dipper-service/src/network/service/chain_events.rs b/bin/dipper-service/src/network/service/chain_events.rs index bffe0934..0251a201 100644 --- a/bin/dipper-service/src/network/service/chain_events.rs +++ b/bin/dipper-service/src/network/service/chain_events.rs @@ -43,6 +43,8 @@ use serde::Deserialize; use thegraph_core::alloy::primitives::Address; use url::Url; +use crate::log_text::with_causes; + /// State of an agreement as recorded by the subgraph. Mirrors the /// `AgreementState` enum in the indexing-payments-subgraph schema. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -396,12 +398,14 @@ impl SubgraphEventSource { } // The HTTP client names the URL in a failed request, and a gateway URL can carry the key. + // A request sent with no answer, as on a kept-open connection the server just closed, is + // retried at once on a fresh connection. let response = request.send().await.map_err(|e| { let e = e.without_url(); - if e.is_timeout() || e.is_connect() { - ChainEventError::Transient(format!("Network error: {}", e)) + if e.is_timeout() || e.is_connect() || e.is_request() { + ChainEventError::Transient(format!("Network error: {}", with_causes(&e))) } else { - ChainEventError::Permanent(format!("Request error: {}", e)) + ChainEventError::Permanent(format!("Request error: {}", with_causes(&e))) } })?; @@ -438,7 +442,10 @@ impl SubgraphEventSource { } let graphql_response: GraphQLResponse = response.json().await.map_err(|e| { - ChainEventError::Permanent(format!("Failed to parse response: {}", e.without_url())) + ChainEventError::Permanent(format!( + "Failed to parse response: {}", + with_causes(&e.without_url()) + )) })?; if let Some(errors) = graphql_response.errors { @@ -1385,6 +1392,56 @@ mod tests { assert!(!err.to_string().contains("secret-key"), "{err}"); } + /// Serves 1 connection per answer in turn. `None` reads the request and closes the connection + /// without replying, as a server does when it drops a kept-open connection just as it is reused. + fn serve_connections(answers: Vec>) -> Url { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()) + .parse() + .unwrap(); + std::thread::spawn(move || { + for answer in answers { + let (mut socket, _) = listener.accept().unwrap(); + let _ = socket.read(&mut [0u8; 4096]); + if let Some(body) = answer { + let head = "HTTP/1.1 200 OK\r\ncontent-type: application/json"; + write!( + socket, + "{head}\r\ncontent-length: {}\r\n\r\n{body}", + body.len() + ) + .unwrap(); + } + } + }); + url + } + + #[tokio::test] + async fn test_dropped_connection_is_retried() { + let meta = r#"{"data":{"_meta":{"block":{"number":1}}}}"#; + let source = event_source_at(serve_connections(vec![None, Some(meta)]), 3); + + let result = query_meta(&source).await; + + assert!(result.is_ok(), "{result:?}"); + } + + #[tokio::test] + async fn test_dropped_connection_error_names_its_cause() { + let source = event_source_at(serve_connections(vec![None]), 0); + + let err = query_meta(&source) + .await + .expect_err("a dropped connection must fail the query"); + + assert!( + err.to_string().to_lowercase().contains("connection"), + "{err}" + ); + } + #[tokio::test] async fn test_graphql_rate_limit_error_is_rate_limited_and_not_retried() { let server = wiremock::MockServer::start().await; diff --git a/bin/dipper-service/src/network/service/entity_count_cache.rs b/bin/dipper-service/src/network/service/entity_count_cache.rs index e58b75f4..dd1350b7 100644 --- a/bin/dipper-service/src/network/service/entity_count_cache.rs +++ b/bin/dipper-service/src/network/service/entity_count_cache.rs @@ -196,14 +196,20 @@ async fn fetch_page( builder = builder.bearer_auth(key); } let response = builder.send().await.map_err(|err| { - anyhow::anyhow!("failed to query entity counts page: {}", err.without_url()) + anyhow::anyhow!( + "failed to query entity counts page: {}", + crate::log_text::with_causes(&err.without_url()) + ) })?; let status = response.status(); if !status.is_success() { anyhow::bail!("entity counts page request failed: HTTP {status}"); } response.json().await.map_err(|err| { - anyhow::anyhow!("failed to decode entity counts page: {}", err.without_url()) + anyhow::anyhow!( + "failed to decode entity counts page: {}", + crate::log_text::with_causes(&err.without_url()) + ) }) } diff --git a/bin/dipper-service/src/network/service/indexer_urls.rs b/bin/dipper-service/src/network/service/indexer_urls.rs index abe6ab72..41116173 100644 --- a/bin/dipper-service/src/network/service/indexer_urls.rs +++ b/bin/dipper-service/src/network/service/indexer_urls.rs @@ -119,14 +119,20 @@ pub(in crate::network) async fn fetch_snapshot( } let response = builder.send().await.map_err(|err| { - anyhow::anyhow!("failed to query indexers page: {}", err.without_url()) + anyhow::anyhow!( + "failed to query indexers page: {}", + crate::log_text::with_causes(&err.without_url()) + ) })?; let status = response.status(); if !status.is_success() { anyhow::bail!("indexers page request failed: HTTP {status}"); } let response: PageResponse = response.json().await.map_err(|err| { - anyhow::anyhow!("failed to decode indexers page: {}", err.without_url()) + anyhow::anyhow!( + "failed to decode indexers page: {}", + crate::log_text::with_causes(&err.without_url()) + ) })?; let Some(data) = response.data else {