Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions bin/dipper-service/src/log_text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,46 @@ pub(crate) fn shortened(error: &impl std::fmt::Display) -> String {
}
}

/// `err` and each error behind it, so a log says why a request failed and not only that it did: an
/// HTTP client's own text for a dropped connection is just "error sending request".
pub(crate) fn with_causes(err: &(dyn std::error::Error + 'static)) -> String {
std::iter::successors(Some(err), |err| err.source())
.map(ToString::to_string)
.collect::<Vec<_>>()
.join(": ")
}

#[cfg(test)]
mod tests {
use super::*;

/// An error with its text and the error behind it, if any.
#[derive(Debug)]
struct Failed(&'static str, Option<Box<Failed>>);

impl std::fmt::Display for Failed {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.0)
}
}

impl std::error::Error for Failed {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.1.as_deref().map(|cause| cause as _)
}
}

#[test]
fn an_error_is_logged_with_each_cause_behind_it() {
let closed = Failed("connection closed before message completed", None);
let err = Failed("error sending request", Some(Box::new(closed)));

assert_eq!(
with_causes(&err),
"error sending request: connection closed before message completed"
);
}

#[test]
fn a_short_text_is_quoted_whole() {
assert_eq!(untrusted("price too low"), r#""price too low""#);
Expand Down
65 changes: 61 additions & 4 deletions bin/dipper-service/src/network/service/chain_events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ use serde::Deserialize;
use thegraph_core::alloy::primitives::Address;
use url::Url;

use crate::log_text::with_causes;

/// State of an agreement as recorded by the subgraph. Mirrors the
/// `AgreementState` enum in the indexing-payments-subgraph schema.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
Expand Down Expand Up @@ -396,12 +398,14 @@ impl SubgraphEventSource {
}

// The HTTP client names the URL in a failed request, and a gateway URL can carry the key.
// A request sent with no answer, as on a kept-open connection the server just closed, is
// retried at once on a fresh connection.
let response = request.send().await.map_err(|e| {
let e = e.without_url();
if e.is_timeout() || e.is_connect() {
ChainEventError::Transient(format!("Network error: {}", e))
if e.is_timeout() || e.is_connect() || e.is_request() {
ChainEventError::Transient(format!("Network error: {}", with_causes(&e)))
} else {
ChainEventError::Permanent(format!("Request error: {}", e))
ChainEventError::Permanent(format!("Request error: {}", with_causes(&e)))
}
})?;

Expand Down Expand Up @@ -438,7 +442,10 @@ impl SubgraphEventSource {
}

let graphql_response: GraphQLResponse<T> = response.json().await.map_err(|e| {
ChainEventError::Permanent(format!("Failed to parse response: {}", e.without_url()))
ChainEventError::Permanent(format!(
"Failed to parse response: {}",
with_causes(&e.without_url())
))
})?;

if let Some(errors) = graphql_response.errors {
Expand Down Expand Up @@ -1385,6 +1392,56 @@ mod tests {
assert!(!err.to_string().contains("secret-key"), "{err}");
}

/// Serves 1 connection per answer in turn. `None` reads the request and closes the connection
/// without replying, as a server does when it drops a kept-open connection just as it is reused.
fn serve_connections(answers: Vec<Option<&'static str>>) -> Url {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap())
.parse()
.unwrap();
std::thread::spawn(move || {
for answer in answers {
let (mut socket, _) = listener.accept().unwrap();
let _ = socket.read(&mut [0u8; 4096]);
if let Some(body) = answer {
let head = "HTTP/1.1 200 OK\r\ncontent-type: application/json";
write!(
socket,
"{head}\r\ncontent-length: {}\r\n\r\n{body}",
body.len()
)
.unwrap();
}
}
});
url
}

#[tokio::test]
async fn test_dropped_connection_is_retried() {
let meta = r#"{"data":{"_meta":{"block":{"number":1}}}}"#;
let source = event_source_at(serve_connections(vec![None, Some(meta)]), 3);

let result = query_meta(&source).await;

assert!(result.is_ok(), "{result:?}");
}

#[tokio::test]
async fn test_dropped_connection_error_names_its_cause() {
let source = event_source_at(serve_connections(vec![None]), 0);

let err = query_meta(&source)
.await
.expect_err("a dropped connection must fail the query");

assert!(
err.to_string().to_lowercase().contains("connection"),
"{err}"
);
}

#[tokio::test]
async fn test_graphql_rate_limit_error_is_rate_limited_and_not_retried() {
let server = wiremock::MockServer::start().await;
Expand Down
10 changes: 8 additions & 2 deletions bin/dipper-service/src/network/service/entity_count_cache.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,14 +196,20 @@ async fn fetch_page(
builder = builder.bearer_auth(key);
}
let response = builder.send().await.map_err(|err| {
anyhow::anyhow!("failed to query entity counts page: {}", err.without_url())
anyhow::anyhow!(
"failed to query entity counts page: {}",
crate::log_text::with_causes(&err.without_url())
)
})?;
let status = response.status();
if !status.is_success() {
anyhow::bail!("entity counts page request failed: HTTP {status}");
}
response.json().await.map_err(|err| {
anyhow::anyhow!("failed to decode entity counts page: {}", err.without_url())
anyhow::anyhow!(
"failed to decode entity counts page: {}",
crate::log_text::with_causes(&err.without_url())
)
})
}

Expand Down
10 changes: 8 additions & 2 deletions bin/dipper-service/src/network/service/indexer_urls.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,20 @@ pub(in crate::network) async fn fetch_snapshot(
}

let response = builder.send().await.map_err(|err| {
anyhow::anyhow!("failed to query indexers page: {}", err.without_url())
anyhow::anyhow!(
"failed to query indexers page: {}",
crate::log_text::with_causes(&err.without_url())
)
})?;
let status = response.status();
if !status.is_success() {
anyhow::bail!("indexers page request failed: HTTP {status}");
}
let response: PageResponse = response.json().await.map_err(|err| {
anyhow::anyhow!("failed to decode indexers page: {}", err.without_url())
anyhow::anyhow!(
"failed to decode indexers page: {}",
crate::log_text::with_causes(&err.without_url())
)
})?;

let Some(data) = response.data else {
Expand Down
Loading