Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor agent event indexing #2246

Merged
merged 51 commits into from
May 22, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
ba41f21
Start preparations for backfill indexing
asaj May 12, 2023
fe97a32
builds
asaj May 13, 2023
1de96b0
Logging, fix bug that caused unnecessary backtracks
asaj May 13, 2023
4b18101
Ready to begin backwards pass
asaj May 13, 2023
fdb53e5
Backwards pass passing e2e tests
asaj May 13, 2023
56dd8e0
cleanup
asaj May 13, 2023
e62f83f
Faster e2e test
asaj May 13, 2023
fc8a8b8
More e2e speedups
asaj May 13, 2023
002a2ce
more speedups
asaj May 13, 2023
a55229c
Add HyperlaneDb trait
asaj May 14, 2023
30a1f6c
HyperlaneDB -> HyperlaneRocksDB
asaj May 14, 2023
2dfbfbd
ContractSync uses HyperlaneDB
asaj May 14, 2023
339454b
HyperlaneDB is async
asaj May 14, 2023
69df330
SqlChainScraper is HyperlaneDB
asaj May 14, 2023
19dc8e1
Dedupe syncing logic in scraper
asaj May 15, 2023
8ef541b
Revert
asaj May 15, 2023
38331ca
cleanup
asaj May 15, 2023
29c918a
Make contract sync generic
asaj May 16, 2023
02899cf
ForwardBackwardMessageSyncCursor
asaj May 16, 2023
1e1b922
fix build
asaj May 16, 2023
0ccccbb
No foundry in e2e
asaj May 16, 2023
e6e5da4
Fix bug in RateLimitingCursor
asaj May 16, 2023
275562c
Index deliveries
asaj May 16, 2023
53975a7
Read high watermarks
asaj May 16, 2023
e2551e6
Update high watermark
asaj May 17, 2023
43af6e2
clippy
asaj May 17, 2023
046abfc
Cleanup
asaj May 17, 2023
88fd97d
Merge branch 'main' into asaj/indexing
asaj May 17, 2023
7f7025b
merge main
asaj May 17, 2023
3644635
Merge branch 'asaj/indexing' of github.com:hyperlane-xyz/hyperlane-mo…
asaj May 17, 2023
09947d8
fmt
asaj May 17, 2023
fd57967
Fix for higher order lifetime issue
mattiekat May 17, 2023
0f9d7c8
Merge remote-tracking branch 'origin/asaj/indexing' into asaj/indexing
mattiekat May 17, 2023
0b0df18
lint
asaj May 17, 2023
08a3f19
fmt
asaj May 17, 2023
fd74c22
some comments
asaj May 18, 2023
2cbf21a
more comments
asaj May 18, 2023
b442787
fix tests
asaj May 18, 2023
743c70f
comments
asaj May 18, 2023
d40081a
fmt
asaj May 18, 2023
1d9fd9f
Merge branch 'main' into asaj/indexing
asaj May 19, 2023
b80b17b
Scraper running in e2e
asaj May 19, 2023
97cf613
Add scraper to e2e test
asaj May 20, 2023
cc5b55a
Merge branch 'asaj/indexing' of github.com:hyperlane-xyz/hyperlane-mo…
asaj May 20, 2023
211b05d
Merge branch 'main' into asaj/indexing
asaj May 20, 2023
fa2533b
lint
asaj May 20, 2023
6cde032
Merge branch 'asaj/indexing' of github.com:hyperlane-xyz/hyperlane-mo…
asaj May 20, 2023
88db437
metrics
asaj May 22, 2023
9afe4eb
Final pass
asaj May 22, 2023
81f3733
update message count
asaj May 22, 2023
e3aca3f
fmt
asaj May 22, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Final pass
  • Loading branch information
asaj committed May 22, 2023
commit 9afe4eb116c2775f41828b20f4fcef5b40028f70
2 changes: 1 addition & 1 deletion rust/agents/scraper/src/agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ impl AsRef<HyperlaneAgentCore> for Scraper {
}
}

/// Generate a call to ChainSetup for the given builder
/// Create a function to spawn task that syncs contract events
macro_rules! spawn_sync_task {
($name:ident, $cursor: ident, $label:literal) => {
async fn $name(
Expand Down
2 changes: 1 addition & 1 deletion rust/agents/scraper/src/conversions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub fn address_to_bytes(data: &H256) -> Vec<u8> {
}
}

// Creates a big-endian hex representation of the address hash
// Creates a big-endian hex representation of the address
pub fn bytes_to_address(data: Vec<u8>) -> eyre::Result<H256> {
if (data.len() != 20) && (data.len() != 32) {
return Err(eyre::eyre!("Invalid address length"));
Expand Down
6 changes: 3 additions & 3 deletions rust/agents/scraper/src/db/message.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use eyre::Result;
use itertools::Itertools;
use sea_orm::{
prelude::*, ActiveValue::*, DeriveColumn, EnumIter, Insert, QueryOrder, QuerySelect,
prelude::*, ActiveValue::*, DeriveColumn, EnumIter, Insert, QuerySelect,
};
use tracing::{debug, instrument, trace};

Expand Down Expand Up @@ -111,9 +111,9 @@ impl ScraperDb {
.filter(message::Column::Origin.eq(origin_domain))
.filter(message::Column::OriginMailbox.eq(address_to_bytes(origin_mailbox)))
.filter(message::Column::Nonce.eq(nonce))
.order_by_desc(message::Column::Nonce)
.select_only()
.column_as(message::Column::OriginTxId, QueryAs::Nonce)
.column_as(message::Column::OriginTxId.max(), QueryAs::Nonce)
.group_by(message::Column::Origin)
.into_values::<i64, QueryAs>()
.one(&self.0)
.await?;
Expand Down
2 changes: 1 addition & 1 deletion rust/hyperlane-base/src/contract_sync/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ where
stored_logs.inc_by(stored as u64);
// We check the value of the current gauge to avoid overwriting a higher value
// when using a ForwardBackwardMessageSyncCursor
if to > indexed_height.get() {
if to as i64 > indexed_height.get() {
indexed_height.set(to as i64);
}
// Update cursor
Expand Down
13 changes: 7 additions & 6 deletions rust/utils/run-locally/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
//! the test for. This
//! does not include the initial setup time. If this timeout is reached before
//! the end conditions are met, the test is a failure. Defaults to 10 min.
//! - `E2E_KATHY_MESSAGES`: Number of kathy messages to dispatch. Defaults to 10 if CI mode is enabled.
//! - `E2E_KATHY_MESSAGES`: Number of kathy messages to dispatch. Defaults to 16 if CI mode is enabled.
//! - `E2E_LOG_ALL`: Log all output instead of writing to log files. Defaults to
//! true if CI mode,
//! else false.
Expand Down Expand Up @@ -132,7 +132,7 @@ fn main() -> ExitCode {
let r = env::var("E2E_KATHY_MESSAGES")
.ok()
.map(|r| r.parse::<u64>().unwrap());
r.unwrap_or(10)
r.unwrap_or(16)
};

let log_all = env::var("E2E_LOG_ALL")
Expand Down Expand Up @@ -369,13 +369,13 @@ fn main() -> ExitCode {
state.validators.push(validator);
}

// Run one round of kathy before the relayer comes up
// Send half the kathy messages before the relayer comes up
let mut kathy = Command::new("yarn");
kathy
.arg("kathy")
.args([
"--messages",
&kathy_messages.to_string(),
&(kathy_messages / 2).to_string(),
"--timeout",
"1000",
])
Expand Down Expand Up @@ -403,12 +403,13 @@ fn main() -> ExitCode {
println!("Setup complete! Agents running in background...");
println!("Ctrl+C to end execution...");

// Send half the kathy messages after the relayer comes up
let mut kathy = Command::new("yarn");
kathy
.arg("kathy")
.args([
"--messages",
&kathy_messages.to_string(),
&(kathy_messages / 2).to_string(),
"--timeout",
"1000",
])
Expand Down Expand Up @@ -440,7 +441,7 @@ fn main() -> ExitCode {
}
if ci_mode {
// for CI we have to look for the end condition.
let num_messages_expected = kathy_messages as u32 * 2;
let num_messages_expected = (kathy_messages / 2) as u32 * 2;
if kathy_done && termination_invariants_met(num_messages_expected) {
// end condition reached successfully
println!("Kathy completed successfully and agent metrics look healthy");
Expand Down