Skip to content

Commit e3c7251

Browse files
michaelsproulmichaelsproul
committed
Merge remote-tracking branch 'origin/unstable' into remove-eth1
2 parents 7a94542 + 6135f41 commit e3c7251

File tree

148 files changed

+4124
-1142
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

148 files changed

+4124
-1142
lines changed

‎.github/workflows/local-testnet.yml‎

Lines changed: 123 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020

2121
- name: Build Docker image
2222
run: |
23-
docker build --build-arg FEATURES=portable -t lighthouse:local .
23+
docker build --build-arg FEATURES=portable,spec-minimal -t lighthouse:local .
2424
docker save lighthouse:local -o lighthouse-docker.tar
2525
2626
- name: Upload Docker image artifact
@@ -52,30 +52,52 @@ jobs:
5252
- name: Load Docker image
5353
run: docker load -i lighthouse-docker.tar
5454

55-
- name: Start local testnet
56-
run: ./start_local_testnet.sh -e local -c -b false && sleep 60
55+
- name: Start local testnet with Assertoor
56+
run: ./start_local_testnet.sh -e local-assertoor -c -a -b false && sleep 60
5757
working-directory: scripts/local_testnet
5858

59-
- name: Stop local testnet and dump logs
60-
run: ./stop_local_testnet.sh local
61-
working-directory: scripts/local_testnet
62-
63-
- name: Start local testnet with blinded block production
64-
run: ./start_local_testnet.sh -e local-blinded -c -p -b false && sleep 60
65-
working-directory: scripts/local_testnet
59+
- name: Await Assertoor test result
60+
id: assertoor_test_result
61+
uses: ethpandaops/assertoor-github-action@v1
62+
with:
63+
kurtosis_enclave_name: local-assertoor
6664

6765
- name: Stop local testnet and dump logs
68-
run: ./stop_local_testnet.sh local-blinded
66+
run: ./stop_local_testnet.sh local-assertoor
6967
working-directory: scripts/local_testnet
7068

7169
- name: Upload logs artifact
70+
if: always()
7271
uses: actions/upload-artifact@v4
7372
with:
7473
name: logs-local-testnet
7574
path: |
7675
scripts/local_testnet/logs
7776
retention-days: 3
7877

78+
- name: Return Assertoor test result
79+
shell: bash
80+
run: |
81+
test_result="${{ steps.assertoor_test_result.outputs.result }}"
82+
test_status=$(
83+
cat <<"EOF"
84+
${{ steps.assertoor_test_result.outputs.test_overview }}
85+
EOF
86+
)
87+
failed_test_status=$(
88+
cat <<"EOF"
89+
${{ steps.assertoor_test_result.outputs.failed_test_details }}
90+
EOF
91+
)
92+
93+
echo "Test Result: $test_result"
94+
echo "$test_status"
95+
if ! [ "$test_result" == "success" ]; then
96+
echo "Failed Test Task Status:"
97+
echo "$failed_test_status"
98+
exit 1
99+
fi
100+
79101
doppelganger-protection-success-test:
80102
needs: dockerfile-ubuntu
81103
runs-on: ubuntu-22.04
@@ -104,6 +126,7 @@ jobs:
104126
working-directory: scripts/tests
105127

106128
- name: Upload logs artifact
129+
if: always()
107130
uses: actions/upload-artifact@v4
108131
with:
109132
name: logs-doppelganger-protection-success
@@ -139,13 +162,99 @@ jobs:
139162
working-directory: scripts/tests
140163

141164
- name: Upload logs artifact
165+
if: always()
142166
uses: actions/upload-artifact@v4
143167
with:
144168
name: logs-doppelganger-protection-failure
145169
path: |
146170
scripts/local_testnet/logs
147171
retention-days: 3
148172

173+
# Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
174+
checkpoint-sync-test:
175+
name: checkpoint-sync-test-${{ matrix.network }}
176+
runs-on: ubuntu-latest
177+
needs: dockerfile-ubuntu
178+
if: contains(github.event.pull_request.labels.*.name, 'syncing')
179+
continue-on-error: true
180+
strategy:
181+
matrix:
182+
network: [sepolia, devnet]
183+
steps:
184+
- uses: actions/checkout@v4
185+
186+
- name: Install Kurtosis
187+
run: |
188+
echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
189+
sudo apt update
190+
sudo apt install -y kurtosis-cli
191+
kurtosis analytics disable
192+
193+
- name: Download Docker image artifact
194+
uses: actions/download-artifact@v4
195+
with:
196+
name: lighthouse-docker
197+
path: .
198+
199+
- name: Load Docker image
200+
run: docker load -i lighthouse-docker.tar
201+
202+
- name: Run the checkpoint sync test script
203+
run: |
204+
./checkpoint-sync.sh "sync-${{ matrix.network }}" "checkpoint-sync-config-${{ matrix.network }}.yaml"
205+
working-directory: scripts/tests
206+
207+
- name: Upload logs artifact
208+
if: always()
209+
uses: actions/upload-artifact@v4
210+
with:
211+
name: logs-checkpoint-sync-${{ matrix.network }}
212+
path: |
213+
scripts/local_testnet/logs
214+
retention-days: 3
215+
216+
# Test syncing from genesis on a local testnet. Aims to cover forward syncing both short and long distances.
217+
genesis-sync-test:
218+
name: genesis-sync-test-${{ matrix.fork }}-${{ matrix.offline_secs }}s
219+
runs-on: ubuntu-latest
220+
needs: dockerfile-ubuntu
221+
if: contains(github.event.pull_request.labels.*.name, 'syncing')
222+
strategy:
223+
matrix:
224+
fork: [electra, fulu]
225+
offline_secs: [120, 300]
226+
steps:
227+
- uses: actions/checkout@v4
228+
229+
- name: Install Kurtosis
230+
run: |
231+
echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
232+
sudo apt update
233+
sudo apt install -y kurtosis-cli
234+
kurtosis analytics disable
235+
236+
- name: Download Docker image artifact
237+
uses: actions/download-artifact@v4
238+
with:
239+
name: lighthouse-docker
240+
path: .
241+
242+
- name: Load Docker image
243+
run: docker load -i lighthouse-docker.tar
244+
245+
- name: Run the genesis sync test script
246+
run: |
247+
./genesis-sync.sh "sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s" "genesis-sync-config-${{ matrix.fork }}.yaml" "${{ matrix.fork }}" "${{ matrix.offline_secs }}"
248+
working-directory: scripts/tests
249+
250+
- name: Upload logs artifact
251+
if: always()
252+
uses: actions/upload-artifact@v4
253+
with:
254+
name: logs-genesis-sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s
255+
path: |
256+
scripts/local_testnet/logs
257+
retention-days: 3
149258

150259
# This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
151260
# a PR is safe to merge. New jobs should be added here.
@@ -161,4 +270,6 @@ jobs:
161270
steps:
162271
- uses: actions/checkout@v4
163272
- name: Check that success job is dependent on all others
164-
run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success
273+
run: |
274+
exclude_jobs='checkpoint-sync-test|genesis-sync-test'
275+
./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs"

‎Cargo.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Makefile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,9 @@ run-state-transition-tests:
218218
# Downloads and runs the EF test vectors.
219219
test-ef: make-ef-tests run-ef-tests
220220

221+
# Downloads and runs the nightly EF test vectors.
222+
test-ef-nightly: make-ef-tests-nightly run-ef-tests
223+
221224
# Downloads and runs the EF test vectors with nextest.
222225
nextest-ef: make-ef-tests nextest-run-ef-tests
223226

@@ -278,6 +281,10 @@ lint-full:
278281
make-ef-tests:
279282
make -C $(EF_TESTS)
280283

284+
# Download/extract the nightly EF test vectors.
285+
make-ef-tests-nightly:
286+
CONSENSUS_SPECS_TEST_VERSION=nightly make -C $(EF_TESTS)
287+
281288
# Verifies that crates compile with fuzzing features enabled
282289
arbitrary-fuzz:
283290
cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES)

‎account_manager/src/validator/import.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ pub fn cli_app() -> Command {
3232
.about(
3333
"Imports one or more EIP-2335 passwords into a Lighthouse VC directory, \
3434
requesting passwords interactively. The directory flag provides a convenient \
35-
method for importing a directory of keys generated by the eth2-deposit-cli \
35+
method for importing a directory of keys generated by the ethstaker-deposit-cli \
3636
Python utility.",
3737
)
3838
.arg(

‎beacon_node/beacon_chain/src/beacon_chain.rs‎

Lines changed: 71 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -56,12 +56,14 @@ use crate::observed_data_sidecars::ObservedDataSidecars;
5656
use crate::observed_operations::{ObservationOutcome, ObservedOperations};
5757
use crate::observed_slashable::ObservedSlashable;
5858
use crate::persisted_beacon_chain::PersistedBeaconChain;
59+
use crate::persisted_custody::persist_custody_context;
5960
use crate::persisted_fork_choice::PersistedForkChoice;
6061
use crate::pre_finalization_cache::PreFinalizationBlockCache;
6162
use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache};
6263
use crate::sync_committee_verification::{
6364
Error as SyncCommitteeError, VerifiedSyncCommitteeMessage, VerifiedSyncContribution,
6465
};
66+
use crate::validator_custody::CustodyContextSsz;
6567
use crate::validator_monitor::{
6668
get_slot_delay_ms, timestamp_now, ValidatorMonitor,
6769
HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS,
@@ -71,7 +73,9 @@ use crate::{
7173
kzg_utils, metrics, AvailabilityPendingExecutedBlock, BeaconChainError, BeaconForkChoiceStore,
7274
BeaconSnapshot, CachedHead,
7375
};
74-
use eth2::types::{EventKind, SseBlobSidecar, SseBlock, SseExtendedPayloadAttributes};
76+
use eth2::types::{
77+
EventKind, SseBlobSidecar, SseBlock, SseDataColumnSidecar, SseExtendedPayloadAttributes,
78+
};
7579
use execution_layer::{
7680
BlockProposalContents, BlockProposalContentsType, BuilderParams, ChainHealth, ExecutionLayer,
7781
FailedCondition, PayloadAttributes, PayloadStatus,
@@ -648,6 +652,23 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
648652
Ok(())
649653
}
650654

655+
/// Persists the custody information to disk.
656+
pub fn persist_custody_context(&self) -> Result<(), Error> {
657+
let custody_context: CustodyContextSsz = self
658+
.data_availability_checker
659+
.custody_context()
660+
.as_ref()
661+
.into();
662+
debug!(?custody_context, "Persisting custody context to store");
663+
664+
persist_custody_context::<T::EthSpec, T::HotStore, T::ColdStore>(
665+
self.store.clone(),
666+
custody_context,
667+
)?;
668+
669+
Ok(())
670+
}
671+
651672
/// Returns the slot _right now_ according to `self.slot_clock`. Returns `Err` if the slot is
652673
/// unavailable.
653674
///
@@ -2960,7 +2981,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
29602981
pub async fn verify_block_for_gossip(
29612982
self: &Arc<Self>,
29622983
block: Arc<SignedBeaconBlock<T::EthSpec>>,
2963-
custody_columns_count: usize,
29642984
) -> Result<GossipVerifiedBlock<T>, BlockError> {
29652985
let chain = self.clone();
29662986
self.task_executor
@@ -2970,7 +2990,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
29702990
let slot = block.slot();
29712991
let graffiti_string = block.message().body().graffiti().as_utf8_lossy();
29722992

2973-
match GossipVerifiedBlock::new(block, &chain, custody_columns_count) {
2993+
match GossipVerifiedBlock::new(block, &chain) {
29742994
Ok(verified) => {
29752995
let commitments_formatted = verified.block.commitments_formatted();
29762996
debug!(
@@ -3059,6 +3079,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
30593079
return Err(BlockError::DuplicateFullyImported(block_root));
30603080
}
30613081

3082+
self.emit_sse_data_column_sidecar_events(
3083+
&block_root,
3084+
data_columns.iter().map(|column| column.as_data_column()),
3085+
);
3086+
30623087
let r = self
30633088
.check_gossip_data_columns_availability_and_import(
30643089
slot,
@@ -3130,10 +3155,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
31303155
return Err(BlockError::DuplicateFullyImported(block_root));
31313156
}
31323157

3133-
// process_engine_blobs is called for both pre and post PeerDAS. However, post PeerDAS
3134-
// consumers don't expect the blobs event to fire erratically.
3135-
if let EngineGetBlobsOutput::Blobs(blobs) = &engine_get_blobs_output {
3136-
self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob()));
3158+
match &engine_get_blobs_output {
3159+
EngineGetBlobsOutput::Blobs(blobs) => {
3160+
self.emit_sse_blob_sidecar_events(&block_root, blobs.iter().map(|b| b.as_blob()));
3161+
}
3162+
EngineGetBlobsOutput::CustodyColumns(columns) => {
3163+
self.emit_sse_data_column_sidecar_events(
3164+
&block_root,
3165+
columns.iter().map(|column| column.as_data_column()),
3166+
);
3167+
}
31373168
}
31383169

31393170
let r = self
@@ -3163,6 +3194,31 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
31633194
}
31643195
}
31653196

3197+
fn emit_sse_data_column_sidecar_events<'a, I>(
3198+
self: &Arc<Self>,
3199+
block_root: &Hash256,
3200+
data_columns_iter: I,
3201+
) where
3202+
I: Iterator<Item = &'a DataColumnSidecar<T::EthSpec>>,
3203+
{
3204+
if let Some(event_handler) = self.event_handler.as_ref() {
3205+
if event_handler.has_data_column_sidecar_subscribers() {
3206+
let imported_data_columns = self
3207+
.data_availability_checker
3208+
.cached_data_column_indexes(block_root)
3209+
.unwrap_or_default();
3210+
let new_data_columns =
3211+
data_columns_iter.filter(|b| !imported_data_columns.contains(&b.index));
3212+
3213+
for data_column in new_data_columns {
3214+
event_handler.register(EventKind::DataColumnSidecar(
3215+
SseDataColumnSidecar::from_data_column_sidecar(data_column),
3216+
));
3217+
}
3218+
}
3219+
}
3220+
}
3221+
31663222
/// Cache the columns in the processing cache, process it, then evict it from the cache if it was
31673223
/// imported or errors.
31683224
pub async fn process_rpc_custody_columns(
@@ -3203,6 +3259,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
32033259
}
32043260
}
32053261

3262+
self.emit_sse_data_column_sidecar_events(
3263+
&block_root,
3264+
custody_columns.iter().map(|column| column.as_ref()),
3265+
);
3266+
32063267
let r = self
32073268
.check_rpc_custody_columns_availability_and_import(slot, block_root, custody_columns)
32083269
.await;
@@ -3610,7 +3671,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
36103671
data_columns.iter().map(|c| c.as_data_column()),
36113672
)?;
36123673
self.data_availability_checker
3613-
.put_gossip_verified_data_columns(block_root, data_columns)?
3674+
.put_kzg_verified_custody_data_columns(block_root, data_columns)?
36143675
}
36153676
};
36163677

@@ -7076,7 +7137,8 @@ impl<T: BeaconChainTypes> Drop for BeaconChain<T> {
70767137
fn drop(&mut self) {
70777138
let drop = || -> Result<(), Error> {
70787139
self.persist_fork_choice()?;
7079-
self.persist_op_pool()
7140+
self.persist_op_pool()?;
7141+
self.persist_custody_context()
70807142
};
70817143

70827144
if let Err(e) = drop() {

‎beacon_node/beacon_chain/src/beacon_proposer_cache.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ pub fn compute_proposer_duties_from_head<T: BeaconChainTypes>(
181181
ensure_state_is_in_epoch(&mut state, head_state_root, request_epoch, &chain.spec)?;
182182

183183
let indices = state
184-
.get_beacon_proposer_indices(&chain.spec)
184+
.get_beacon_proposer_indices(request_epoch, &chain.spec)
185185
.map_err(BeaconChainError::from)?;
186186

187187
let dependent_root = state

0 commit comments

Comments
 (0)