eof: Introduce EOF container format support in Assembly::assemble (EIP-3540)

Co-authored-by: Daniel Kirchner <daniel@ekpyron.org>

Author: rodiazet

libevmasm/Assembly.cpp

@@ -49,11 +49,14 @@ class Assembly
 {
 public:
 	Assembly(langutil::EVMVersion _evmVersion, bool _creation, std::optional<uint8_t> _eofVersion, std::string _name):
-	m_evmVersion(_evmVersion),
-	m_creation(_creation),
-	m_eofVersion(_eofVersion),
-	m_name(std::move(_name))
-	{}
+		m_evmVersion(_evmVersion),
+		m_creation(_creation),
+		m_eofVersion(_eofVersion),
+		m_name(std::move(_name))
+	{
+		// Code section number 0 has to be non-returning.
+		m_codeSections.emplace_back(CodeSection{0, 0x80, {}});
+	}
 
 	std::optional<uint8_t> eofVersion() const { return m_eofVersion; }
 	AssemblyItem newTag() { assertThrow(m_usedTags < 0xffffffff, AssemblyException, ""); return AssemblyItem(Tag, m_usedTags++); }
@@ -103,12 +106,6 @@ class Assembly
 	/// Appends @a _data literally to the very end of the bytecode.
 	void appendToAuxiliaryData(bytes const& _data) { m_auxiliaryData += _data; }
 
-	/// Returns the assembly items.
-	AssemblyItems const& items() const { return m_items; }
-
-	/// Returns the mutable assembly items. Use with care!
-	AssemblyItems& items() { return m_items; }
-
 	int deposit() const { return m_deposit; }
 	void adjustDeposit(int _adjustment) { m_deposit += _adjustment; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }
 	void setDeposit(int _deposit) { m_deposit = _deposit; assertThrow(m_deposit >= 0, InvalidDeposit, ""); }
@@ -170,7 +167,8 @@ class Assembly
 	static std::pair<std::shared_ptr<Assembly>, std::vector<std::string>> fromJSON(
 		Json const& _json,
 		std::vector<std::string> const& _sourceList = {},
-		size_t _level = 0
+		size_t _level = 0,
+		std::optional<uint8_t> _eofVersion = std::nullopt
 	);
 
 	/// Mark this assembly as invalid. Calling ``assemble`` on it will throw.
@@ -181,12 +179,30 @@ class Assembly
 
 	bool isCreation() const { return m_creation; }
 
+	struct CodeSection
+	{
+		uint8_t inputs = 0;
+		uint8_t outputs = 0;
+		AssemblyItems items{};
+	};
+
+	std::vector<CodeSection>& codeSections()
+	{
+		return m_codeSections;
+	}
+
+	std::vector<CodeSection> const& codeSections() const
+	{
+		return m_codeSections;
+	}
+
 protected:
 	/// Does the same operations as @a optimise, but should only be applied to a sub and
 	/// returns the replaced tags. Also takes an argument containing the tags of this assembly
 	/// that are referenced in a super-assembly.
 	std::map<u256, u256> const& optimiseInternal(OptimiserSettings const& _settings, std::set<size_t> _tagsReferencedFromOutside);
 
+	/// For EOF and legacy it calculates approximate size of "pure" code without data.
 	unsigned codeSize(unsigned subTagSize) const;
 
 	/// Add all assembly items from given JSON array. This function imports the items by iterating through
@@ -210,6 +226,11 @@ class Assembly
 
 	std::shared_ptr<std::string const> sharedSourceName(std::string const& _name) const;
 
+	void appendEOFHeader(bytes& bytecode, std::vector<size_t>& codeSectionSizeOffsets, size_t& startOfContainerSectionHeader, size_t& dataSectionSizeOffset) const;
+
+	LinkerObject const& assembleLegacy() const;
+	LinkerObject const& assembleEOF() const;
+
 protected:
 	/// 0 is reserved for exception
 	unsigned m_usedTags = 1;
@@ -223,11 +244,12 @@ class Assembly
 	};
 
 	std::map<std::string, NamedTagInfo> m_namedTags;
-	AssemblyItems m_items;
 	std::map<util::h256, bytes> m_data;
 	/// Data that is appended to the very end of the contract.
 	bytes m_auxiliaryData;
 	std::vector<std::shared_ptr<Assembly>> m_subs;
+	std::vector<CodeSection> m_codeSections;
+	uint16_t m_currentCodeSection = 0;
 	std::map<util::h256, std::string> m_strings;
 	std::map<util::h256, std::string> m_libraries; ///< Identifiers of libraries to be linked.
 	std::map<util::h256, std::string> m_immutables; ///< Identifiers of immutables.

libevmasm/Assembly.h

@@ -35,46 +35,49 @@ unsigned ConstantOptimisationMethod::optimiseConstants(
 )
 {
 	// TODO: design the optimiser in a way this is not needed
-	AssemblyItems& _items = _assembly.items();
-
 	unsigned optimisations = 0;
-	std::map<AssemblyItem, size_t> pushes;
-	for (AssemblyItem const& item: _items)
-		if (item.type() == Push)
-			pushes[item]++;
-	std::map<u256, AssemblyItems> pendingReplacements;
-	for (auto it: pushes)
+	for (auto& codeSection: _assembly.codeSections())
 	{
-		AssemblyItem const& item = it.first;
-		if (item.data() < 0x100)
-			continue;
-		Params params;
-		params.multiplicity = it.second;
-		params.isCreation = _isCreation;
-		params.runs = _runs;
-		params.evmVersion = _evmVersion;
-		LiteralMethod lit(params, item.data());
-		bigint literalGas = lit.gasNeeded();
-		CodeCopyMethod copy(params, item.data());
-		bigint copyGas = copy.gasNeeded();
-		ComputeMethod compute(params, item.data());
-		bigint computeGas = compute.gasNeeded();
-		AssemblyItems replacement;
-		if (copyGas < literalGas && copyGas < computeGas)
-		{
-			replacement = copy.execute(_assembly);
-			optimisations++;
-		}
-		else if (computeGas < literalGas && computeGas <= copyGas)
+		AssemblyItems& _items = codeSection.items;
+
+		std::map<AssemblyItem, size_t> pushes;
+		for (AssemblyItem const& item: _items)
+			if (item.type() == Push)
+				pushes[item]++;
+		std::map<u256, AssemblyItems> pendingReplacements;
+		for (auto it: pushes)
 		{
-			replacement = compute.execute(_assembly);
-			optimisations++;
+			AssemblyItem const& item = it.first;
+			if (item.data() < 0x100)
+				continue;
+			Params params;
+			params.multiplicity = it.second;
+			params.isCreation = _isCreation;
+			params.runs = _runs;
+			params.evmVersion = _evmVersion;
+			LiteralMethod lit(params, item.data());
+			bigint literalGas = lit.gasNeeded();
+			CodeCopyMethod copy(params, item.data());
+			bigint copyGas = copy.gasNeeded();
+			ComputeMethod compute(params, item.data());
+			bigint computeGas = compute.gasNeeded();
+			AssemblyItems replacement;
+			if (copyGas < literalGas && copyGas < computeGas)
+			{
+				replacement = copy.execute(_assembly);
+				optimisations++;
+			}
+			else if (computeGas < literalGas && computeGas <= copyGas)
+			{
+				replacement = compute.execute(_assembly);
+				optimisations++;
+			}
+			if (!replacement.empty())
+				pendingReplacements[item.data()] = replacement;
 		}
-		if (!replacement.empty())
-			pendingReplacements[item.data()] = replacement;
+		if (!pendingReplacements.empty())
+			replaceConstants(_items, pendingReplacements);
 	}
-	if (!pendingReplacements.empty())
-		replaceConstants(_items, pendingReplacements);
 	return optimisations;
 }
 

libevmasm/ConstantOptimiser.cpp

@@ -56,12 +56,16 @@ void EVMAssemblyStack::assemble()
 	solAssert(!m_evmRuntimeAssembly);
 
 	m_object = m_evmAssembly->assemble();
-	m_sourceMapping = AssemblyItem::computeSourceMapping(m_evmAssembly->items(), sourceIndices());
+	// TODO: Check for EOF
+	solAssert(m_evmAssembly->codeSections().size() == 1);
+	m_sourceMapping = AssemblyItem::computeSourceMapping(m_evmAssembly->codeSections().front().items, sourceIndices());
 	if (m_evmAssembly->numSubs() > 0)
 	{
 		m_evmRuntimeAssembly = std::make_shared<evmasm::Assembly>(m_evmAssembly->sub(0));
 		solAssert(m_evmRuntimeAssembly && !m_evmRuntimeAssembly->isCreation());
-		m_runtimeSourceMapping = AssemblyItem::computeSourceMapping(m_evmRuntimeAssembly->items(), sourceIndices());
+		// TODO: Check for EOF
+		solAssert(m_evmRuntimeAssembly->codeSections().size() == 1);
+		m_runtimeSourceMapping = AssemblyItem::computeSourceMapping(m_evmRuntimeAssembly->codeSections().front().items, sourceIndices());
 		m_runtimeObject = m_evmRuntimeAssembly->assemble();
 	}
 }

libevmasm/EVMAssemblyStack.cpp

@@ -808,15 +808,19 @@ evmasm::AssemblyItems const* CompilerStack::assemblyItems(std::string const& _co
 	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
 
 	Contract const& currentContract = contract(_contractName);
-	return currentContract.evmAssembly ? &currentContract.evmAssembly->items() : nullptr;
+	if (currentContract.evmAssembly)
+		solAssert(currentContract.evmAssembly->codeSections().size() == 1, "Expected a single code section in legacy codegen.");
+	return currentContract.evmAssembly ? &currentContract.evmAssembly->codeSections().front().items : nullptr;
 }
 
 evmasm::AssemblyItems const* CompilerStack::runtimeAssemblyItems(std::string const& _contractName) const
 {
 	solAssert(m_stackState == CompilationSuccessful, "Compilation was not successful.");
 
 	Contract const& currentContract = contract(_contractName);
-	return currentContract.evmRuntimeAssembly ? &currentContract.evmRuntimeAssembly->items() : nullptr;
+	if (currentContract.evmRuntimeAssembly)
+		solAssert(currentContract.evmRuntimeAssembly->codeSections().size() == 1, "Expected a single code section in legacy codegen.");
+	return currentContract.evmRuntimeAssembly ? &currentContract.evmRuntimeAssembly->codeSections().front().items : nullptr;
 }
 
 Json CompilerStack::generatedSources(std::string const& _contractName, bool _runtime) const

libsolidity/interface/CompilerStack.cpp

@@ -262,9 +262,11 @@ YulStack::assembleWithDeployed(std::optional<std::string_view> _deployName)
 		creationObject.bytecode = std::make_shared<evmasm::LinkerObject>(creationAssembly->assemble());
 		yulAssert(creationObject.bytecode->immutableReferences.empty(), "Leftover immutables.");
 		creationObject.assembly = creationAssembly;
+		solAssert(creationAssembly->codeSections().size() == 1);
 		creationObject.sourceMappings = std::make_unique<std::string>(
+			// TODO: fix for EOF
 			evmasm::AssemblyItem::computeSourceMapping(
-				creationAssembly->items(),
+				creationAssembly->codeSections().front().items,
 				{{m_charStream->name(), 0}}
 			)
 		);
@@ -273,11 +275,12 @@ YulStack::assembleWithDeployed(std::optional<std::string_view> _deployName)
 		{
 			deployedObject.bytecode = std::make_shared<evmasm::LinkerObject>(deployedAssembly->assemble());
 			deployedObject.assembly = deployedAssembly;
+			solAssert(deployedAssembly->codeSections().size() == 1);
 			deployedObject.sourceMappings = std::make_unique<std::string>(
 				evmasm::AssemblyItem::computeSourceMapping(
-					deployedAssembly->items(),
+					deployedAssembly->codeSections().front().items,
 					{{m_charStream->name(), 0}}
-				)
+					)
 			);
 		}
 	}

libyul/YulStack.cpp

@@ -270,7 +270,8 @@ BOOST_AUTO_TEST_CASE(immutables_and_its_source_maps)
 
 			checkCompilation(assembly);
 
-			std::string const sourceMappings = AssemblyItem::computeSourceMapping(assembly.items(), indices);
+			BOOST_REQUIRE(assembly.codeSections().size() == 1);
+			std::string const sourceMappings = AssemblyItem::computeSourceMapping(assembly.codeSections().at(0).items, indices);
 			auto const numberOfMappings = std::count(sourceMappings.begin(), sourceMappings.end(), ';');
 
 			LinkerObject const& obj = assembly.assemble();

test/libevmasm/Assembler.cpp

@@ -1381,16 +1381,18 @@ BOOST_AUTO_TEST_CASE(jumpdest_removal_subassemblies)
 		t1.toSubAssemblyTag(subId).pushTag(),
 		u256(8)
 	};
+	BOOST_REQUIRE(main.codeSections().size() == 1);
 	BOOST_CHECK_EQUAL_COLLECTIONS(
-		main.items().begin(), main.items().end(),
+		main.codeSections().at(0).items.begin(),main.codeSections().at(0).items.end(),
 		expectationMain.begin(), expectationMain.end()
 	);
 
 	AssemblyItems expectationSub{
 		u256(1), t1.tag(), u256(2), Instruction::JUMP, t4.tag(), u256(7), t4.pushTag(), Instruction::JUMP
 	};
+	BOOST_REQUIRE(sub->codeSections().size() == 1);
 	BOOST_CHECK_EQUAL_COLLECTIONS(
-		sub->items().begin(), sub->items().end(),
+		sub->codeSections().at(0).items.begin(), sub->codeSections().at(0).items.end(),
 		expectationSub.begin(), expectationSub.end()
 	);
 }

test/libevmasm/Optimiser.cpp

@@ -91,7 +91,8 @@ evmasm::AssemblyItems compileContract(std::shared_ptr<CharStream> _sourceCode)
 			);
 			compiler.compileContract(*contract, std::map<ContractDefinition const*, std::shared_ptr<Compiler const>>{}, bytes());
 
-			return compiler.runtimeAssembly().items();
+			BOOST_REQUIRE(compiler.runtimeAssembly().codeSections().size() == 1);
+			return compiler.runtimeAssembly().codeSections().at(0).items;
 		}
 	BOOST_FAIL("No contract found in source.");
 	return AssemblyItems();