Add the batch of release v3.3.0 commits

CMakeLists.txt

@@ -1,13 +1,14 @@
 # Copyright (c) 2012-2018, The CryptoNote developers, The Bytecoin developers.
 # Licensed under the GNU Lesser General Public License. See LICENSE for details.
 
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.0)
 project(bytecoin)
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_C_STANDARD 11)
 # message(STATUS "Bytecoind profile: According to cmake, sizeof(void *) == " ${CMAKE_SIZEOF_VOID_P})
 option(USE_INSTRUMENTATION "For testing - builds with address sanitizer instrument" OFF)
 option(WITH_THREAD_SANITIZER "For testing - builds with thread sanitizer instrument, USE_INSTRUMENTATION must be also set" OFF)
+option(BETTER_DEBUG "Disables optimizations. We do not use standard debug/realease configurations because they change too much" OFF)
 option(USE_SSL "Builds with support of https between walletd and bytecoind" ON)
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)
     option(USE_SQLITE "Builds with SQLite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" OFF)
@@ -17,12 +18,20 @@ else()
     set(OPENSSL_ROOT ../openssl32)
 endif()
 if(WIN32)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+#    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+#    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
     add_definitions(-D_SCL_SECURE_NO_WARNINGS=1 -D_CRT_SECURE_NO_WARNINGS=1 -D_WIN32_WINNT=0x0501)
 else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes -g -Wall -Wextra -Werror=return-type -Wno-unused-parameter")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -g -Wall -Wextra -Werror=return-type -Wno-unused-parameter")
+    if(BETTER_DEBUG)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0")
+    else()
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2")
+    endif()
+    message(STATUS "Better debug: " ${BETTER_DEBUG})
     message(STATUS "Instrumentation usage: " ${USE_INSTRUMENTATION})
     message(STATUS "Thread sanitizer usage: " ${WITH_THREAD_SANITIZER})
     if(USE_INSTRUMENTATION)
@@ -58,32 +67,22 @@ else()
     message(STATUS "SSL usage: OFF")
 endif()
 file(GLOB SRC_CRYPTO
-    src/crypto/*.cpp src/crypto/*.hpp src/crypto/*.h
-    src/crypto/blake256.c
-    src/crypto/chacha8.c
-    src/crypto/crypto-ops-data.c
-    src/crypto/crypto-ops.c
-    src/crypto/crypto-util.c
-    src/crypto/groestl.c
-    src/crypto/hash-extra-blake.c
-    src/crypto/hash-extra-groestl.c
-    src/crypto/hash-extra-jh.c
-    src/crypto/hash-extra-skein.c
-    src/crypto/hash.c
-    src/crypto/jh.c
-    src/crypto/keccak.c
-    src/crypto/oaes_lib.c
-    src/crypto/random.c
-    src/crypto/skein.c
-    src/crypto/slow-hash_x86.c
-    src/crypto/tree-hash.c
+        src/crypto/*.cpp src/crypto/*.hpp src/crypto/*.h
+        src/crypto/*.c src/crypto/*.h
+        src/crypto/bernstein/*.h src/crypto/bernstein/*.c
+        src/crypto/blake/*.h src/crypto/blake/*.c
+        src/crypto/groestl/*.h src/crypto/groestl/*.c
+        src/crypto/jh/*.h src/crypto/jh/*.c
+        src/crypto/keccak/*.c
+        src/crypto/oaes/*.h src/crypto/oaes/*.c
+        src/crypto/skein/*.h src/crypto/skein/*.c
 )
 file(GLOB SRC_COMMON src/common/*.cpp src/common/*.hpp)
 file(GLOB SRC_SERIALIZATION src/Serialization/*.cpp src/Serialization/*.hpp)
 file(GLOB SRC_SERIA src/seria/*.cpp src/seria/*.hpp)
 file(GLOB SRC_LOGGING src/logging/*.cpp src/logging/*.hpp)
 file(GLOB SRC_P2P src/p2p/*.cpp src/p2p/*.hpp)
-file(GLOB SRC_CORE src/Core/*.cpp src/Core/*.hpp)
+file(GLOB SRC_CORE src/Core/*.cpp src/Core/*.hpp src/CryptoNote.hpp src/CryptoNote.cpp src/rpc_api.hpp src/rpc_api.cpp)
 file(GLOB SRC_HTTP src/http/*.cpp src/http/*.hpp)
 file(GLOB SRC_PLATFORM
     src/platform/ExclusiveLock.cpp src/platform/ExclusiveLock.hpp

ReleaseNotes.md

@@ -1,5 +1,95 @@
 ## Release Notes
 
+### v3.3.0
+
+*Consensus update (hard fork)*
+
+- Voting starts immediately, once 90% of mined blocks contain votes for update, the update height will be automatically selected so that consensus update will happen approximately 2 weeks after that.
+- Market fees - any transaction fee including 0 is now legal for all transactions. Miners will increase block size only if it is profitable for them in a short run.
+
+*General improvements*
+
+- Better priority and exclusive nodes logic.
+- Seed nodes are now contacted approximately once per day (greatly helps to catch up after `bytecoind` is started for users who run it after delay of several weeks or more).
+- Binary methods now share single access point `/binary_rpc`.
+- Code for consensus upgrade voting correctly counts votes on both main chain and each side chain.
+- Limited on incoming connections (default is 100).
+- Fixed ignored external port on peer handshake (made connects through exposed non-standard ports to nodes behind NAT impossible).
+- Groestl hash function is updated from the official source.
+- Keccak permutation function is updated from the official source.
+- `bytecoind` never searches for `blocks.bin` and `blockindex.bin` outside the data folder.
+- now when you specify `--p2p-bind-address`, but not `--p2p-external-port`, p2p external port will be set to p2p bind port. When you wish NAT tunneling, good practice is to specify both.
+
+*Command line changes/additions*
+
+- Paranoid mode to check every byte of blockchain when downloading (usually checks only blocks beyond the last checkpoint).
+- *Warning:* Now `walletd` exits by default after `--create-wallet` and `--set-password` operations. This change can break your scripts. If you need the wallet running after those commands, you can add `--launch-after-command` parameter
+- Now you can use `--set-password` with `--export-view-only` and `--backup-wallet-data`, to encrypt result wallet with a different password.
+- Fixed bug with `walletd` not returning `api::WALLETD_BIND_PORT_IN_USE` error code when the JSON API port is in use and using inproc `bytecoind`.
+- Fixed bug with `bytecoind` not returning `api::BYTECOIND_BIND_PORT_IN_USE` error code when the JSON API port is in use.
+- `walletd` now prints address after creation of wallet.
+- `walletd` now prints deprecation warning when using inproc `bytecoind`.
+- `walletd` now binds to port `9070` on testnet and `10070` on stagenet resp. by default.
+
+*General API improvements*
+
+- Optimisation of JSON RPC calls (2x speed up on very large responses).
+- Made the `jsonrpc` argument mandatory with value "2.0" in all JSON RPC calls according to the spec.
+- JSON RPC `id` is required now according to spec.
+- JSON RPC error's additional data moved into `data` object inside the `error` object according to spec.
+- Now any field in requests that daemons do not understand will be reported as a error.
+- Much better error handling, more specific error codes.
+
+*Specific API improvements*
+
+- New `get_block_header` method for blockchain structure inspection (to replace `getblockheaderbyhash`, `getblockheaderbyheight`, and `getlastblockheader` legacy methods).
+- New `get_wallet_info` method.
+- New `VIEW_ONLY_WALLET` (`-304`) error code, returned from `create_transaction`.
+- In methods supporting longpoll (`get_statu`s and `get_block_template`) all longpoll arguments are now optional. So, for example, if you are interested in `outgoing_peer_count` only, you can specify only `outgoing_peer_count` in request and get response when `outgoing_peer_count` changes. Changes to other fields will not trigger response to longpoll.
+- New `ADDRESS_FAILED_TO_PARSE` (`-4`) and `ADDRESS_NOT_IN_WALLET` (`-1002`) error codes, returned from lots of methods
+- New fields in get_addresses request/response to iterate through list of addresses
+- Now `top_block_timestamp_median` returned correctly from get_status JSON RPC methods.
+- New `need_signatures` fields in APIs returning raw transactions.
+- `check_sendproof` now returns values from sendproof in response if proof is valid.
+- All methods return new correct values for `block_size` and `transactions_cumulative_size`.
+- `get_raw_block` and `get_block_header` now return `orphan_status` and `depth` (consistent with `height_or_depth` fields where top block is `-1`).
+- `get_random_amounts` has no more depth limit of 128 block (distribution would be skewed a bit for very large depths).
+- `get_statistics` response now includes much more information.
+- `submit_block` now returns `block_header` in result.
+- All `transfer` objects now have `transaction_hash` field - especially useful when processing `unlocked_transfers` in result of `get_transfers` method.
+
+*API deprecations (will be removed in version 3.4.0)*
+
+- In all `output` and `transaction` objects `unlock_time` is deprecated (renamed to `unlock_block_or_timestamp`).
+- In all `output` objects `global_index` deprecated (renamed to `index`).
+- In `get_random_outputs` request `outs_count` is deprecated (renamed to `output_count`).
+- In `get_transfers` request `desired_transactions_count` is deprecated (renamed to `desired_transaction_count`).
+- In all `transaction` objects 'binary_size' is deprecated (renamed to `size`).
+
+*Incompatible API changes*
+
+- `get_raw_transaction` method now returns json error `-5` if transaction not found.
+- Deprecated `prev_hash` field remains only in result of legacy methods (`getblockheaderbyhash`, `getblockheaderbyheight`, and `getlastblockheader` legacy methods), use 'previous_block_hash' instead.
+- Deprecated `total_fee_amount` field remains only in result of legacy methods (`getblockheaderbyhash`, `getblockheaderbyheight`, and `getlastblockheader`), use `transactions_fee` instead.
+- Deprecated `transactions_cumulative_size` field remains only in result of legacy methods (`getblockheaderbyhash`, `getblockheaderbyheight`, and `getlastblockheader`), use `transactions_size` instead.
+
+*Incompatible API changes (likely to affect only developers of block explorers)*
+
+- In all raw block objects `global_indices` renamed to `output_indexes`.
+- In all raw transaction objects `vin`, `vout` renamed to `inputs`, `outputs` resp. 
+- In all raw output objects `key` renamed to `public_key`.
+- In all raw output objects `target` object removed and all its fields moved into raw output object.
+- In all raw coinbase input objects `block_index` renamed to `height`.
+- In all raw header objects (including `parent_block`) `miner_tx`, `base_transaction_branch` renamed to `coinbase_transaction`, `coinbase_transaction_branch` resp.
+- In all raw input and raw output objects, `tag`:`ff` renamed to `type`:`coinbase` and `tag`:`02` renamed to `type`:`key`.
+
+*Testnet/Stagenet related*
+
+- New command line parameter `--net=test|stage|main` configures daemons for testnet, stagenet, or mainnet resp.
+- For testnet time multiplier can now be set to speed up all processes 10x, 100x or even more.
+- When participating in testnet or stagenet, `bytecoind` now uses UDP Multicast to announce/discover other bytecoind nodes in local network. Thus in most local networks testnet will self-assemble without seed nodes. In mainnet multicasts are disabled due to anonymity concerns.
+- Testnet/Stagenet now have fixed 1MB max block size limit.
+
 ### v3.2.4
 
 - Added the testnet functionality.
@@ -97,7 +187,6 @@
 - Added early support of JSON-RPC API basic authentification that prevents CSRF attacks.
 - Added (experimental) support of 32-bit platforms.
 
-
 ### v3.0.0-beta-20180206
 
 - Project is moved to the new public GitHub repository.

src/Core/Archive.cpp

@@ -23,12 +23,12 @@ const std::string Archive::BLOCK("b");
 const std::string Archive::TRANSACTION("t");
 const std::string Archive::CHECKPOINT("c");
 
-//static const float DB_COMMIT_PERIOD = 60;  // 1 minute sounds good for archive
+// static const float DB_COMMIT_PERIOD = 60;  // 1 minute sounds good for archive
 
 Archive::Archive(bool read_only, const std::string &path)
- : read_only(read_only)
+    : read_only(read_only)
 // , commit_timer(std::bind(&Archive::db_commit, this))
- {
+{
 #if !platform_USE_SQLITE
 	try {
 		m_db = std::make_unique<DB>(read_only, path);
@@ -49,7 +49,7 @@ Archive::Archive(bool read_only, const std::string &path)
 			throw;
 	}
 #endif
-//	commit_timer.once(DB_COMMIT_PERIOD);
+	//	commit_timer.once(DB_COMMIT_PERIOD);
 }
 
 // struct Record {
@@ -83,17 +83,16 @@ void Archive::db_commit() {
 	if (!m_db || read_only)
 		return;
 	m_db->commit_db_txn();
-//	commit_timer.once(DB_COMMIT_PERIOD);
+	//	commit_timer.once(DB_COMMIT_PERIOD);
 }
 
 void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytecoind::GetArchive::Response &resp) {
-	if (req.archive_id != unique_id) {
-		api::bytecoind::GetArchive::Error err;
-		err.code       = api::bytecoind::GetArchive::WRONG_ARCHIVE_ID;
-		err.message    = "Archive id changed";
-		err.archive_id = unique_id;
-		throw err;
-	}
+	if (unique_id.empty())
+		throw api::bytecoind::GetArchive::Error(
+		    api::bytecoind::GetArchive::ARCHIVE_NOT_ENABLED, "Archive was never enabled on this bytecoind", unique_id);
+	if (req.archive_id != unique_id)
+		throw api::bytecoind::GetArchive::Error(
+		    api::bytecoind::GetArchive::WRONG_ARCHIVE_ID, "Archive id changed", unique_id);
 	resp.from_record = req.from_record;
 	if (resp.from_record > next_record_id)
 		resp.from_record = next_record_id;
@@ -109,7 +108,7 @@ void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytec
 		api::bytecoind::GetArchive::ArchiveRecord rec;
 		seria::from_binary(rec, cur.get_value_array());
 		resp.records.push_back(rec);
-		if(req.records_only)
+		if (req.records_only)
 			continue;
 		std::string str_hash = common::pod_to_hex(rec.hash);
 		const auto hash_key  = HASHES_PREFIX + DB::to_binary_key(rec.hash.data, sizeof(rec.hash.data));
@@ -124,6 +123,9 @@ void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytec
 				invariant(block.from_raw_block(raw_block), "");
 				bl.raw_header = block.header;
 				bl.raw_transactions.reserve(block.transactions.size());
+				bl.transaction_binary_sizes.reserve(block.transactions.size() + 1);
+				auto coinbase_size = static_cast<uint32_t>(seria::binary_size(block.header.base_transaction));
+				bl.transaction_binary_sizes.push_back(coinbase_size);
 				for (size_t i = 0; i != block.transactions.size(); ++i) {
 					bl.raw_transactions.push_back(static_cast<TransactionPrefix &>(block.transactions.at(i)));
 					bl.transaction_binary_sizes.push_back(static_cast<uint32_t>(raw_block.transactions.at(i).size()));
@@ -145,7 +147,7 @@ void Archive::read_archive(api::bytecoind::GetArchive::Request &&req, api::bytec
 			if (resp.checkpoints.count(str_hash) == 0) {
 				BinaryArray data;
 				invariant(m_db->get(hash_key, data), "");
-				SignedCheckPoint &ch = resp.checkpoints[str_hash];
+				SignedCheckpoint &ch = resp.checkpoints[str_hash];
 				seria::from_binary(ch, data);
 			}
 		}

src/Core/Archive.hpp

@@ -15,7 +15,7 @@ class Archive {
 	uint64_t next_record_id = 0;
 	std::string unique_id;
 
-//	platform::Timer commit_timer;
+	//	platform::Timer commit_timer;
 public:
 	explicit Archive(bool read_only, const std::string &path);
 	std::string get_unique_id() const { return unique_id; }