Skip to content

Commit 4a74978

Browse files
cyrossignolcyrossignol
committed
Refactor superblock for storage of magnitudes with greater precision
1 parent 3d6e068 commit 4a74978

File tree

3 files changed

+1254
-497
lines changed

3 files changed

+1254
-497
lines changed

‎src/neuralnet/superblock.cpp‎

Lines changed: 155 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,9 @@ class ScraperStatsQuorumHasher
171171
struct HasherProxy
172172
{
173173
CHashWriter m_hasher; //!< Hashes the supplied data.
174+
CHashWriter m_small_hasher; //!< Hashes small mag segment.
175+
CHashWriter m_medium_hasher; //!< Hashes medium mag segment.
176+
CHashWriter m_large_hasher; //!< Hashes large mag segment.
174177
uint32_t m_zero_magnitude_count; //!< Tracks zero-magnitude CPIDs.
175178
bool m_zero_magnitude_hashed; //!< Tracks when to hash the zeros.
176179

@@ -180,6 +183,9 @@ class ScraperStatsQuorumHasher
180183
//!
181184
HasherProxy()
182185
: m_hasher(CHashWriter(SER_GETHASH, PROTOCOL_VERSION))
186+
, m_small_hasher(CHashWriter(SER_GETHASH, PROTOCOL_VERSION))
187+
, m_medium_hasher(CHashWriter(SER_GETHASH, PROTOCOL_VERSION))
188+
, m_large_hasher(CHashWriter(SER_GETHASH, PROTOCOL_VERSION))
183189
, m_zero_magnitude_count(0)
184190
, m_zero_magnitude_hashed(false)
185191
{
@@ -192,13 +198,29 @@ class ScraperStatsQuorumHasher
192198
//! \param cpid The CPID value to hash.
193199
//! \param magnitude The magnitude value to hash.
194200
//!
195-
void Add(Cpid cpid, uint16_t magnitude)
201+
void Add(const Cpid cpid, const Magnitude magnitude)
196202
{
197-
if (magnitude > 0) {
198-
m_hasher << cpid;
199-
WriteCompactSize(m_hasher, magnitude);
200-
} else {
201-
m_zero_magnitude_count++;
203+
switch (magnitude.Which()) {
204+
case Magnitude::Kind::ZERO:
205+
m_zero_magnitude_count++;
206+
break;
207+
208+
case Magnitude::Kind::SMALL:
209+
m_small_hasher
210+
<< cpid
211+
<< static_cast<uint8_t>(magnitude.Compact());
212+
break;
213+
214+
case Magnitude::Kind::MEDIUM:
215+
m_medium_hasher
216+
<< cpid
217+
<< static_cast<uint8_t>(magnitude.Compact());
218+
break;
219+
220+
case Magnitude::Kind::LARGE:
221+
m_large_hasher << cpid;
222+
WriteCompactSize(m_large_hasher, magnitude.Compact());
223+
break;
202224
}
203225
}
204226

@@ -211,7 +233,7 @@ class ScraperStatsQuorumHasher
211233
//!
212234
void RoundAndAdd(Cpid cpid, double magnitude)
213235
{
214-
Add(cpid, std::nearbyint(magnitude));
236+
Add(cpid, Magnitude::RoundFrom(magnitude));
215237
}
216238

217239
//!
@@ -229,7 +251,14 @@ class ScraperStatsQuorumHasher
229251
// first project:
230252
//
231253
if (!m_zero_magnitude_hashed) {
232-
m_hasher << VARINT(m_zero_magnitude_count);
254+
m_hasher
255+
<< (CHashWriter(SER_GETHASH, PROTOCOL_VERSION)
256+
<< m_small_hasher.GetHash()
257+
<< m_medium_hasher.GetHash()
258+
<< m_large_hasher.GetHash())
259+
.GetHash()
260+
<< VARINT(m_zero_magnitude_count);
261+
233262
m_zero_magnitude_hashed = true;
234263
}
235264

@@ -1272,7 +1301,7 @@ class LegacySuperblockParser
12721301
const size_t binary_size = m_binary_magnitudes.size();
12731302

12741303
for (size_t x = 0; x < binary_size && binary_size - x >= 18; x += 18) {
1275-
magnitudes.Add(
1304+
magnitudes.AddLegacy(
12761305
*reinterpret_cast<const Cpid*>(byte_ptr + x),
12771306
be16toh(*reinterpret_cast<const int16_t*>(byte_ptr + x + 16)));
12781307
}
@@ -1303,11 +1332,13 @@ class LegacySuperblockParser
13031332
continue;
13041333
}
13051334

1306-
try {
1307-
magnitudes.Add(MiningId::Parse(parts[0]), std::stoi(parts[1]));
1308-
} catch(...) {
1309-
if (fDebug) {
1310-
LogPrintf("LegacySuperblock: Failed to parse magnitude.");
1335+
if (const CpidOption cpid = MiningId::Parse(parts[0]).TryCpid()) {
1336+
try {
1337+
magnitudes.AddLegacy(*cpid, std::stoi(parts[1]));
1338+
} catch(...) {
1339+
if (fDebug) {
1340+
LogPrintf("LegacySuperblock: Failed to parse magnitude.");
1341+
}
13111342
}
13121343
}
13131344
}
@@ -1496,7 +1527,7 @@ std::string Superblock::PackLegacy() const
14961527
out << "<ZERO>" << m_cpids.Zeros() << "</ZERO>"
14971528
<< "<BINARY>";
14981529

1499-
for (const auto& cpid_pair : m_cpids) {
1530+
for (const auto& cpid_pair : m_cpids.Legacy()) {
15001531
uint16_t mag = htobe16(cpid_pair.second);
15011532

15021533
out.write(reinterpret_cast<const char*>(cpid_pair.first.Raw().data()), 16);
@@ -1564,22 +1595,55 @@ Superblock::CpidIndex::CpidIndex(uint32_t zero_magnitude_count)
15641595

15651596
Superblock::CpidIndex::const_iterator Superblock::CpidIndex::begin() const
15661597
{
1567-
return m_magnitudes.begin();
1598+
if (m_legacy) {
1599+
return const_iterator(
1600+
NN::Magnitude::SCALE_FACTOR,
1601+
m_legacy_magnitudes.begin(),
1602+
m_legacy_magnitudes.end());
1603+
}
1604+
1605+
return const_iterator(
1606+
decltype(m_small_magnitudes)::SCALE_FACTOR,
1607+
m_small_magnitudes.begin(),
1608+
m_small_magnitudes.end(),
1609+
const_iterator(
1610+
decltype(m_medium_magnitudes)::SCALE_FACTOR,
1611+
m_medium_magnitudes.begin(),
1612+
m_medium_magnitudes.end(),
1613+
const_iterator(
1614+
decltype(m_large_magnitudes)::SCALE_FACTOR,
1615+
m_large_magnitudes.begin(),
1616+
m_large_magnitudes.end())));
15681617
}
15691618

15701619
Superblock::CpidIndex::const_iterator Superblock::CpidIndex::end() const
15711620
{
1572-
return m_magnitudes.end();
1621+
if (m_legacy) {
1622+
return const_iterator(m_legacy_magnitudes.end());
1623+
}
1624+
1625+
return const_iterator(m_large_magnitudes.end());
15731626
}
15741627

15751628
Superblock::CpidIndex::size_type Superblock::CpidIndex::size() const
15761629
{
1577-
return m_magnitudes.size();
1630+
if (m_legacy) {
1631+
return m_legacy_magnitudes.size();
1632+
}
1633+
1634+
return m_small_magnitudes.size()
1635+
+ m_medium_magnitudes.size()
1636+
+ m_large_magnitudes.size();
15781637
}
15791638

15801639
bool Superblock::CpidIndex::empty() const
15811640
{
1582-
return m_magnitudes.empty();
1641+
return size() == 0;
1642+
}
1643+
1644+
const Superblock::MagnitudeStorageType& Superblock::CpidIndex::Legacy() const
1645+
{
1646+
return m_legacy_magnitudes;
15831647
}
15841648

15851649
uint32_t Superblock::CpidIndex::Zeros() const
@@ -1589,64 +1653,81 @@ uint32_t Superblock::CpidIndex::Zeros() const
15891653

15901654
size_t Superblock::CpidIndex::TotalCount() const
15911655
{
1592-
return m_magnitudes.size() + m_zero_magnitude_count;
1656+
return size() + m_zero_magnitude_count;
15931657
}
15941658

1595-
uint64_t Superblock::CpidIndex::TotalMagnitude() const
1659+
double Superblock::CpidIndex::TotalMagnitude() const
15961660
{
1597-
return m_total_magnitude;
1661+
return static_cast<double>(m_total_magnitude) / NN::Magnitude::SCALE_FACTOR;
15981662
}
15991663

16001664
double Superblock::CpidIndex::AverageMagnitude() const
16011665
{
1602-
if (m_magnitudes.empty()) {
1666+
if (empty()) {
16031667
return 0;
16041668
}
16051669

1606-
return static_cast<double>(m_total_magnitude) / m_magnitudes.size();
1670+
return TotalMagnitude() / size();
16071671
}
16081672

1609-
uint16_t Superblock::CpidIndex::MagnitudeOf(const Cpid& cpid) const
1673+
Magnitude Superblock::CpidIndex::MagnitudeOf(const Cpid& cpid) const
16101674
{
1611-
const auto iter = m_magnitudes.find(cpid);
1675+
if (m_legacy) {
1676+
const auto iter = m_legacy_magnitudes.find(cpid);
16121677

1613-
if (iter == m_magnitudes.end()) {
1614-
return 0;
1678+
if (iter == m_legacy_magnitudes.end()) {
1679+
return Magnitude::Zero();
1680+
}
1681+
1682+
return Magnitude::FromScaled(iter->second * NN::Magnitude::SCALE_FACTOR);
16151683
}
16161684

1617-
return iter->second;
1685+
if (const auto mag_option = m_small_magnitudes.MagnitudeOf(cpid)) {
1686+
return *mag_option;
1687+
}
1688+
1689+
if (const auto mag_option = m_medium_magnitudes.MagnitudeOf(cpid)) {
1690+
return *mag_option;
1691+
}
1692+
1693+
if (const auto mag_option = m_large_magnitudes.MagnitudeOf(cpid)) {
1694+
return *mag_option;
1695+
}
1696+
1697+
return Magnitude::Zero();
16181698
}
16191699

16201700
Superblock::CpidIndex::const_iterator
16211701
Superblock::CpidIndex::At(const size_t offset) const
16221702
{
16231703
// Not very efficient--we can optimize this if needed:
1624-
return std::next(m_magnitudes.begin(), offset);
1704+
return std::next(begin(), offset);
16251705
}
16261706

1627-
void Superblock::CpidIndex::Add(const Cpid cpid, const uint16_t magnitude)
1707+
void Superblock::CpidIndex::Add(const Cpid cpid, const Magnitude magnitude)
16281708
{
1629-
if (magnitude > 0 || m_legacy) {
1630-
// Only increment the total magnitude if the CPID does not already
1631-
// exist in the index:
1632-
if (m_magnitudes.emplace(cpid, magnitude).second == true) {
1633-
m_total_magnitude += magnitude;
1634-
}
1635-
} else {
1636-
m_zero_magnitude_count++;
1637-
}
1638-
}
1709+
// Only increment the total magnitude if the CPID does not already
1710+
// exist in the index:
1711+
switch (magnitude.Which()) {
1712+
case Magnitude::Kind::ZERO:
1713+
m_zero_magnitude_count++;
1714+
break;
16391715

1640-
void Superblock::CpidIndex::Add(const MiningId id, const uint16_t magnitude)
1641-
{
1642-
if (const CpidOption cpid = id.TryCpid()) {
1643-
Add(*cpid, magnitude);
1644-
} else if (!m_legacy) {
1645-
m_zero_magnitude_count++;
1716+
case Magnitude::Kind::SMALL:
1717+
m_total_magnitude += m_small_magnitudes.Add(cpid, magnitude);
1718+
break;
1719+
1720+
case Magnitude::Kind::MEDIUM:
1721+
m_total_magnitude += m_medium_magnitudes.Add(cpid, magnitude);
1722+
break;
1723+
1724+
case Magnitude::Kind::LARGE:
1725+
m_total_magnitude += m_large_magnitudes.Add(cpid, magnitude);
1726+
break;
16461727
}
16471728
}
16481729

1649-
void Superblock::CpidIndex::RoundAndAdd(const MiningId id, const double magnitude)
1730+
void Superblock::CpidIndex::RoundAndAdd(const Cpid cpid, const double magnitude)
16501731
{
16511732
// The ScraperGetNeuralContract() function that these classes replace
16521733
// rounded magnitude values using a half-away-from-zero rounding mode
@@ -1658,13 +1739,35 @@ void Superblock::CpidIndex::RoundAndAdd(const MiningId id, const double magnitud
16581739
// To create legacy superblocks from scraper statistics with matching
16591740
// hashes, we filter magnitudes using the same rounding rules:
16601741
//
1661-
if (!m_legacy || std::round(magnitude) > 0) {
1662-
Add(id, std::nearbyint(magnitude));
1742+
if (m_legacy) {
1743+
if (std::round(magnitude) > 0) {
1744+
AddLegacy(cpid, std::nearbyint(magnitude));
1745+
} else {
1746+
m_zero_magnitude_count++;
1747+
}
16631748
} else {
1664-
m_zero_magnitude_count++;
1749+
Add(cpid, Magnitude::RoundFrom(magnitude));
1750+
}
1751+
}
1752+
1753+
void Superblock::CpidIndex::AddLegacy(const Cpid cpid, const uint16_t magnitude)
1754+
{
1755+
if (m_legacy_magnitudes.emplace(cpid, magnitude).second == true) {
1756+
m_total_magnitude += magnitude * NN::Magnitude::SCALE_FACTOR;
16651757
}
16661758
}
16671759

1760+
uint256 Superblock::CpidIndex::HashSegments() const
1761+
{
1762+
CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION);
1763+
1764+
hasher << SerializeHash(m_small_magnitudes);
1765+
hasher << SerializeHash(m_medium_magnitudes);
1766+
hasher << SerializeHash(m_large_magnitudes);
1767+
1768+
return hasher.GetHash();
1769+
}
1770+
16681771
// -----------------------------------------------------------------------------
16691772
// Class: Superblock::ProjectStats
16701773
// -----------------------------------------------------------------------------
@@ -1819,7 +1922,7 @@ QuorumHash QuorumHash::Hash(const Superblock& superblock)
18191922
std::string input;
18201923
input.reserve(superblock.m_cpids.size() * (32 + 1 + 5 + 5));
18211924

1822-
for (const auto& cpid_pair : superblock.m_cpids) {
1925+
for (const auto& cpid_pair : superblock.m_cpids.Legacy()) {
18231926
double dMagLength = RoundToString(cpid_pair.second, 0).length();
18241927
double dExponent = pow(dMagLength, 5);
18251928

0 commit comments

Comments
 (0)