From 8f27605fc3a1dde16c4e8fdd8b7991e6bd6fedd1 Mon Sep 17 00:00:00 2001 From: Om Doshi Date: Mon, 25 May 2026 13:00:46 +0100 Subject: [PATCH 1/5] Add CRC32 checksum verification for pushdata/fetchdata replication data channels (#581) During SM long-running tests, followers observed hash mismatches and invalid headers during read-verify. Root cause: PushData and FetchData channels transmitted raw payloads with no integrity check, allowing in-flight corruption to be silently written to disk. Changes: - Add checksum: uint32 to PushDataRequest and ResponseEntry FlatBuffer schemas - Add data_checksum_enabled: bool = true (hotswap) to Consensus config so the overhead can be toggled off at runtime for performance benchmarking - Compute CRC32 over the data payload before sending on both push and fetch paths - Verify CRC32 on receipt; drop and let retry on mismatch rather than writing corrupt data to disk - FetchData response now optionally prepends a FetchDataResponse FlatBuffer header carrying per-entry checksums, gated on data_checksum_enabled to ensure backward compatibility during rolling upgrades --- src/lib/common/homestore_config.fbs | 4 + src/lib/replication/fetch_data_rpc.fbs | 1 + src/lib/replication/push_data_rpc.fbs | 1 + .../replication/repl_dev/raft_repl_dev.cpp | 112 ++++++++++++++++-- 4 files changed, 110 insertions(+), 8 deletions(-) diff --git a/src/lib/common/homestore_config.fbs b/src/lib/common/homestore_config.fbs index 3d3ec3b0f..83129d93f 100644 --- a/src/lib/common/homestore_config.fbs +++ b/src/lib/common/homestore_config.fbs @@ -324,6 +324,10 @@ table Consensus { // Log frequency for gc_repl_reqs messages (log every N times) // 60 * repl_dev_cleanup_interval_sec (60s) means every 1 hour we will log the gc repl reqs info. gc_repl_reqs_log_frequency: uint32 = 60 (hotswap); + + // Enable CRC32 checksum verification on pushdata/fetchdata payloads. + // Disable only for performance benchmarking; leaving it off in production risks silent data corruption. + data_checksum_enabled: bool = true (hotswap); } table HomeStoreSettings { diff --git a/src/lib/replication/fetch_data_rpc.fbs b/src/lib/replication/fetch_data_rpc.fbs index e809cde42..044e188b1 100644 --- a/src/lib/replication/fetch_data_rpc.fbs +++ b/src/lib/replication/fetch_data_rpc.fbs @@ -19,6 +19,7 @@ table ResponseEntry { dsn : uint64; // Data Sequence number raft_term : uint64; // Raft term number data_size : uint32; // Size of the data which is sent as separate non flatbuffer + checksum: uint32; // CRC32 over the data for this entry; 0 when checksum is disabled } table FetchDataResponse { diff --git a/src/lib/replication/push_data_rpc.fbs b/src/lib/replication/push_data_rpc.fbs index d9a981e7c..5b31d0e1d 100644 --- a/src/lib/replication/push_data_rpc.fbs +++ b/src/lib/replication/push_data_rpc.fbs @@ -10,6 +10,7 @@ table PushDataRequest { user_key : [ubyte]; // User key data data_size : uint32; // Data size, actual data is sent as separate blob not by flatbuffer time_ms: uint64; // time point when originator pushed this request; + checksum: uint32; // CRC32 over the data payload; 0 when checksum is disabled } root_type PushDataRequest; diff --git a/src/lib/replication/repl_dev/raft_repl_dev.cpp b/src/lib/replication/repl_dev/raft_repl_dev.cpp index b83a9993b..8202d278d 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.cpp +++ b/src/lib/replication/repl_dev/raft_repl_dev.cpp @@ -1097,11 +1097,21 @@ void RaftReplDev::async_alloc_write(sisl::blob const& header, sisl::blob const& void RaftReplDev::push_data_to_all_followers(repl_req_ptr_t rreq, sisl::sg_list const& data) { auto& builder = rreq->create_fb_builder(); + // Compute CRC32 over the data payload before serialising so the follower can detect corruption in transit. + uint32_t checksum = 0; + if (HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled)) { + checksum = init_crc32; + for (auto const& iov : data.iovs) { + checksum = crc32_ieee(checksum, r_cast< const unsigned char* >(iov.iov_base), iov.iov_len); + } + } + // Prepare the rpc request packet with all repl_reqs details builder.FinishSizePrefixed(CreatePushDataRequest( builder, rreq->traceID(), server_id(), rreq->term(), rreq->dsn(), builder.CreateVector(rreq->header().cbytes(), rreq->header().size()), - builder.CreateVector(rreq->key().cbytes(), rreq->key().size()), data.size, get_time_since_epoch_ms())); + builder.CreateVector(rreq->key().cbytes(), rreq->key().size()), data.size, get_time_since_epoch_ms(), + checksum)); rreq->m_pkts = sisl::io_blob::sg_list_to_ioblob_list(data); rreq->m_pkts.insert(rreq->m_pkts.begin(), sisl::io_blob{builder.GetBufferPointer(), builder.GetSize(), false}); @@ -1165,6 +1175,18 @@ void RaftReplDev::on_push_data_received(intrusive< sisl::GenericRpcData >& rpc_d RD_LOGD(rkey.traceID, "Data Channel: PushData received: time diff={} ms.", get_elapsed_time_ms(req_orig_time_ms)); + if (HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled) && push_req->checksum() != 0) { + auto const data_ptr = r_cast< const unsigned char* >(incoming_buf.cbytes() + fb_size); + auto const computed = crc32_ieee(init_crc32, data_ptr, push_req->data_size()); + if (computed != push_req->checksum()) { + RD_LOGE(NO_TRACE_ID, + "Data Channel: PushData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}, dropping", + push_req->dsn(), push_req->checksum(), computed); + rpc_data->send_response(); + return; + } + } + #ifdef _PRERELEASE if (iomgr_flip::instance()->test_flip("drop_push_data_request")) { RD_LOGI(rkey.traceID, @@ -1582,20 +1604,48 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ RD_LOGT(NO_TRACE_ID, "Data Channel: FetchData data read completed for {} buffers", sgs_vec.size()); + // When checksums are enabled, prepend a size-prefixed FetchDataResponse FlatBuffer header that carries + // a per-entry CRC32. The header is omitted entirely when checksums are disabled so that nodes + // running old code (which have no notion of this header) can safely receive the response. + bool const compute_checksum = HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled); + uint8_t* hdr_buf = nullptr; + uint32_t hdr_size = 0; + + if (compute_checksum) { + flatbuffers::FlatBufferBuilder resp_builder; + std::vector< flatbuffers::Offset< ResponseEntry > > resp_entries; + for (auto const& sgs : sgs_vec) { + uint32_t checksum = init_crc32; + for (auto const& iov : sgs.iovs) { + checksum = crc32_ieee(checksum, r_cast< const unsigned char* >(iov.iov_base), iov.iov_len); + } + resp_entries.push_back( + CreateResponseEntry(resp_builder, 0, 0, 0, static_cast< uint32_t >(sgs.size), checksum)); + } + resp_builder.FinishSizePrefixed( + CreateFetchDataResponse(resp_builder, server_id(), resp_builder.CreateVector(resp_entries))); + hdr_size = resp_builder.GetSize(); + hdr_buf = iomanager.iobuf_alloc(512, hdr_size); + std::memcpy(hdr_buf, resp_builder.GetBufferPointer(), hdr_size); + } + // now prepare the io_blob_list to response back to requester; nuraft_mesg::io_blob_list_t pkts = sisl::io_blob_list_t{}; + if (hdr_buf) { pkts.emplace_back(sisl::io_blob{hdr_buf, hdr_size, true}); } for (auto const& sgs : sgs_vec) { auto const ret = sisl::io_blob::sg_list_to_ioblob_list(sgs); pkts.insert(pkts.end(), ret.begin(), ret.end()); } - rpc_data->set_comp_cb([sgs_vec = std::move(sgs_vec)](boost::intrusive_ptr< sisl::GenericRpcData >&) { - for (auto const& sgs : sgs_vec) { - for (auto const& iov : sgs.iovs) { - iomanager.iobuf_free(reinterpret_cast< uint8_t* >(iov.iov_base)); + rpc_data->set_comp_cb( + [sgs_vec = std::move(sgs_vec), hdr_buf](boost::intrusive_ptr< sisl::GenericRpcData >&) { + if (hdr_buf) { iomanager.iobuf_free(hdr_buf); } + for (auto const& sgs : sgs_vec) { + for (auto const& iov : sgs.iovs) { + iomanager.iobuf_free(reinterpret_cast< uint8_t* >(iov.iov_base)); + } } - } - }); + }); rpc_data->send_response(pkts); }); @@ -1614,11 +1664,57 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons COUNTER_INCREMENT(m_metrics, fetch_total_blk_size, total_size); + // When checksums are enabled, the sender prepends a size-prefixed FetchDataResponse FlatBuffer header. + // Gate all header parsing on the same config flag so nodes with checksums disabled (including old nodes + // that predate this feature) are fully compatible — they neither send nor expect the header. + bool const verify_checksum = HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled); + const flatbuffers::Vector< flatbuffers::Offset< ResponseEntry > >* resp_entries = nullptr; + + if (verify_checksum) { + auto const fb_hdr_size = + flatbuffers::ReadScalar< flatbuffers::uoffset_t >(raw_data) + sizeof(flatbuffers::uoffset_t); + if (fb_hdr_size > total_size) { + RD_LOGE(NO_TRACE_ID, + "Data Channel: FetchData response header size {} exceeds blob size {}, ignoring response", + fb_hdr_size, total_size); + return; + } + auto const fetch_resp = flatbuffers::GetSizePrefixedRoot< FetchDataResponse >(raw_data); + raw_data += fb_hdr_size; + total_size -= fb_hdr_size; + if (!fetch_resp || !fetch_resp->entries()) { + RD_LOGW(NO_TRACE_ID, + "Data Channel: FetchData response header is malformed, skipping checksum verification"); + } else { + resp_entries = fetch_resp->entries(); + if (resp_entries->size() != rreqs.size()) { + RD_LOGW(NO_TRACE_ID, + "Data Channel: FetchData response entry count {} != request count {}, " + "some entries will not be checksum-verified", + resp_entries->size(), rreqs.size()); + } + } + } + RD_LOGD(NO_TRACE_ID, "Data Channel: FetchData completed for {} requests", rreqs.size()); - for (auto const& rreq : rreqs) { + for (size_t i = 0; i < rreqs.size(); ++i) { + auto const& rreq = rreqs[i]; auto const data_size = rreq->remote_blkid().blkid.blk_count() * get_blk_size(); + if (resp_entries && i < resp_entries->size() && (*resp_entries)[i]->checksum() != 0) { + auto const computed = crc32_ieee(init_crc32, r_cast< const unsigned char* >(raw_data), data_size); + if (computed != (*resp_entries)[i]->checksum()) { + RD_LOGE(rreq->traceID(), + "Data Channel: FetchData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}, " + "skipping write to avoid corrupting storage", + rreq->dsn(), (*resp_entries)[i]->checksum(), computed); + raw_data += data_size; + total_size -= data_size; + continue; + } + } + if (!rreq->save_fetched_data(response, raw_data, data_size)) { RD_DBG_ASSERT(rreq->local_blkid().is_valid(), "Invalid blkid for rreq={}", rreq->to_string()); auto const local_size = rreq->local_blkid().blk_count() * get_blk_size(); From 09e52c779c15b2fe76dfb6ed7711c4cc97c0eabd Mon Sep 17 00:00:00 2001 From: Om Doshi Date: Wed, 27 May 2026 12:15:16 +0100 Subject: [PATCH 2/5] Refine CRC32 checksum implementation for safe rolling upgrades - Switch framing detection to magic-byte prefix (FETCH_DATA_RESPONSE_MAGIC) so receivers self-describe the header without consulting per-node config, making detection safe under hotswap config asymmetry between nodes - Default data_checksum_enabled to false; operators enable via hotswap after all nodes are running the new binary to avoid old nodes misinterpreting the framing header as block data - Add FlatBuffer verifier on the receive path to guard against malformed headers - Fix header memory management: heap-allocate with new/delete[] instead of iobuf - Attach per-entry lsn/dsn/raft_term to ResponseEntry for richer diagnostics - Increment data_checksum_mismatch_cnt metric on every mismatch - Add Checksum_Enabled_PushData_Path and Checksum_Enabled_FetchData_Path tests --- src/lib/common/homestore_config.fbs | 9 +- .../replication/repl_dev/raft_repl_dev.cpp | 99 +++++++++++++------ src/lib/replication/repl_dev/raft_repl_dev.h | 8 ++ src/tests/test_raft_repl_dev.cpp | 51 ++++++++++ 4 files changed, 134 insertions(+), 33 deletions(-) diff --git a/src/lib/common/homestore_config.fbs b/src/lib/common/homestore_config.fbs index 83129d93f..c537ddcad 100644 --- a/src/lib/common/homestore_config.fbs +++ b/src/lib/common/homestore_config.fbs @@ -325,9 +325,12 @@ table Consensus { // 60 * repl_dev_cleanup_interval_sec (60s) means every 1 hour we will log the gc repl reqs info. gc_repl_reqs_log_frequency: uint32 = 60 (hotswap); - // Enable CRC32 checksum verification on pushdata/fetchdata payloads. - // Disable only for performance benchmarking; leaving it off in production risks silent data corruption. - data_checksum_enabled: bool = true (hotswap); + // CRC32 checksum verification on pushdata/fetchdata data payloads. + // Safe upgrade path: leave false (the default) until every node in the cluster is running the new + // binary, then enable via hotswap. Enabling while any node still runs pre-checksum code causes the + // FetchData framing header emitted by upgraded senders to be misinterpreted as block data by old + // receivers, silently writing corrupted bytes to disk. + data_checksum_enabled: bool = false (hotswap); } table HomeStoreSettings { diff --git a/src/lib/replication/repl_dev/raft_repl_dev.cpp b/src/lib/replication/repl_dev/raft_repl_dev.cpp index 8202d278d..44509c5c7 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.cpp +++ b/src/lib/replication/repl_dev/raft_repl_dev.cpp @@ -1179,8 +1179,10 @@ void RaftReplDev::on_push_data_received(intrusive< sisl::GenericRpcData >& rpc_d auto const data_ptr = r_cast< const unsigned char* >(incoming_buf.cbytes() + fb_size); auto const computed = crc32_ieee(init_crc32, data_ptr, push_req->data_size()); if (computed != push_req->checksum()) { - RD_LOGE(NO_TRACE_ID, - "Data Channel: PushData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}, dropping", + COUNTER_INCREMENT(m_metrics, data_checksum_mismatch_cnt, 1); + RD_LOGE(rkey.traceID, + "Data Channel: PushData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}, dropping " + "(follower will fetch from remote on next Raft retry)", push_req->dsn(), push_req->checksum(), computed); rpc_data->send_response(); return; @@ -1547,9 +1549,16 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ RD_LOGT(NO_TRACE_ID, "Data Channel: FetchData received: fetch_req.size={}", fetch_req->request()->entries()->size()); + struct FetchEntryMeta { + int64_t lsn; + uint64_t dsn; + uint64_t raft_term; + }; std::vector< sisl::sg_list > sgs_vec; + std::vector< FetchEntryMeta > entry_metas; std::vector< folly::Future< std::error_code > > futs; sgs_vec.reserve(fetch_req->request()->entries()->size()); + entry_metas.reserve(fetch_req->request()->entries()->size()); futs.reserve(fetch_req->request()->entries()->size()); for (auto const& req : *(fetch_req->request()->entries())) { @@ -1565,8 +1574,9 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ sgs.iovs.emplace_back( iovec{.iov_base = iomanager.iobuf_alloc(get_blk_size(), total_size), .iov_len = total_size}); - // accumulate the sgs for later use (send back to the requester)); + // accumulate the sgs and per-entry metadata for later use (send back to the requester); sgs_vec.push_back(sgs); + entry_metas.push_back({req->lsn(), req->dsn(), req->raft_term()}); if (originator != server_id()) { RD_LOGD(NO_TRACE_ID, "non-originator FetchData received: dsn={} lsn={} originator={}, my_server_id={}", @@ -1582,7 +1592,8 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ } folly::collectAllUnsafe(futs).thenValue( - [this, rpc_data = std::move(rpc_data), sgs_vec = std::move(sgs_vec)](auto&& vf) { + [this, rpc_data = std::move(rpc_data), sgs_vec = std::move(sgs_vec), + entry_metas = std::move(entry_metas)](auto&& vf) { for (auto const& err_c : vf) { const auto& err = err_c.value(); if (err) { @@ -1604,34 +1615,44 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ RD_LOGT(NO_TRACE_ID, "Data Channel: FetchData data read completed for {} buffers", sgs_vec.size()); - // When checksums are enabled, prepend a size-prefixed FetchDataResponse FlatBuffer header that carries - // a per-entry CRC32. The header is omitted entirely when checksums are disabled so that nodes - // running old code (which have no notion of this header) can safely receive the response. - bool const compute_checksum = HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled); + // When checksums are enabled, prepend a self-describing framing header: + // [8-byte FETCH_DATA_RESPONSE_MAGIC][size-prefixed FetchDataResponse FlatBuffer] + // The magic makes the header detectable by the receiver without consulting any per-node + // config, so it is safe under hotswap config asymmetry. When disabled, no header is + // written and the response is raw block data, identical to pre-checksum wire format. uint8_t* hdr_buf = nullptr; uint32_t hdr_size = 0; - if (compute_checksum) { + if (HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled)) { flatbuffers::FlatBufferBuilder resp_builder; std::vector< flatbuffers::Offset< ResponseEntry > > resp_entries; - for (auto const& sgs : sgs_vec) { + for (size_t i = 0; i < sgs_vec.size(); ++i) { + auto const& sgs = sgs_vec[i]; uint32_t checksum = init_crc32; for (auto const& iov : sgs.iovs) { checksum = crc32_ieee(checksum, r_cast< const unsigned char* >(iov.iov_base), iov.iov_len); } + int64_t const lsn = (i < entry_metas.size()) ? entry_metas[i].lsn : 0; + uint64_t const dsn = (i < entry_metas.size()) ? entry_metas[i].dsn : 0; + uint64_t const raft_term = (i < entry_metas.size()) ? entry_metas[i].raft_term : 0; resp_entries.push_back( - CreateResponseEntry(resp_builder, 0, 0, 0, static_cast< uint32_t >(sgs.size), checksum)); + CreateResponseEntry(resp_builder, lsn, dsn, raft_term, + static_cast< uint32_t >(sgs.size), checksum)); } resp_builder.FinishSizePrefixed( CreateFetchDataResponse(resp_builder, server_id(), resp_builder.CreateVector(resp_entries))); - hdr_size = resp_builder.GetSize(); - hdr_buf = iomanager.iobuf_alloc(512, hdr_size); - std::memcpy(hdr_buf, resp_builder.GetBufferPointer(), hdr_size); + + // Allocate: [magic (8 bytes)] + [size-prefixed FetchDataResponse FlatBuffer] + auto const fb_size = resp_builder.GetSize(); + hdr_size = static_cast< uint32_t >(sizeof(FETCH_DATA_RESPONSE_MAGIC)) + fb_size; + hdr_buf = new uint8_t[hdr_size]; + std::memcpy(hdr_buf, &FETCH_DATA_RESPONSE_MAGIC, sizeof(FETCH_DATA_RESPONSE_MAGIC)); + std::memcpy(hdr_buf + sizeof(FETCH_DATA_RESPONSE_MAGIC), resp_builder.GetBufferPointer(), fb_size); } // now prepare the io_blob_list to response back to requester; nuraft_mesg::io_blob_list_t pkts = sisl::io_blob_list_t{}; - if (hdr_buf) { pkts.emplace_back(sisl::io_blob{hdr_buf, hdr_size, true}); } + if (hdr_buf) { pkts.emplace_back(sisl::io_blob{hdr_buf, hdr_size, false}); } for (auto const& sgs : sgs_vec) { auto const ret = sisl::io_blob::sg_list_to_ioblob_list(sgs); pkts.insert(pkts.end(), ret.begin(), ret.end()); @@ -1639,7 +1660,7 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ rpc_data->set_comp_cb( [sgs_vec = std::move(sgs_vec), hdr_buf](boost::intrusive_ptr< sisl::GenericRpcData >&) { - if (hdr_buf) { iomanager.iobuf_free(hdr_buf); } + delete[] hdr_buf; // header is heap-allocated (not iobuf); delete[] nullptr is a no-op for (auto const& sgs : sgs_vec) { for (auto const& iov : sgs.iovs) { iomanager.iobuf_free(reinterpret_cast< uint8_t* >(iov.iov_base)); @@ -1662,30 +1683,44 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons return; } - COUNTER_INCREMENT(m_metrics, fetch_total_blk_size, total_size); - - // When checksums are enabled, the sender prepends a size-prefixed FetchDataResponse FlatBuffer header. - // Gate all header parsing on the same config flag so nodes with checksums disabled (including old nodes - // that predate this feature) are fully compatible — they neither send nor expect the header. - bool const verify_checksum = HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled); + // Detect whether the sender included a checksum framing header by checking for FETCH_DATA_RESPONSE_MAGIC. + // This is self-describing: receivers check for the magic regardless of their own data_checksum_enabled + // setting, making the check safe under hotswap config asymmetry between nodes. const flatbuffers::Vector< flatbuffers::Offset< ResponseEntry > >* resp_entries = nullptr; - if (verify_checksum) { + if (total_size >= sizeof(FETCH_DATA_RESPONSE_MAGIC) && + std::memcmp(raw_data, &FETCH_DATA_RESPONSE_MAGIC, sizeof(FETCH_DATA_RESPONSE_MAGIC)) == 0) { + + raw_data += sizeof(FETCH_DATA_RESPONSE_MAGIC); + total_size -= sizeof(FETCH_DATA_RESPONSE_MAGIC); + + if (total_size < sizeof(flatbuffers::uoffset_t)) { + RD_LOGE(NO_TRACE_ID, + "Data Channel: FetchData response framing header too short ({} bytes), ignoring response", + total_size); + return; + } auto const fb_hdr_size = flatbuffers::ReadScalar< flatbuffers::uoffset_t >(raw_data) + sizeof(flatbuffers::uoffset_t); if (fb_hdr_size > total_size) { RD_LOGE(NO_TRACE_ID, - "Data Channel: FetchData response header size {} exceeds blob size {}, ignoring response", + "Data Channel: FetchData response FlatBuffer size {} exceeds remaining blob size {}, " + "ignoring response", fb_hdr_size, total_size); return; } + + flatbuffers::Verifier verifier{raw_data, fb_hdr_size}; + if (!verifier.VerifySizePrefixedBuffer< FetchDataResponse >(nullptr)) { + RD_LOGE(NO_TRACE_ID, + "Data Channel: FetchData response FlatBuffer failed verification, ignoring response"); + return; + } auto const fetch_resp = flatbuffers::GetSizePrefixedRoot< FetchDataResponse >(raw_data); raw_data += fb_hdr_size; total_size -= fb_hdr_size; - if (!fetch_resp || !fetch_resp->entries()) { - RD_LOGW(NO_TRACE_ID, - "Data Channel: FetchData response header is malformed, skipping checksum verification"); - } else { + + if (fetch_resp->entries()) { resp_entries = fetch_resp->entries(); if (resp_entries->size() != rreqs.size()) { RD_LOGW(NO_TRACE_ID, @@ -1696,6 +1731,9 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons } } + // Count only actual block data bytes (framing header excluded). + COUNTER_INCREMENT(m_metrics, fetch_total_blk_size, total_size); + RD_LOGD(NO_TRACE_ID, "Data Channel: FetchData completed for {} requests", rreqs.size()); for (size_t i = 0; i < rreqs.size(); ++i) { @@ -1705,9 +1743,10 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons if (resp_entries && i < resp_entries->size() && (*resp_entries)[i]->checksum() != 0) { auto const computed = crc32_ieee(init_crc32, r_cast< const unsigned char* >(raw_data), data_size); if (computed != (*resp_entries)[i]->checksum()) { + COUNTER_INCREMENT(m_metrics, data_checksum_mismatch_cnt, 1); RD_LOGE(rreq->traceID(), - "Data Channel: FetchData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}, " - "skipping write to avoid corrupting storage", + "Data Channel: FetchData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}; " + "skipping write. Raft will retry after data_receive_timeout_ms.", rreq->dsn(), (*resp_entries)[i]->checksum(), computed); raw_data += data_size; total_size -= data_size; diff --git a/src/lib/replication/repl_dev/raft_repl_dev.h b/src/lib/replication/repl_dev/raft_repl_dev.h index c27155025..17dc12594 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.h +++ b/src/lib/replication/repl_dev/raft_repl_dev.h @@ -18,6 +18,12 @@ namespace homestore { static constexpr uint64_t max_replace_member_task_id_len = 64; +// Magic prefix for the FetchData response framing header (first 8 bytes of +// echo 'homestore_fetch_response' | md5sum). Receivers detect the magic +// independently of their own data_checksum_enabled setting, making framing +// self-describing and safe under hotswap config asymmetry between nodes. +static constexpr uint64_t FETCH_DATA_RESPONSE_MAGIC = 0x9E3A7F2C4B8D1065ULL; + struct replace_member_task_superblk { char task_id[max_replace_member_task_id_len]; replica_id_t replica_out; @@ -71,6 +77,8 @@ class RaftReplDevMetrics : public sisl::MetricsGroup { REGISTER_COUNTER(read_err_cnt, "total read error count", "read_err_cnt", {"op", "read"}); REGISTER_COUNTER(write_err_cnt, "total write error count", "write_err_cnt", {"op", "write"}); REGISTER_COUNTER(fetch_err_cnt, "total fetch data error count", "fetch_err_cnt", {"op", "fetch"}); + REGISTER_COUNTER(data_checksum_mismatch_cnt, "CRC32 mismatches on push/fetch data channels", + "data_checksum_mismatch_cnt", {"op", "checksum"}); REGISTER_COUNTER(fetch_rreq_cnt, "total fetch data count", "fetch_data_req_cnt", {"op", "fetch"}); REGISTER_COUNTER(fetch_total_blk_size, "total fetch data blocks size", "fetch_total_blk_size", {"op", "fetch"}); diff --git a/src/tests/test_raft_repl_dev.cpp b/src/tests/test_raft_repl_dev.cpp index eafad14a8..c4c497e2c 100644 --- a/src/tests/test_raft_repl_dev.cpp +++ b/src/tests/test_raft_repl_dev.cpp @@ -111,6 +111,57 @@ TEST_F(RaftReplDevTest, Follower_Fetch_OnActive_ReplicaGroup) { if (g_helper->replica_num() != 0) { g_helper->remove_flip("drop_push_data_request"); } } +// Verifies the happy path with checksums explicitly enabled: writes should commit correctly +// and all replicas should hold the same data. +TEST_F(RaftReplDevTest, Checksum_Enabled_PushData_Path) { + LOGINFO("Homestore replica={} setup completed", g_helper->replica_num()); + g_helper->sync_for_test_start(); + + LOGINFO("Enabling data_checksum_enabled"); + HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.consensus.data_checksum_enabled = true; }); + HS_SETTINGS_FACTORY().save(); + + this->write_on_leader(20, true /* wait_for_commit */); + + g_helper->sync_for_verify_start(); + LOGINFO("Validate all data written so far by reading them"); + this->validate_data(); + + HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.consensus.data_checksum_enabled = false; }); + HS_SETTINGS_FACTORY().save(); + g_helper->sync_for_cleanup_start(); +} + +#ifdef _PRERELEASE +// Verifies that the fetch path works correctly with checksums enabled. +// Drops all push-data on non-leader replicas so they are forced to fetch, then checks that +// the framing header is correctly parsed and data arrives intact. +TEST_F(RaftReplDevTest, Checksum_Enabled_FetchData_Path) { + LOGINFO("Homestore replica={} setup completed", g_helper->replica_num()); + g_helper->sync_for_test_start(); + + LOGINFO("Enabling data_checksum_enabled"); + HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.consensus.data_checksum_enabled = true; }); + HS_SETTINGS_FACTORY().save(); + + if (g_helper->replica_num() != 0) { + LOGINFO("Drop all push-data so follower {} must fetch with checksum header", g_helper->replica_num()); + g_helper->set_basic_flip("drop_push_data_request"); + } + + this->write_on_leader(20, true /* wait_for_commit */); + + g_helper->sync_for_verify_start(); + LOGINFO("Validate all data written so far by reading them"); + this->validate_data(); + + HS_SETTINGS_FACTORY().modifiable_settings([](auto& s) { s.consensus.data_checksum_enabled = false; }); + HS_SETTINGS_FACTORY().save(); + g_helper->sync_for_cleanup_start(); + if (g_helper->replica_num() != 0) { g_helper->remove_flip("drop_push_data_request"); } +} +#endif + TEST_F(RaftReplDevTest, Write_With_Diabled_Leader_Push_Data) { g_helper->set_basic_flip("disable_leader_push_data", std::numeric_limits< int >::max(), 100); LOGINFO("Homestore replica={} setup completed, all the push_data from leader are disabled", From 9a2ec70b3fe6edf0bf103a86392851c2c045bbad Mon Sep 17 00:00:00 2001 From: Om Doshi Date: Wed, 27 May 2026 13:26:20 +0100 Subject: [PATCH 3/5] Fix three correctness issues in checksum verification F1: On FetchData checksum mismatch, immediately re-fetch via check_and_fetch_remote_data() instead of waiting for the full data_receive_timeout_ms (10s) stall. Mismatched rreqs are collected in the response loop and re-queued after processing the valid entries. F2: Guard against total_size unsigned underflow before each total_size -= data_size subtraction in handle_fetch_data_response. A truncated or malformed response could wrap total_size to ~4GB and advance raw_data past the blob boundary in release builds. F3: Add flatbuffers::Verifier call in on_push_data_received before any PushDataRequest field is accessed. Without this, a corrupted or crafted push RPC could cause out-of-bounds reads via the unverified FlatBuffer accessor, matching the verification already present on the FetchData response path. --- .../replication/repl_dev/raft_repl_dev.cpp | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/lib/replication/repl_dev/raft_repl_dev.cpp b/src/lib/replication/repl_dev/raft_repl_dev.cpp index 44509c5c7..50e295b8f 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.cpp +++ b/src/lib/replication/repl_dev/raft_repl_dev.cpp @@ -1158,6 +1158,12 @@ void RaftReplDev::on_push_data_received(intrusive< sisl::GenericRpcData >& rpc_d auto const fb_size = flatbuffers::ReadScalar< flatbuffers::uoffset_t >(incoming_buf.cbytes()) + sizeof(flatbuffers::uoffset_t); auto push_req = GetSizePrefixedPushDataRequest(incoming_buf.cbytes()); + flatbuffers::Verifier push_verifier{incoming_buf.cbytes(), fb_size}; + if (!push_verifier.VerifySizePrefixedBuffer< PushDataRequest >(nullptr)) { + RD_LOGW(NO_TRACE_ID, "Data Channel: PushData FlatBuffer verification failed, ignoring"); + rpc_data->send_response(); + return; + } if (fb_size + push_req->data_size() != incoming_buf.size()) { RD_LOGW(NO_TRACE_ID, "Data Channel: PushData received with size mismatch, header size {}, data size {}, received size {}", @@ -1736,20 +1742,30 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons RD_LOGD(NO_TRACE_ID, "Data Channel: FetchData completed for {} requests", rreqs.size()); + std::vector< repl_req_ptr_t > checksum_mismatch_rreqs; for (size_t i = 0; i < rreqs.size(); ++i) { auto const& rreq = rreqs[i]; auto const data_size = rreq->remote_blkid().blkid.blk_count() * get_blk_size(); + if (data_size > total_size) { + RD_LOGE(NO_TRACE_ID, + "Data Channel: FetchData response truncated: need {} bytes for dsn={} but only {} bytes remain, " + "aborting response processing", + data_size, rreq->dsn(), total_size); + return; + } + if (resp_entries && i < resp_entries->size() && (*resp_entries)[i]->checksum() != 0) { auto const computed = crc32_ieee(init_crc32, r_cast< const unsigned char* >(raw_data), data_size); if (computed != (*resp_entries)[i]->checksum()) { COUNTER_INCREMENT(m_metrics, data_checksum_mismatch_cnt, 1); RD_LOGE(rreq->traceID(), "Data Channel: FetchData checksum mismatch dsn={}, expected={:#010x}, computed={:#010x}; " - "skipping write. Raft will retry after data_receive_timeout_ms.", + "re-fetching immediately.", rreq->dsn(), (*resp_entries)[i]->checksum(), computed); raw_data += data_size; total_size -= data_size; + checksum_mismatch_rreqs.emplace_back(rreq); continue; } } @@ -1801,6 +1817,12 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons total_size -= data_size; } + if (!checksum_mismatch_rreqs.empty()) { + RD_LOGD(NO_TRACE_ID, "Data Channel: Re-fetching {} rreqs that had checksum mismatches", + checksum_mismatch_rreqs.size()); + check_and_fetch_remote_data(std::move(checksum_mismatch_rreqs)); + } + RD_DBG_ASSERT_EQ(total_size, 0, "Total size mismatch, some data is not consumed"); } From 147fcc8a1969c57a171af607889b7eb13e365d46 Mon Sep 17 00:00:00 2001 From: Om Doshi Date: Tue, 2 Jun 2026 22:33:15 +0100 Subject: [PATCH 4/5] Refactor FetchData wire format: replace magic header with try-and-fallback Replace FETCH_DATA_RESPONSE_MAGIC framing with a size-prefix try-and-fallback to detect new vs old FetchDataResponse format on the receiver side. Sender (on_fetch_data_received): emit a size-prefixed FetchDataResponse FlatBuffer only when data_checksum_enabled=true. When disabled the old raw wire format is preserved unchanged, so old receivers are not affected during rolling upgrades. Receiver (handle_fetch_data_response): attempt to parse a FlatBuffer by reading the 4-byte uoffset_t prefix, checking fb_hdr_size <= total_size, and running flatbuffers::Verifier. Falls back to raw format on any failure. Raw block data from HomeObject begins with DataHeader magic (0x21fdffdba8d68fc6) whose first 4 bytes as a uoffset_t read as ~2.8 GB, reliably failing the size check. Additional fixes: - Move GetSizePrefixedPushDataRequest to after the flatbuffers::Verifier guard in on_push_data_received so no field access precedes buffer verification. - data_checksum_enabled now gates send-side only; receivers skip CRC when checksum field is zero (0 = not computed sentinel). - Widen uoffset_t arithmetic to uint64_t in on_push_data_received and handle_fetch_data_response to prevent integer wrap when ReadScalar returns a value near 0xFFFFFFFF. - Update homestore_config.fbs comment to document send-side-only semantics. --- src/lib/common/homestore_config.fbs | 12 ++- .../replication/repl_dev/raft_repl_dev.cpp | 101 ++++++++---------- src/lib/replication/repl_dev/raft_repl_dev.h | 6 -- 3 files changed, 50 insertions(+), 69 deletions(-) diff --git a/src/lib/common/homestore_config.fbs b/src/lib/common/homestore_config.fbs index c537ddcad..77a28bffd 100644 --- a/src/lib/common/homestore_config.fbs +++ b/src/lib/common/homestore_config.fbs @@ -325,11 +325,13 @@ table Consensus { // 60 * repl_dev_cleanup_interval_sec (60s) means every 1 hour we will log the gc repl reqs info. gc_repl_reqs_log_frequency: uint32 = 60 (hotswap); - // CRC32 checksum verification on pushdata/fetchdata data payloads. - // Safe upgrade path: leave false (the default) until every node in the cluster is running the new - // binary, then enable via hotswap. Enabling while any node still runs pre-checksum code causes the - // FetchData framing header emitted by upgraded senders to be misinterpreted as block data by old - // receivers, silently writing corrupted bytes to disk. + // Controls whether senders compute and attach a CRC32 checksum to pushdata/fetchdata payloads. + // The receiver always skips CRC verification when the checksum field is zero, so this flag only + // affects the send side. Safe to enable via hotswap once all nodes in the cluster are running a + // version that emits the FetchDataResponse FlatBuffer header; old nodes that still send raw block + // data will simply have checksum=0 and receivers fall back gracefully. + // Not enabled by default: TCP already detects partial-transfer corruption; CRC is opt-in for + // environments requiring additional durability guarantees. data_checksum_enabled: bool = false (hotswap); } diff --git a/src/lib/replication/repl_dev/raft_repl_dev.cpp b/src/lib/replication/repl_dev/raft_repl_dev.cpp index 50e295b8f..6b7fbfe40 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.cpp +++ b/src/lib/replication/repl_dev/raft_repl_dev.cpp @@ -1098,6 +1098,9 @@ void RaftReplDev::push_data_to_all_followers(repl_req_ptr_t rreq, sisl::sg_list auto& builder = rreq->create_fb_builder(); // Compute CRC32 over the data payload before serialising so the follower can detect corruption in transit. + // checksum=0 is the sentinel for "not computed"; the receiver skips verification when it sees 0. + // A legitimately computed CRC of 0 (~1 in 4B) is indistinguishable from the sentinel and will + // also skip verification for that packet — an accepted limitation of this design. uint32_t checksum = 0; if (HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled)) { checksum = init_crc32; @@ -1155,15 +1158,16 @@ void RaftReplDev::on_push_data_received(intrusive< sisl::GenericRpcData >& rpc_d return; } - auto const fb_size = - flatbuffers::ReadScalar< flatbuffers::uoffset_t >(incoming_buf.cbytes()) + sizeof(flatbuffers::uoffset_t); - auto push_req = GetSizePrefixedPushDataRequest(incoming_buf.cbytes()); - flatbuffers::Verifier push_verifier{incoming_buf.cbytes(), fb_size}; + auto const fb_size = static_cast< uint64_t >( + flatbuffers::ReadScalar< flatbuffers::uoffset_t >(incoming_buf.cbytes())) + + sizeof(flatbuffers::uoffset_t); + flatbuffers::Verifier push_verifier{incoming_buf.cbytes(), static_cast< size_t >(fb_size)}; if (!push_verifier.VerifySizePrefixedBuffer< PushDataRequest >(nullptr)) { RD_LOGW(NO_TRACE_ID, "Data Channel: PushData FlatBuffer verification failed, ignoring"); rpc_data->send_response(); return; } + auto push_req = GetSizePrefixedPushDataRequest(incoming_buf.cbytes()); if (fb_size + push_req->data_size() != incoming_buf.size()) { RD_LOGW(NO_TRACE_ID, "Data Channel: PushData received with size mismatch, header size {}, data size {}, received size {}", @@ -1181,7 +1185,7 @@ void RaftReplDev::on_push_data_received(intrusive< sisl::GenericRpcData >& rpc_d RD_LOGD(rkey.traceID, "Data Channel: PushData received: time diff={} ms.", get_elapsed_time_ms(req_orig_time_ms)); - if (HS_DYNAMIC_CONFIG(consensus.data_checksum_enabled) && push_req->checksum() != 0) { + if (push_req->checksum() != 0) { auto const data_ptr = r_cast< const unsigned char* >(incoming_buf.cbytes() + fb_size); auto const computed = crc32_ieee(init_crc32, data_ptr, push_req->data_size()); if (computed != push_req->checksum()) { @@ -1621,11 +1625,10 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ RD_LOGT(NO_TRACE_ID, "Data Channel: FetchData data read completed for {} buffers", sgs_vec.size()); - // When checksums are enabled, prepend a self-describing framing header: - // [8-byte FETCH_DATA_RESPONSE_MAGIC][size-prefixed FetchDataResponse FlatBuffer] - // The magic makes the header detectable by the receiver without consulting any per-node - // config, so it is safe under hotswap config asymmetry. When disabled, no header is - // written and the response is raw block data, identical to pre-checksum wire format. + // Emit a size-prefixed FetchDataResponse FlatBuffer before the block data only when + // data_checksum_enabled is true. When disabled, send raw block data (pre-checksum wire + // format) so old receivers are never surprised by an unexpected header during rolling + // upgrades. Receivers use try-and-fallback to detect the FlatBuffer transparently. uint8_t* hdr_buf = nullptr; uint32_t hdr_size = 0; @@ -1648,12 +1651,10 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ resp_builder.FinishSizePrefixed( CreateFetchDataResponse(resp_builder, server_id(), resp_builder.CreateVector(resp_entries))); - // Allocate: [magic (8 bytes)] + [size-prefixed FetchDataResponse FlatBuffer] - auto const fb_size = resp_builder.GetSize(); - hdr_size = static_cast< uint32_t >(sizeof(FETCH_DATA_RESPONSE_MAGIC)) + fb_size; + // Heap-copy the FlatBuffer so it outlives resp_builder until the send completion callback. + hdr_size = resp_builder.GetSize(); hdr_buf = new uint8_t[hdr_size]; - std::memcpy(hdr_buf, &FETCH_DATA_RESPONSE_MAGIC, sizeof(FETCH_DATA_RESPONSE_MAGIC)); - std::memcpy(hdr_buf + sizeof(FETCH_DATA_RESPONSE_MAGIC), resp_builder.GetBufferPointer(), fb_size); + std::memcpy(hdr_buf, resp_builder.GetBufferPointer(), hdr_size); } // now prepare the io_blob_list to response back to requester; @@ -1666,7 +1667,7 @@ void RaftReplDev::on_fetch_data_received(intrusive< sisl::GenericRpcData >& rpc_ rpc_data->set_comp_cb( [sgs_vec = std::move(sgs_vec), hdr_buf](boost::intrusive_ptr< sisl::GenericRpcData >&) { - delete[] hdr_buf; // header is heap-allocated (not iobuf); delete[] nullptr is a no-op + delete[] hdr_buf; // delete[] nullptr is a no-op when checksums are disabled for (auto const& sgs : sgs_vec) { for (auto const& iov : sgs.iovs) { iomanager.iobuf_free(reinterpret_cast< uint8_t* >(iov.iov_base)); @@ -1689,50 +1690,34 @@ void RaftReplDev::handle_fetch_data_response(sisl::GenericClientResponse respons return; } - // Detect whether the sender included a checksum framing header by checking for FETCH_DATA_RESPONSE_MAGIC. - // This is self-describing: receivers check for the magic regardless of their own data_checksum_enabled - // setting, making the check safe under hotswap config asymmetry between nodes. + // Try-and-fallback: attempt to parse a size-prefixed FetchDataResponse FlatBuffer at the start + // of the blob. New senders (data_checksum_enabled=true) prepend this header; old senders emit + // raw block data with no prefix. In HomeObject (the layer above HomeStore), raw block data + // begins with DataHeader magic (0x21fdffdba8d68fc6), whose first 4 bytes as a little-endian + // uoffset_t read as ~2.8 GB — far larger than any real response — reliably failing the + // fb_hdr_size <= total_size check below. flatbuffers::Verifier provides a further structural + // guard for any other raw data whose size prefix happens to look plausible. const flatbuffers::Vector< flatbuffers::Offset< ResponseEntry > >* resp_entries = nullptr; - if (total_size >= sizeof(FETCH_DATA_RESPONSE_MAGIC) && - std::memcmp(raw_data, &FETCH_DATA_RESPONSE_MAGIC, sizeof(FETCH_DATA_RESPONSE_MAGIC)) == 0) { - - raw_data += sizeof(FETCH_DATA_RESPONSE_MAGIC); - total_size -= sizeof(FETCH_DATA_RESPONSE_MAGIC); - - if (total_size < sizeof(flatbuffers::uoffset_t)) { - RD_LOGE(NO_TRACE_ID, - "Data Channel: FetchData response framing header too short ({} bytes), ignoring response", - total_size); - return; - } - auto const fb_hdr_size = - flatbuffers::ReadScalar< flatbuffers::uoffset_t >(raw_data) + sizeof(flatbuffers::uoffset_t); - if (fb_hdr_size > total_size) { - RD_LOGE(NO_TRACE_ID, - "Data Channel: FetchData response FlatBuffer size {} exceeds remaining blob size {}, " - "ignoring response", - fb_hdr_size, total_size); - return; - } - - flatbuffers::Verifier verifier{raw_data, fb_hdr_size}; - if (!verifier.VerifySizePrefixedBuffer< FetchDataResponse >(nullptr)) { - RD_LOGE(NO_TRACE_ID, - "Data Channel: FetchData response FlatBuffer failed verification, ignoring response"); - return; - } - auto const fetch_resp = flatbuffers::GetSizePrefixedRoot< FetchDataResponse >(raw_data); - raw_data += fb_hdr_size; - total_size -= fb_hdr_size; - - if (fetch_resp->entries()) { - resp_entries = fetch_resp->entries(); - if (resp_entries->size() != rreqs.size()) { - RD_LOGW(NO_TRACE_ID, - "Data Channel: FetchData response entry count {} != request count {}, " - "some entries will not be checksum-verified", - resp_entries->size(), rreqs.size()); + if (total_size >= sizeof(flatbuffers::uoffset_t)) { + auto const fb_hdr_size = static_cast< uint64_t >( + flatbuffers::ReadScalar< flatbuffers::uoffset_t >(raw_data)) + + sizeof(flatbuffers::uoffset_t); + if (fb_hdr_size <= static_cast< uint64_t >(total_size)) { + flatbuffers::Verifier verifier{raw_data, fb_hdr_size}; + if (verifier.VerifySizePrefixedBuffer< FetchDataResponse >(nullptr)) { + auto const fetch_resp = flatbuffers::GetSizePrefixedRoot< FetchDataResponse >(raw_data); + raw_data += fb_hdr_size; + total_size -= fb_hdr_size; + if (fetch_resp->entries()) { + resp_entries = fetch_resp->entries(); + if (resp_entries->size() != rreqs.size()) { + RD_LOGW(NO_TRACE_ID, + "Data Channel: FetchData response entry count {} != request count {}, " + "some entries will not be checksum-verified", + resp_entries->size(), rreqs.size()); + } + } } } } diff --git a/src/lib/replication/repl_dev/raft_repl_dev.h b/src/lib/replication/repl_dev/raft_repl_dev.h index 17dc12594..da58334e6 100644 --- a/src/lib/replication/repl_dev/raft_repl_dev.h +++ b/src/lib/replication/repl_dev/raft_repl_dev.h @@ -18,12 +18,6 @@ namespace homestore { static constexpr uint64_t max_replace_member_task_id_len = 64; -// Magic prefix for the FetchData response framing header (first 8 bytes of -// echo 'homestore_fetch_response' | md5sum). Receivers detect the magic -// independently of their own data_checksum_enabled setting, making framing -// self-describing and safe under hotswap config asymmetry between nodes. -static constexpr uint64_t FETCH_DATA_RESPONSE_MAGIC = 0x9E3A7F2C4B8D1065ULL; - struct replace_member_task_superblk { char task_id[max_replace_member_task_id_len]; replica_id_t replica_out; From ef7a447a3ba5b80bd628fceecdc4ccc1f4e13d2e Mon Sep 17 00:00:00 2001 From: Om Doshi Date: Sat, 6 Jun 2026 18:51:56 +0100 Subject: [PATCH 5/5] Bump version to 7.5.10 --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 0d20a477d..b5db85ddd 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomestoreConan(ConanFile): name = "homestore" - version = "7.5.9" + version = "7.5.10" homepage = "https://github.com/eBay/Homestore" description = "HomeStore Storage Engine"