Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
1c251b2
fix(ha): align applied oplog metric with local status progress
Nov 13, 2026
30c4e32
fix(ha): report snapshot readiness from running services
Nov 13, 2026
b4f80a1
fix(ha): complete snapshot recovery diagnostics and reduce retry noise
Nov 13, 2026
797c68e
fix(ha): distinguish election readiness waits from failures
Nov 13, 2026
08ef246
fix(p2p): preserve HA startup failure diagnostics
Nov 13, 2026
b7ffd80
fix(ha): reduce shutdown write log noise
Nov 13, 2026
15338d9
fix(ha): include Redis commit failure reason in doctor
Nov 13, 2026
d9d1a5c
test(ha): verify Redis pipeline fault recovery boundaries
Nov 13, 2026
04d11c3
test(ha): cover promotion catch-up boundaries and diagnostics
Nov 13, 2026
b6545f9
test(ha): cover Redis election watch recovery in both modes
Nov 13, 2026
cd551a5
test(ha): stabilize latest stall checks and restore fault coverage
Nov 13, 2026
4ef32c2
test(ha): release blocked initialization after failed checks
Nov 13, 2026
2684bb7
test(ha): synchronize polling mock read counter
Nov 13, 2026
e4026f7
test(ha): cover stopping with a blocked latest reply
Nov 13, 2026
f086554
test(ha): cover stopping with an active oplog producer
Nov 13, 2026
06d4d3d
test(ha): prevent stopped writers retrying after Redis recovery
Nov 13, 2026
ddc6612
test(ha): cover SIGTERM during continuous client writes
Nov 13, 2026
8b03c35
test(ha): cover leadership loss during continuous writes
Nov 13, 2026
6fbd4f9
test(ha): restore sequence initialization assertions
Nov 13, 2026
d5cae04
test(ha): preserve readiness across repeated metadata restore
Nov 13, 2026
dbc05ec
test(ha): verify configured pending capacity diagnostics
Nov 13, 2026
5d74a75
test(ha): cover pipeline batch configuration through factory
Nov 13, 2026
aeac605
test(ha): cover pipeline metrics in HTTP export
Nov 13, 2026
73931db
test(ha): cover latest stall diagnostic cleanup
Nov 13, 2026
635785b
test(ha): preserve catchup stall timing across status updates
Nov 13, 2026
1f3ce14
test(ha): standardize OpLog spelling in test names
Nov 13, 2026
ab55346
test(ha): cover master view aliases in HTTP status
Nov 13, 2026
dbd44f5
test(ha): allow 360 seconds for the Redis E2E suite
Nov 13, 2026
7c5e18e
test(ha): keep Redis E2E listeners outside ephemeral ports
Nov 13, 2026
b8889ba
refactor(p2p): configure heartbeat timeout from client entrypoints
Nov 13, 2026
3e9cfb2
Merge P2P-Mooncake-Store into wl177541/p2p-ha-stepdown-fixes
Nov 16, 2026
a236d09
style(p2p): format config test member pointer with clang-format 20
Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions mooncake-integration/store/store_py.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1581,8 +1581,8 @@ void BindP2PMethods(StoreBinding& store) {
uint64_t metric_report_interval_seconds = 60,
uint16_t heartbeat_rpc_port = 0, bool start_local_only = false,
size_t te_async_poll_worker_num = 32,
size_t business_coro_worker_num = 32) {
auto real_client = self.init_real_client();
size_t business_coro_worker_num = 32,
int32_t heartbeat_rpc_timeout_ms = 1000) {
std::shared_ptr<mooncake::TransferEngine> transfer_engine =
nullptr;
if (!engine.is_none()) {
Expand All @@ -1608,13 +1608,18 @@ void BindP2PMethods(StoreBinding& store) {
enable_metric_collection, metric_report_interval_seconds,
DEFAULT_CLUSTER_ID, "", 0, 4, 1, "", heartbeat_rpc_port,
te_async_poll_worker_num, start_local_only,
business_coro_worker_num);
business_coro_worker_num, heartbeat_rpc_timeout_ms);

if (!config) {
LOG(ERROR) << "Store setup configuration failed: "
<< config.error();
return to_py_ret(config.error());
}
std::shared_ptr<RealClient> real_client;
{
py::gil_scoped_acquire acquire;
real_client = self.init_real_client();
}
auto result = real_client->setup(*config);
if (result != 0) {
LOG(ERROR) << "Store native setup failed: " << result;
Expand Down Expand Up @@ -1645,6 +1650,7 @@ void BindP2PMethods(StoreBinding& store) {
py::arg("start_local_only") = false,
py::arg("te_async_poll_worker_num") = 32,
py::arg("business_coro_worker_num") = 32,
py::arg("heartbeat_rpc_timeout_ms") = 1000,
"Setup the store in P2P architecture. start_local_only skips "
"Master "
"connection and heartbeat until an explicit POST /register.")
Expand Down Expand Up @@ -1730,7 +1736,9 @@ void BindP2PMethods(StoreBinding& store) {
" redis_heartbeat_interval_sec: Redis heartbeat interval "
"(default 1).\n"
" heartbeat_rpc_port: Dedicated heartbeat RPC port on the master "
"(0 uses the main RPC port, default 0).");
"(0 uses the main RPC port, default 0).\n"
" heartbeat_rpc_timeout_ms: Client heartbeat connection/request "
"timeout in milliseconds (positive, default 1000).");
}

#else
Expand Down
21 changes: 17 additions & 4 deletions mooncake-store/include/p2p/client/p2p_client_config_builder.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ struct P2PClientConfig : RealClientConfigBase {
// Dedicated master heartbeat port; must match --heartbeat_rpc_port.
// Zero sends heartbeats to the main RPC port.
uint16_t heartbeat_rpc_port = 0;
// Client connection/request timeout for heartbeat RPCs; must be positive.
int32_t heartbeat_rpc_timeout_ms = 1000;

// Port for HTTP server.
// Only used when enable_http_server is true.
Expand Down Expand Up @@ -149,7 +151,12 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
int redis_master_view_ttl_sec = 4, int redis_heartbeat_interval_sec = 1,
const std::string& redis_username = "", uint16_t heartbeat_rpc_port = 0,
size_t te_async_poll_worker_num = 32, bool start_local_only = false,
size_t business_coro_worker_num = 32) {
size_t business_coro_worker_num = 32,
int32_t heartbeat_rpc_timeout_ms = 1000) {
if (heartbeat_rpc_timeout_ms <= 0) {
LOG(ERROR) << "heartbeat_rpc_timeout_ms must be greater than 0";
return tl::unexpected(ErrorCode::INVALID_PARAMS);
}
P2PClientConfig config;
fill_real_client_config_base(
config, local_hostname, metadata_connstring, protocol, rdma_devices,
Expand Down Expand Up @@ -224,6 +231,7 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
config.transfer_direction_mode = *parsed_direction;

config.heartbeat_rpc_port = heartbeat_rpc_port;
config.heartbeat_rpc_timeout_ms = heartbeat_rpc_timeout_ms;
return config;
}

Expand Down Expand Up @@ -297,6 +305,8 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
auto heartbeat_rpc_port =
get_config_integer<uint16_t>(config, DictP2P::kHeartbeatRpcPort,
DictP2P::kDefaultHeartbeatRpcPort);
auto heartbeat_rpc_timeout_ms = get_config_integer<int32_t>(
config, DictP2P::kHeartbeatRpcTimeoutMs, 1000);
auto http_port =
get_config_integer<uint16_t>(config, "http_port", 9003);
bool enable_http_server =
Expand All @@ -314,8 +324,9 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
!te_async_poll_worker_num || !business_coro_worker_num ||
!async_sender_thread_count || !async_max_batch_size ||
!async_route_queue_size || !metric_report_interval_seconds ||
!heartbeat_rpc_port || !http_port || !key_lease_duration_ms ||
!key_lease_scan_interval_ms || !redis_config) {
!heartbeat_rpc_port || !heartbeat_rpc_timeout_ms || !http_port ||
!key_lease_duration_ms || !key_lease_scan_interval_ms ||
!redis_config) {
LOG(ERROR) << "Failed to parse P2P numeric configuration";
return tl::unexpected(ErrorCode::INVALID_PARAMS);
}
Expand All @@ -336,7 +347,7 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
redis_config->heartbeat_interval_sec, redis_config->username,
*heartbeat_rpc_port, *te_async_poll_worker_num,
get_config_bool(config, DictP2P::kStartLocalOnly, false),
*business_coro_worker_num);
*business_coro_worker_num, *heartbeat_rpc_timeout_ms);
if (!result) {
LOG(ERROR) << "P2P dict configuration is invalid: "
<< result.error();
Expand Down Expand Up @@ -408,6 +419,8 @@ class P2PClientConfigBuilder : private ClientConfigBuilder {
static constexpr const char* kEnableMetricCollection =
"enable_metric_collection";
static constexpr const char* kHeartbeatRpcPort = "heartbeat_rpc_port";
static constexpr const char* kHeartbeatRpcTimeoutMs =
"heartbeat_rpc_timeout_ms";
static constexpr uint64_t kDefaultMetricReportIntervalSeconds = 60;
static constexpr bool kDefaultEnableMetricCollection = true;
static constexpr uint16_t kDefaultHeartbeatRpcPort = 0;
Expand Down
22 changes: 13 additions & 9 deletions mooncake-store/include/p2p/ha/ha_status.h
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,13 @@ class HAStatusManager {
void ClearLastErrorIfPrefix(std::string prefix);
void RecordRegistryHeartbeat(ErrorCode result, std::string error);
// Event-driven flag set from the P2P election readiness wait loop and
// Redis election callback when the standby oplog lag exceeds the readiness
// gate, cleared once the standby catches up or stops trying to win
// election. The doctor report formats the message at query time from the
// stored decision values and the current snapshot.
void SetElectionReadinessBlocked(bool blocked, uint64_t lag_entries = 0,
uint64_t max_lag_entries = 0);
// Redis election callback when latest is unknown or lag exceeds the gate.
// Capture the same sample used for the decision, independently of status
// updates from background replication. Clear when ready or election ends.
void SetElectionReadinessBlocked(
bool blocked, uint64_t applied_sequence_id = 0,
std::optional<uint64_t> latest_sequence_id = std::nullopt,
uint64_t max_lag_entries = 0);

HAStatusSnapshot Snapshot() const;
std::string SerializeJson() const;
Expand Down Expand Up @@ -249,9 +250,12 @@ class HAStatusManager {
std::optional<std::chrono::steady_clock::time_point> last_success_at;
int64_t last_success_ms{0};
} oplog_poll_;
bool election_readiness_blocked_{false};
uint64_t election_readiness_blocked_lag_{0};
uint64_t election_readiness_blocked_max_{0};
struct ElectionReadiness {
bool blocked{false};
uint64_t applied_sequence_id{0};
std::optional<uint64_t> latest_sequence_id;
uint64_t max_lag_entries{0};
} election_readiness_;
};

} // namespace mooncake
17 changes: 17 additions & 0 deletions mooncake-store/include/p2p/ha/oplog/p2p_hot_standby_service.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ struct P2PStandbySyncStatus {
struct P2PSnapshotSourceSelection {
std::vector<std::string> endpoints;
bool used_primary_fallback{false};
size_t alive_count{0};
size_t qualified_standby_count{0};
size_t primary_fallback_count{0};
size_t excluded_count{0};
// Exclusion reasons are mutually exclusive.
size_t excluded_self_count{0};
size_t excluded_unavailable_count{0};
size_t excluded_stale_count{0};
size_t excluded_primary_not_used_count{0};
size_t excluded_role_count{0};
};

P2PSnapshotSourceSelection SelectP2PSnapshotSources(
Expand Down Expand Up @@ -122,7 +132,10 @@ class P2PHotStandbyService {
return metadata_store_.get();
}
const std::string& GetClusterId() const { return config_.cluster_id; }
// Metadata readiness is also used by in-process snapshot sources.
bool IsReadyForSnapshot() const;
// External advertisement additionally requires a started RPC server.
bool CanServeSnapshot() const;

private:
friend struct mooncake::test::P2PHotStandbyServiceTestAccess;
Expand Down Expand Up @@ -176,6 +189,10 @@ class P2PHotStandbyService {

std::unique_ptr<P2PStandbySnapshotService> snapshot_service_;
std::unique_ptr<coro_rpc::coro_rpc_server> snapshot_server_;
// StopSnapshotServer cannot hold mutex_ while joining snapshot handlers.
// Publish availability separately so heartbeat readers never race the
// pointer.
std::atomic<bool> snapshot_server_running_{false};
};

} // namespace mooncake
5 changes: 5 additions & 0 deletions mooncake-store/include/p2p/ha/oplog/redis_oplog_store.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#ifdef STORE_USE_REDIS

#include <atomic>
#include <chrono>
#include <condition_variable>
#include <deque>
#include <map>
Expand Down Expand Up @@ -155,6 +156,10 @@ class RedisOpLogStore : public OpLogStore {
std::set<uint64_t> dropped_sequences_ GUARDED_BY(async_mutex_);
uint64_t committed_sequence_id_ GUARDED_BY(async_mutex_) = 0;
size_t failed_write_count_ GUARDED_BY(async_mutex_) = 0;
// Per-writer sampling, serialized by the existing queue lock.
std::chrono::steady_clock::time_point next_queue_warning_at_
GUARDED_BY(async_mutex_){};
uint64_t queue_overflows_since_last_log_ GUARDED_BY(async_mutex_) = 0;
std::atomic<bool> async_running_{false};
std::atomic<int64_t> consecutive_write_failures_{0};
std::vector<std::thread> async_workers_;
Expand Down
4 changes: 2 additions & 2 deletions mooncake-store/include/p2p/master/p2p_client_manager.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

namespace mooncake {

class P2PRecordOplogTestPeer;
class P2PRecordOpLogTestPeer;

/**
* @brief Manages P2P clients' lifecycle and heartbeat state.
Expand Down Expand Up @@ -83,7 +83,7 @@ class P2PClientManager final {
void SetClientCrashedCallback(ClientCrashedCallback cb);

private:
friend class P2PRecordOplogTestPeer;
friend class P2PRecordOpLogTestPeer;

static constexpr uint64_t kClientMonitorSleepMs = 1000;

Expand Down
6 changes: 4 additions & 2 deletions mooncake-store/include/p2p/master/p2p_master.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <csignal>
#include <cstdint>
#include <functional>
#include <string>
#include <ylt/coro_rpc/coro_rpc_server.hpp>

#include "p2p/common/p2p_master_config.h"
Expand Down Expand Up @@ -32,10 +33,11 @@ class P2PMaster {
const char* reason = "stepdown");
int RunActiveRpcServers(coro_rpc::coro_rpc_server& server,
P2PMasterRpcService& service,
std::function<void()> before_start = {},
std::function<void()> after_start = {},
std::function<void()> after_stop = {},
std::function<bool()> stopping = {},
std::function<bool()> stepping_down = {});
std::function<bool()> stepping_down = {},
std::string* startup_error = nullptr);

P2PMasterConfig config_;
static std::atomic<bool> shutdown_requested_;
Expand Down
5 changes: 5 additions & 0 deletions mooncake-store/include/p2p/master/p2p_master_client.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ class P2PMasterClient final {

void SetHeartbeatRpcPort(uint16_t port) { heartbeat_rpc_port_ = port; }

// Configure before the first Connect; also applies when using the main
// port.
[[nodiscard]] ErrorCode SetHeartbeatRpcTimeoutMs(int32_t timeout_ms);

[[nodiscard]] tl::expected<bool, ErrorCode> ExistKey(
std::string_view object_key);

Expand Down Expand Up @@ -310,6 +314,7 @@ class P2PMasterClient final {
RpcClientAccessor client_accessor_;
RpcClientAccessor heartbeat_accessor_;
uint16_t heartbeat_rpc_port_ = 0;
std::chrono::milliseconds heartbeat_rpc_timeout_{1000};

const UUID client_id_;
P2PMasterClientMetric* metrics_;
Expand Down
7 changes: 7 additions & 0 deletions mooncake-store/src/p2p/client/p2p_client_service.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,13 @@ ErrorCode P2PClientService::Init(const P2PClientConfig& config) {
LOG(ERROR) << "Cannot initialize an active or stopped P2P service";
return ErrorCode::UNAVAILABLE_IN_CURRENT_STATUS;
}
const auto heartbeat_config_result =
master_client_.SetHeartbeatRpcTimeoutMs(
config.heartbeat_rpc_timeout_ms);
if (heartbeat_config_result != ErrorCode::OK) {
// SetHeartbeatRpcTimeoutMs already logs the configuration error.
return heartbeat_config_result;
}
transfer_direction_mode_ = config.transfer_direction_mode;
master_client_.SetHeartbeatRpcPort(config.heartbeat_rpc_port);
// Reuse this entry when a LOCAL_ONLY service rejoins the cluster.
Expand Down
13 changes: 12 additions & 1 deletion mooncake-store/src/p2p/client/p2p_real_client_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ DEFINE_uint32(
"heartbeats are sent to <master host>:heartbeat_rpc_port instead of "
"the main RPC port so heavy metadata RPCs cannot head-of-line-block "
"them. Must match the master's --heartbeat_rpc_port. 0 = disabled.");
DEFINE_int32(heartbeat_rpc_timeout_ms, 1000,
"Client heartbeat RPC connection/request timeout in milliseconds; "
"must be greater than 0.");
DEFINE_validator(heartbeat_rpc_timeout_ms, [](const char* name, int32_t value) {
if (value <= 0) {
LOG(ERROR) << name << " must be greater than 0";
return false;
}
return true;
});
DEFINE_string(local_buffer_size, "0", "Size of local buffer (e.g., 16MB, 1GB)");
DEFINE_int32(threads, 1, "Number of rpc threads for dummy client");
DEFINE_string(tiered_backend_config, "conf/tiered_backend.json",
Expand Down Expand Up @@ -133,7 +143,8 @@ int main(int argc, char* argv[]) {
FLAGS_redis_username, static_cast<uint16_t>(FLAGS_heartbeat_rpc_port),
static_cast<size_t>(FLAGS_te_async_poll_worker_num),
FLAGS_p2p_start_local_only,
static_cast<size_t>(FLAGS_business_coro_worker_num));
static_cast<size_t>(FLAGS_business_coro_worker_num),
FLAGS_heartbeat_rpc_timeout_ms);
if (!config) {
LOG(ERROR) << "Failed to build P2P client config: " << config.error();
return -1;
Expand Down
Loading
Loading