Merge branch 'master' into store-memory

proto/brpb.proto

@@ -85,6 +85,16 @@ message BackupMeta {
     repeated PitrDBMap db_maps = 22;
 
     BackupMode mode = 23;
+
+    // record the backup range and the correspond SST files when using file-copy backup.
+    repeated BackupRange ranges = 24;
+}
+
+message BackupRange {
+    bytes start_key = 1;
+    bytes end_key = 2;
+
+    repeated File files = 3;
 }
 
 message File {
@@ -118,6 +128,8 @@ message MetaFile {
     repeated Schema schemas = 3;
     // A set of files that contains RawRanges.
     repeated RawRange raw_ranges = 4;
+    // A set of files that contains BackupRanges.
+    repeated BackupRange backup_ranges = 6;
     // A set of files that contains DDLs.
     repeated bytes ddls = 5;
 }
@@ -335,6 +347,7 @@ message S3 {
     bool   object_lock_enabled = 14;
     string session_token = 15;
     string provider = 16;
+    string profile = 17;
 }
 
 // GCS storage backend saves files into google cloud storage.

proto/import_sstpb.proto

@@ -63,6 +63,27 @@ service ImportSST {
 
     // ClearFiles clear applied file after restore succeed.
     rpc ClearFiles(ClearRequest) returns (ClearResponse) {}
+
+    // Suspend ingest for data listeners don't support catching import data.
+    rpc SuspendImportRPC(SuspendImportRPCRequest) returns (SuspendImportRPCResponse) {}
+
+}
+
+message SuspendImportRPCRequest {
+    // whether to suspend new imports.
+    bool should_suspend_imports = 1;
+
+    // the duration of import service suspension
+    // when should_deny_imports is false,
+    // this won't take effect.
+    uint64 duration_in_secs = 2;
+    // The identifier for the caller.
+    string caller = 3;
+}
+
+message SuspendImportRPCResponse {
+    // The last state before this RPC.
+    bool already_suspended = 1;
 }
 
 enum SwitchMode {
@@ -154,6 +175,12 @@ message CompactResponse {
 }
 
 message DownloadRequest {
+    // Map represents the map of <name, SSTMeta>.
+    // We'll generate all SSTMeta into one SST File.
+    map<string, SSTMeta> ssts = 1;
+    // resolved_ts is used to merge related SST Files.
+    uint64 resolved_ts = 3;
+
     // The SST meta used to identify the downloaded file.
     // Must be the same among all nodes in the same Raft group.
     // Note: the "crc32" and "cf_name" fields are ignored in this request,
@@ -229,6 +256,10 @@ message DownloadResponse {
     uint32 crc32 = 4;
     // The actual length of the rewritten SST file.
     uint64 length = 5;
+
+    // This field only return when file-copy backup enabled.
+    // Because it will merge many SST files in a download request.
+    repeated SSTMeta ssts = 6;
 }
 
 message SetDownloadSpeedLimitRequest {

proto/raft_serverpb.proto

@@ -190,7 +190,10 @@ message MergeState {
 message MergedRecord {
     uint64 source_region_id = 1;
     metapb.RegionEpoch source_epoch = 2;
+    // Peers of source region when merge is committed.
     repeated metapb.Peer source_peers = 3;
+    // Removed peers (by confchange) of source region when merge is committed.
+    repeated metapb.Peer source_removed_records = 9;
     uint64 target_region_id = 4;
     metapb.RegionEpoch target_epoch = 5;
     repeated metapb.Peer target_peers = 6;
@@ -206,11 +209,13 @@ message RegionLocalState {
     MergeState merge_state = 3;
     // The apply index corresponding to the storage when it's initialized.
     uint64 tablet_index = 4;
-    // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds out its destiny
-    // by logs or broadcast; in v2, leader is responsible to ensure removed peers are destroyed.
+    // Raft doesn't guarantee peer will be removed in the end. In v1, peer finds
+    // out its destiny by logs or broadcast; in v2, leader is responsible to
+    // ensure removed peers are destroyed.
+    // Note: only peers who has been part of this region can be in this list.
     repeated metapb.Peer removed_records = 5;
-    // Merged peer can't be deleted like gc peers. Instead, leader needs to query target peer to
-    // decide whether source peer can be destroyed.
+    // Merged peer can't be deleted like gc peers. Instead, leader needs to
+    // query target peer to decide whether source peer can be destroyed.
     repeated MergedRecord merged_records = 6;
 }
 

proto/schedulingpb.proto

@@ -1,8 +1,10 @@
 syntax = "proto3";
 package schedulingpb;
 
+import "pdpb.proto";
 import "gogoproto/gogo.proto";
 import "rustproto.proto";
+import "metapb.proto";
 
 option (gogoproto.sizer_all) = true;
 option (gogoproto.marshaler_all) = true;
@@ -11,6 +13,39 @@ option (rustproto.lite_runtime_all) = true;
 
 option java_package = "org.tikv.kvproto";
 
+service Scheduling {
+    rpc StoreHeartbeat(StoreHeartbeatRequest) returns (StoreHeartbeatResponse) {}
+
+    rpc RegionHeartbeat(stream RegionHeartbeatRequest) returns (stream RegionHeartbeatResponse) {}
+}
+
+message RequestHeader {
+    // cluster_id is the ID of the cluster which be sent to.
+    uint64 cluster_id = 1;
+    // sender_id is the ID of the sender server.
+    uint64 sender_id = 2;
+}
+
+message ResponseHeader {
+    // cluster_id is the ID of the cluster which sent the response.
+    uint64 cluster_id = 1;
+    Error error = 2;
+}
+
+enum ErrorType {
+    OK = 0;
+    UNKNOWN = 1;
+    NOT_BOOTSTRAPPED = 2;
+    ALREADY_BOOTSTRAPPED = 3;
+    INVALID_VALUE = 4;
+    CLUSTER_MISMATCHED = 5;
+}
+
+message Error {
+    ErrorType type = 1;
+    string message = 2;
+}
+
 message Participant {
     // name is the unique name of the scheduling participant.
     string name = 1;
@@ -20,3 +55,82 @@ message Participant {
     // listen_urls[0] is primary service endpoint.
     repeated string listen_urls = 3;
 }
+
+message StoreHeartbeatRequest {
+    RequestHeader header = 1;
+    pdpb.StoreStats stats = 2;
+}
+
+message StoreHeartbeatResponse {
+    ResponseHeader header = 1;
+    string cluster_version = 2;
+}
+
+
+message RegionHeartbeatRequest {
+    RequestHeader header = 1;
+
+    metapb.Region region = 2;
+    // Leader Peer sending the heartbeat.
+    metapb.Peer leader = 3;
+    // Term is the term of raft group.
+    uint64 term = 4;
+    // Leader considers that these peers are down.
+    repeated pdpb.PeerStats down_peers = 5;
+    // Pending peers are the peers that the leader can't consider as
+    // working followers.
+    repeated metapb.Peer pending_peers = 6;
+    // Bytes read/written during this period.
+    uint64 bytes_written = 7;
+    uint64 bytes_read = 8;
+    // Keys read/written during this period.
+    uint64 keys_written = 9;
+    uint64 keys_read = 10;
+    // Approximate region size.
+    uint64 approximate_size = 11;
+    // Approximate number of keys.
+    uint64 approximate_keys = 12;
+    // QueryStats reported write query stats, and there are read query stats in store heartbeat
+    pdpb.QueryStats query_stats = 13;
+    // Actually reported time interval
+    pdpb.TimeInterval interval = 14;
+}
+
+
+message RegionHeartbeatResponse {
+    ResponseHeader header = 1;
+
+   // ID of the region
+    uint64 region_id = 2;
+    metapb.RegionEpoch region_epoch = 3;
+    // Leader of the region at the moment of the corresponding request was made.
+    metapb.Peer target_peer = 4;
+    // Notice, Pd only allows handling reported epoch >= current pd's.
+    // Leader peer reports region status with RegionHeartbeatRequest
+    // to pd regularly, pd will determine whether this region
+    // should do ChangePeer or not.
+    // E,g, max peer number is 3, region A, first only peer 1 in A.
+    // 1. Pd region state -> Peers (1), ConfVer (1).
+    // 2. Leader peer 1 reports region state to pd, pd finds the
+    // peer number is < 3, so first changes its current region
+    // state -> Peers (1, 2), ConfVer (1), and returns ChangePeer Adding 2.
+    // 3. Leader does ChangePeer, then reports Peers (1, 2), ConfVer (2),
+    // pd updates its state -> Peers (1, 2), ConfVer (2).
+    // 4. Leader may report old Peers (1), ConfVer (1) to pd before ConfChange
+    // finished, pd stills responses ChangePeer Adding 2, of course, we must
+    // guarantee the second ChangePeer can't be applied in TiKV.
+    pdpb.ChangePeer change_peer = 5;
+    // Pd can return transfer_leader to let TiKV does leader transfer itself.
+    pdpb.TransferLeader transfer_leader = 6;
+    pdpb.Merge merge = 7;
+    // PD sends split_region to let TiKV split a region into two regions.
+    pdpb.SplitRegion split_region = 8;
+    // Multiple change peer operations atomically.
+    // Note: PD can use both ChangePeer and ChangePeerV2 at the same time
+    //       (not in the same RegionHeartbeatResponse).
+    //       Now, PD use ChangePeerV2 in following scenarios:
+    //       1. replacing peers
+    //       2. demoting voter directly
+    pdpb.ChangePeerV2 change_peer_v2 = 9;
+    pdpb.BatchSwitchWitness switch_witnesses = 10;
+}