From 5ab0c65b629164ffdc19c96f06ab0cedb71ca6f9 Mon Sep 17 00:00:00 2001 From: zeminzhou Date: Thu, 15 May 2025 17:03:29 +0800 Subject: [PATCH 1/3] [release-8.5-keyspace] add option to skip rawkv region bound (#292) (#358) * [release 8.1] cp disable-raw-kv-region-split (#292) * Add option to skip split rawkv regions (#217) * add option to skip rawkv region bound Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> --------- Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> * make check Signed-off-by: zeminzhou --------- Signed-off-by: AmoebaProtozoa <8039876+AmoebaProtozoa@users.noreply.github.com> Signed-off-by: zeminzhou Co-authored-by: David <8039876+AmoebaProtozoa@users.noreply.github.com> --- pkg/keyspace/keyspace.go | 59 ++++++++++++++++--- pkg/keyspace/keyspace_test.go | 5 ++ pkg/keyspace/util.go | 18 ++++-- pkg/keyspace/util_test.go | 51 +++++++++++++++- .../scheduling/server/rule/watcher_test.go | 9 ++- server/config/config.go | 11 ++++ 6 files changed, 136 insertions(+), 17 deletions(-) diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index 153872d313c..5a68502cde2 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -38,7 +38,6 @@ import ( "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/keypath" - "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/versioninfo/kerneltype" ) @@ -74,6 +73,7 @@ const ( type Config interface { GetPreAlloc() []string ToWaitRegionSplit() bool + GetDisableRawKVRegionSplit() bool GetWaitRegionSplitTimeout() time.Duration GetCheckRegionSplitInterval() time.Duration } @@ -488,7 +488,8 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er }) start := time.Now() - keyspaceRule := MakeLabelRule(id) + skipRaw := manager.config.GetDisableRawKVRegionSplit() + keyspaceRule := makeLabelRule(id, skipRaw) cl, ok := manager.cluster.(interface{ GetRegionLabeler() *labeler.RegionLabeler }) if !ok { return errors.New("cluster does not support region label") @@ -509,7 +510,13 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er zap.Error(err), ) } + return } + log.Info("[keyspace] added region label for keyspace", + zap.Uint32("keyspace-id", id), + zap.Any("label-rule", keyspaceRule), + zap.Duration("takes", time.Since(start)), + ) }() if waitRegionSplit { @@ -523,12 +530,48 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er return err } - log.Info("[keyspace] added region label for keyspace", - zap.Uint32("keyspace-id", id), - logutil.ZapRedactString("label-rule", keyspaceRule.String()), - zap.Duration("takes", time.Since(start)), - ) - return + // Stat waiting for region split. + var splitKeys [][]byte + for _, regionRange := range keyspaceRule.Data.([]*labeler.KeyRangeRule) { + splitKeys = append(splitKeys, regionRange.StartKey, regionRange.EndKey) + } + tickerInterval := manager.config.GetCheckRegionSplitInterval() + ticker := time.NewTicker(tickerInterval) + timer := time.NewTimer(manager.config.GetWaitRegionSplitTimeout()) + defer func() { + ticker.Stop() + timer.Stop() + }() + for { + select { + case <-ticker.C: + // Reset ticker if it's interval has been changed. + newInterval := manager.config.GetCheckRegionSplitInterval() + if tickerInterval != newInterval { + tickerInterval = newInterval + ticker.Reset(newInterval) + } + regionsInfo := manager.cluster.GetBasicCluster().RegionsInfo + totalSplit := 0 // number of split keys that have been split. + for _, splitKey := range splitKeys { + region := regionsInfo.GetRegionByKey(splitKey) + if region != nil && bytes.Equal(region.GetStartKey(), splitKey) { + totalSplit++ + } + } + if totalSplit == len(splitKeys) { + log.Info("[keyspace] wait region split successfully", zap.Uint32("keyspace-id", id)) + return + } + case <-timer.C: + log.Warn("[keyspace] wait region split timeout", + zap.Uint32("keyspace-id", id), + zap.Error(err), + ) + err = ErrRegionSplitTimeout + return + } + } } func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error { diff --git a/pkg/keyspace/keyspace_test.go b/pkg/keyspace/keyspace_test.go index 60c782a8e72..374c2e60f15 100644 --- a/pkg/keyspace/keyspace_test.go +++ b/pkg/keyspace/keyspace_test.go @@ -65,6 +65,7 @@ func TestKeyspaceTestSuite(t *testing.T) { type mockConfig struct { PreAlloc []string WaitRegionSplit bool + DisableRawKVRegionSplit bool WaitRegionSplitTimeout typeutil.Duration CheckRegionSplitInterval typeutil.Duration } @@ -85,6 +86,10 @@ func (m *mockConfig) GetCheckRegionSplitInterval() time.Duration { return m.CheckRegionSplitInterval.Duration } +func (m *mockConfig) GetDisableRawKVRegionSplit() bool { + return m.DisableRawKVRegionSplit +} + func (suite *keyspaceTestSuite) SetupTest() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) diff --git a/pkg/keyspace/util.go b/pkg/keyspace/util.go index 82c27f1faa9..98ef8cbad5c 100644 --- a/pkg/keyspace/util.go +++ b/pkg/keyspace/util.go @@ -126,9 +126,17 @@ func MakeRegionBound(id uint32) *RegionBound { } } -// MakeKeyRanges encodes keyspace ID to correct LabelRule data. -func MakeKeyRanges(id uint32) []any { +// makeKeyRanges encodes keyspace ID to correct LabelRule data. +func makeKeyRanges(id uint32, skipRaw bool) any { regionBound := MakeRegionBound(id) + if skipRaw { + return []any{ + map[string]any{ + "start_key": hex.EncodeToString(regionBound.TxnLeftBound), + "end_key": hex.EncodeToString(regionBound.TxnRightBound), + }, + } + } return []any{ map[string]any{ "start_key": hex.EncodeToString(regionBound.RawLeftBound), @@ -146,8 +154,8 @@ func getRegionLabelID(id uint32) string { return regionLabelIDPrefix + strconv.FormatUint(uint64(id), endpoint.SpaceIDBase) } -// MakeLabelRule makes the label rule for the given keyspace id. -func MakeLabelRule(id uint32) *labeler.LabelRule { +// makeLabelRule makes the label rule for the given keyspace id. +func makeLabelRule(id uint32, skipRaw bool) *labeler.LabelRule { return &labeler.LabelRule{ ID: getRegionLabelID(id), Index: 0, @@ -158,7 +166,7 @@ func MakeLabelRule(id uint32) *labeler.LabelRule { }, }, RuleType: labeler.KeyRange, - Data: MakeKeyRanges(id), + Data: makeKeyRanges(id, skipRaw), } } diff --git a/pkg/keyspace/util_test.go b/pkg/keyspace/util_test.go index a4b44e2bdce..517e65796e6 100644 --- a/pkg/keyspace/util_test.go +++ b/pkg/keyspace/util_test.go @@ -144,10 +144,12 @@ func TestMakeLabelRule(t *testing.T) { re := require.New(t) testCases := []struct { id uint32 + skipRaw bool expectedLabelRule *labeler.LabelRule }{ { - id: 0, + id: 0, + skipRaw: false, expectedLabelRule: &labeler.LabelRule{ ID: getRegionLabelID(0), Index: 0, @@ -171,7 +173,8 @@ func TestMakeLabelRule(t *testing.T) { }, }, { - id: 4242, + id: 4242, + skipRaw: false, expectedLabelRule: &labeler.LabelRule{ ID: getRegionLabelID(4242), Index: 0, @@ -194,9 +197,51 @@ func TestMakeLabelRule(t *testing.T) { }, }, }, + { + id: 0, + skipRaw: true, + expectedLabelRule: &labeler.LabelRule{ + ID: "keyspaces/0", + Index: 0, + Labels: []labeler.RegionLabel{ + { + Key: "id", + Value: "0", + }, + }, + RuleType: "key-range", + Data: []any{ + map[string]any{ + "start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 0})), + "end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0, 1})), + }, + }, + }, + }, + { + id: 4242, + skipRaw: true, + expectedLabelRule: &labeler.LabelRule{ + ID: "keyspaces/4242", + Index: 0, + Labels: []labeler.RegionLabel{ + { + Key: "id", + Value: "4242", + }, + }, + RuleType: "key-range", + Data: []any{ + map[string]any{ + "start_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x92})), + "end_key": hex.EncodeToString(codec.EncodeBytes([]byte{'x', 0, 0x10, 0x93})), + }, + }, + }, + }, } for _, testCase := range testCases { - re.Equal(testCase.expectedLabelRule, MakeLabelRule(testCase.id)) + re.Equal(testCase.expectedLabelRule, makeLabelRule(testCase.id, testCase.skipRaw)) } } diff --git a/pkg/mcs/scheduling/server/rule/watcher_test.go b/pkg/mcs/scheduling/server/rule/watcher_test.go index 8c3782ac4ea..33a0d08b95d 100644 --- a/pkg/mcs/scheduling/server/rule/watcher_test.go +++ b/pkg/mcs/scheduling/server/rule/watcher_test.go @@ -16,6 +16,7 @@ package rule import ( "context" + "encoding/hex" "encoding/json" "os" "strconv" @@ -100,11 +101,17 @@ func prepare(t require.TestingT) (context.Context, *clientv3.Client, func()) { <-etcd.Server.ReadyNotify() for i := 1; i < rulesNum+1; i++ { + regionBound := keyspace.MakeRegionBound(uint32(i)) rule := &labeler.LabelRule{ ID: "test_" + strconv.Itoa(i), Labels: []labeler.RegionLabel{{Key: "test", Value: "test"}}, RuleType: labeler.KeyRange, - Data: keyspace.MakeKeyRanges(uint32(i)), + Data: labeler.MakeKeyRanges( + hex.EncodeToString(regionBound.TxnLeftBound), + hex.EncodeToString(regionBound.TxnRightBound), + hex.EncodeToString(regionBound.RawLeftBound), + hex.EncodeToString(regionBound.RawRightBound), + ), } value, err := json.Marshal(rule) re.NoError(err) diff --git a/server/config/config.go b/server/config/config.go index b222bf45b6c..aa012464a63 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -245,6 +245,7 @@ const ( defaultGCTunerThreshold = 0.6 minGCTunerThreshold = 0 maxGCTunerThreshold = 0.9 + defaultDisableRawKVRegionSplit = false defaultWaitRegionSplitTimeout = 30 * time.Second defaultCheckRegionSplitInterval = 50 * time.Millisecond @@ -877,6 +878,8 @@ type KeyspaceConfig struct { WaitRegionSplit bool `toml:"wait-region-split" json:"wait-region-split"` // WaitRegionSplitTimeout indicates the max duration to wait region split. WaitRegionSplitTimeout typeutil.Duration `toml:"wait-region-split-timeout" json:"wait-region-split-timeout"` + // DisableRawKVRegionSplit indicates whether to skip raw kv region split. + DisableRawKVRegionSplit bool `toml:"disable-raw-kv-region-split" json:"disable-raw-kv-region-split,string"` // CheckRegionSplitInterval indicates the interval to check whether the region split is complete CheckRegionSplitInterval typeutil.Duration `toml:"check-region-split-interval" json:"check-region-split-interval"` } @@ -903,6 +906,9 @@ func (c *KeyspaceConfig) adjust(meta *configutil.ConfigMetaData) { if !meta.IsDefined("check-region-split-interval") { c.CheckRegionSplitInterval = typeutil.NewDuration(defaultCheckRegionSplitInterval) } + if !meta.IsDefined("disable-raw-kv-region-split") { + c.DisableRawKVRegionSplit = defaultDisableRawKVRegionSplit + } } // Clone makes a deep copy of the keyspace config. @@ -923,6 +929,11 @@ func (c *KeyspaceConfig) ToWaitRegionSplit() bool { return c.WaitRegionSplit } +// GetDisableRawKVRegionSplit returns whether to skip raw kv region split. +func (c *KeyspaceConfig) GetDisableRawKVRegionSplit() bool { + return c.DisableRawKVRegionSplit +} + // GetWaitRegionSplitTimeout returns the max duration to wait region split. func (c *KeyspaceConfig) GetWaitRegionSplitTimeout() time.Duration { return c.WaitRegionSplitTimeout.Duration From b4444b84bac68f65dfcc916bc275c67a01e4caeb Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 24 Dec 2025 18:57:50 +0800 Subject: [PATCH 2/3] resolve conflicts Signed-off-by: Ryan Leung --- pkg/keyspace/keyspace.go | 49 ++++++---------------------------------- 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index 5a68502cde2..f43a86b1902 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/keypath" + "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "github.com/tikv/pd/pkg/versioninfo/kerneltype" ) @@ -530,48 +531,12 @@ func (manager *Manager) splitKeyspaceRegion(id uint32, waitRegionSplit bool) (er return err } - // Stat waiting for region split. - var splitKeys [][]byte - for _, regionRange := range keyspaceRule.Data.([]*labeler.KeyRangeRule) { - splitKeys = append(splitKeys, regionRange.StartKey, regionRange.EndKey) - } - tickerInterval := manager.config.GetCheckRegionSplitInterval() - ticker := time.NewTicker(tickerInterval) - timer := time.NewTimer(manager.config.GetWaitRegionSplitTimeout()) - defer func() { - ticker.Stop() - timer.Stop() - }() - for { - select { - case <-ticker.C: - // Reset ticker if it's interval has been changed. - newInterval := manager.config.GetCheckRegionSplitInterval() - if tickerInterval != newInterval { - tickerInterval = newInterval - ticker.Reset(newInterval) - } - regionsInfo := manager.cluster.GetBasicCluster().RegionsInfo - totalSplit := 0 // number of split keys that have been split. - for _, splitKey := range splitKeys { - region := regionsInfo.GetRegionByKey(splitKey) - if region != nil && bytes.Equal(region.GetStartKey(), splitKey) { - totalSplit++ - } - } - if totalSplit == len(splitKeys) { - log.Info("[keyspace] wait region split successfully", zap.Uint32("keyspace-id", id)) - return - } - case <-timer.C: - log.Warn("[keyspace] wait region split timeout", - zap.Uint32("keyspace-id", id), - zap.Error(err), - ) - err = ErrRegionSplitTimeout - return - } - } + log.Info("[keyspace] added region label for keyspace", + zap.Uint32("keyspace-id", id), + logutil.ZapRedactString("label-rule", keyspaceRule.String()), + zap.Duration("takes", time.Since(start)), + ) + return } func (manager *Manager) waitKeyspaceRegionSplit(id uint32) error { From 167c61a98e72f948a4b3bdda19dae3a4ca438777 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 24 Dec 2025 18:59:45 +0800 Subject: [PATCH 3/3] resolve conflicts Signed-off-by: Ryan Leung --- pkg/keyspace/util_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/keyspace/util_test.go b/pkg/keyspace/util_test.go index 517e65796e6..7a80fbdca1b 100644 --- a/pkg/keyspace/util_test.go +++ b/pkg/keyspace/util_test.go @@ -254,7 +254,7 @@ func TestParseKeyspaceIDFromLabelRule(t *testing.T) { }{ // Valid keyspace label rule. { - labelRule: MakeLabelRule(1), + labelRule: makeLabelRule(1, false), expectedID: 1, expectedOK: true, },