Skip to content

Commit 3171cd1

Browse files
pdrobnjakclaude
pdrobnjak
and
claude
committed
perf(store): replace sync.Map with plain map in cachekv stores
Within OCC, each worker gets its own CacheMultiStore chain with dedicated cachekv.Store instances — zero concurrent access. The sync.Map thread-safety overhead (CAS, internal node allocs) is pure waste in this single-goroutine context. Replace *sync.Map with typed Go maps in both sei-cosmos and giga cachekv implementations: - cache: map[string]*types.CValue - deleted: map[string]struct{} - unsortedCache: map[string]struct{} Keep sync.RWMutex for defense-in-depth (uncontested, <20ns). Micro-benchmarks: 32-56% faster across all cachekv operations, 4 allocs/op eliminated from Set paths. TPS benchmark: ~8,936 avg (+14.6% cumulative over origin/main). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 024e61c commit 3171cd1

3 files changed

Lines changed: 64 additions & 76 deletions

File tree

‎giga/deps/store/cachekv.go‎

Lines changed: 27 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import (
1313
// Store wraps an in-memory cache around an underlying types.KVStore.
1414
type Store struct {
1515
mtx sync.RWMutex
16-
cache *sync.Map
17-
deleted *sync.Map
16+
cache map[string]*types.CValue
17+
deleted map[string]struct{}
1818
parent types.KVStore
1919
storeKey types.StoreKey
2020
cacheSize int
@@ -25,8 +25,8 @@ var _ types.CacheKVStore = (*Store)(nil)
2525
// NewStore creates a new Store object
2626
func NewStore(parent types.KVStore, storeKey types.StoreKey, cacheSize int) *Store {
2727
return &Store{
28-
cache: &sync.Map{},
29-
deleted: &sync.Map{},
28+
cache: make(map[string]*types.CValue),
29+
deleted: make(map[string]struct{}),
3030
parent: parent,
3131
storeKey: storeKey,
3232
cacheSize: cacheSize,
@@ -44,8 +44,8 @@ func (store *Store) GetStoreType() types.StoreType {
4444

4545
// getFromCache queries the write-through cache for a value by key.
4646
func (store *Store) getFromCache(key []byte) []byte {
47-
if cv, ok := store.cache.Load(UnsafeBytesToStr(key)); ok {
48-
return cv.(*types.CValue).Value()
47+
if cv, ok := store.cache[UnsafeBytesToStr(key)]; ok {
48+
return cv.Value()
4949
}
5050
return store.parent.Get(key)
5151
}
@@ -84,12 +84,11 @@ func (store *Store) Write() {
8484
// Not the best, but probably not a bottleneck depending.
8585
keys := []string{}
8686

87-
store.cache.Range(func(key, value any) bool {
88-
if value.(*types.CValue).Dirty() {
89-
keys = append(keys, key.(string))
87+
for key, value := range store.cache {
88+
if value.Dirty() {
89+
keys = append(keys, key)
9090
}
91-
return true
92-
})
91+
}
9392
sort.Strings(keys)
9493
// TODO: Consider allowing usage of Batch, which would allow the write to
9594
// at least happen atomically.
@@ -103,10 +102,10 @@ func (store *Store) Write() {
103102
continue
104103
}
105104

106-
cacheValue, ok := store.cache.Load(key)
107-
if ok && cacheValue.(*types.CValue).Value() != nil {
105+
cacheValue, ok := store.cache[key]
106+
if ok && cacheValue.Value() != nil {
108107
// It already exists in the parent, hence delete it.
109-
store.parent.Set([]byte(key), cacheValue.(*types.CValue).Value())
108+
store.parent.Set([]byte(key), cacheValue.Value())
110109
}
111110
}
112111

@@ -115,14 +114,12 @@ func (store *Store) Write() {
115114
// writes immediately visible until Commit(). By keeping the cache populated
116115
// with clean entries, subsequent reads will still hit the cache instead of
117116
// falling through to the parent which can't read uncommitted data.
118-
store.cache.Range(func(key, value any) bool {
119-
cv := value.(*types.CValue)
117+
for key, cv := range store.cache {
120118
// Replace with a clean (non-dirty) version of the same value
121-
store.cache.Store(key, types.NewCValue(cv.Value(), false))
122-
return true
123-
})
119+
store.cache[key] = types.NewCValue(cv.Value(), false)
120+
}
124121
// Clear the deleted map since those deletes have been sent to parent
125-
store.deleted = &sync.Map{}
122+
store.deleted = make(map[string]struct{})
126123
}
127124

128125
// CacheWrap implements CacheWrapper.
@@ -144,16 +141,16 @@ func (store *Store) setCacheValue(key, value []byte, deleted bool, dirty bool) {
144141
types.AssertValidKey(key)
145142

146143
keyStr := UnsafeBytesToStr(key)
147-
store.cache.Store(keyStr, types.NewCValue(value, dirty))
144+
store.cache[keyStr] = types.NewCValue(value, dirty)
148145
if deleted {
149-
store.deleted.Store(keyStr, struct{}{})
146+
store.deleted[keyStr] = struct{}{}
150147
} else {
151-
store.deleted.Delete(keyStr)
148+
delete(store.deleted, keyStr)
152149
}
153150
}
154151

155152
func (store *Store) isDeleted(key string) bool {
156-
_, ok := store.deleted.Load(key)
153+
_, ok := store.deleted[key]
157154
return ok
158155
}
159156

@@ -173,20 +170,18 @@ func (store *Store) GetAllKeyStrsInRange(start, end []byte) (res []string) {
173170
for _, pk := range store.parent.GetAllKeyStrsInRange(start, end) {
174171
keyStrs[pk] = struct{}{}
175172
}
176-
store.cache.Range(func(key, value any) bool {
177-
kbz := []byte(key.(string))
173+
for key, cv := range store.cache {
174+
kbz := []byte(key)
178175
if bytes.Compare(kbz, start) < 0 || bytes.Compare(kbz, end) >= 0 {
179176
// we don't want to break out of the iteration since cache isn't sorted
180-
return true
177+
continue
181178
}
182-
cv := value.(*types.CValue)
183179
if cv.Value() == nil {
184-
delete(keyStrs, key.(string))
180+
delete(keyStrs, key)
185181
} else {
186-
keyStrs[key.(string)] = struct{}{}
182+
keyStrs[key] = struct{}{}
187183
}
188-
return true
189-
})
184+
}
190185
for k := range keyStrs {
191186
res = append(res, k)
192187
}

‎sei-cosmos/store/cachekv/memiterator.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package cachekv
22

33
import (
44
"bytes"
5-
"sync"
65

76
dbm "github.com/tendermint/tm-db"
87

@@ -16,13 +15,13 @@ type memIterator struct {
1615
types.Iterator
1716

1817
lastKey []byte
19-
deleted *sync.Map
18+
deleted map[string]struct{}
2019
}
2120

2221
func newMemIterator(
2322
start, end []byte,
2423
items *dbm.MemDB,
25-
deleted *sync.Map,
24+
deleted map[string]struct{},
2625
ascending bool,
2726
) *memIterator {
2827
var iter types.Iterator
@@ -56,7 +55,7 @@ func (mi *memIterator) Value() []byte {
5655
// then we are calling value on the same thing as last time.
5756
// Therefore we don't check the mi.deleted to see if this key is included in there.
5857
reCallingOnOldLastKey := (mi.lastKey != nil) && bytes.Equal(key, mi.lastKey)
59-
if _, ok := mi.deleted.Load(string(key)); ok && !reCallingOnOldLastKey {
58+
if _, ok := mi.deleted[string(key)]; ok && !reCallingOnOldLastKey {
6059
return nil
6160
}
6261
mi.lastKey = key

‎sei-cosmos/store/cachekv/store.go‎

Lines changed: 34 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ import (
1616
// Store wraps an in-memory cache around an underlying types.KVStore.
1717
type Store struct {
1818
mtx sync.RWMutex
19-
cache *sync.Map
20-
deleted *sync.Map
21-
unsortedCache *sync.Map
19+
cache map[string]*types.CValue
20+
deleted map[string]struct{}
21+
unsortedCache map[string]struct{}
2222
sortedCache *dbm.MemDB // always ascending sorted
2323
parent types.KVStore
2424
storeKey types.StoreKey
@@ -30,9 +30,9 @@ var _ types.CacheKVStore = (*Store)(nil)
3030
// NewStore creates a new Store object
3131
func NewStore(parent types.KVStore, storeKey types.StoreKey, cacheSize int) *Store {
3232
return &Store{
33-
cache: &sync.Map{},
34-
deleted: &sync.Map{},
35-
unsortedCache: &sync.Map{},
33+
cache: make(map[string]*types.CValue),
34+
deleted: make(map[string]struct{}),
35+
unsortedCache: make(map[string]struct{}),
3636
sortedCache: nil,
3737
parent: parent,
3838
storeKey: storeKey,
@@ -51,8 +51,8 @@ func (store *Store) GetStoreType() types.StoreType {
5151

5252
// getFromCache queries the write-through cache for a value by key.
5353
func (store *Store) getFromCache(key []byte) []byte {
54-
if cv, ok := store.cache.Load(conv.UnsafeBytesToStr(key)); ok {
55-
return cv.(*types.CValue).Value()
54+
if cv, ok := store.cache[conv.UnsafeBytesToStr(key)]; ok {
55+
return cv.Value()
5656
}
5757
return store.parent.Get(key)
5858
}
@@ -91,12 +91,11 @@ func (store *Store) Write() {
9191
// Not the best, but probably not a bottleneck depending.
9292
keys := []string{}
9393

94-
store.cache.Range(func(key, value any) bool {
95-
if value.(*types.CValue).Dirty() {
96-
keys = append(keys, key.(string))
94+
for key, value := range store.cache {
95+
if value.Dirty() {
96+
keys = append(keys, key)
9797
}
98-
return true
99-
})
98+
}
10099
sort.Strings(keys)
101100
// TODO: Consider allowing usage of Batch, which would allow the write to
102101
// at least happen atomically.
@@ -110,16 +109,16 @@ func (store *Store) Write() {
110109
continue
111110
}
112111

113-
cacheValue, ok := store.cache.Load(key)
114-
if ok && cacheValue.(*types.CValue).Value() != nil {
112+
cacheValue, ok := store.cache[key]
113+
if ok && cacheValue.Value() != nil {
115114
// It already exists in the parent, hence delete it.
116-
store.parent.Set([]byte(key), cacheValue.(*types.CValue).Value())
115+
store.parent.Set([]byte(key), cacheValue.Value())
117116
}
118117
}
119118

120-
store.cache = &sync.Map{}
121-
store.deleted = &sync.Map{}
122-
store.unsortedCache = &sync.Map{}
119+
store.cache = make(map[string]*types.CValue)
120+
store.deleted = make(map[string]struct{})
121+
store.unsortedCache = make(map[string]struct{})
123122
store.sortedCache = nil
124123
}
125124

@@ -281,16 +280,13 @@ func (store *Store) dirtyItems(start, end []byte) {
281280
// Even without that, too many range checks eventually becomes more expensive
282281
// than just not having the cache.
283282
// store.emitUnsortedCacheSizeMetric()
284-
store.unsortedCache.Range(func(key, value any) bool {
285-
cKey := key.(string)
283+
for cKey := range store.unsortedCache {
286284
if dbm.IsKeyInDomain(conv.UnsafeStrToBytes(cKey), start, end) {
287-
cacheValue, ok := store.cache.Load(key)
288-
if ok {
289-
unsorted = append(unsorted, &kv.Pair{Key: []byte(cKey), Value: cacheValue.(*types.CValue).Value()})
285+
if cacheValue, ok := store.cache[cKey]; ok {
286+
unsorted = append(unsorted, &kv.Pair{Key: []byte(cKey), Value: cacheValue.Value()})
290287
}
291288
}
292-
return true
293-
})
289+
}
294290
store.clearUnsortedCacheSubset(unsorted, stateUnsorted)
295291
return
296292
}
@@ -323,7 +319,7 @@ func (store *Store) clearUnsortedCacheSubset(unsorted []*kv.Pair, sortState sort
323319
func (store *Store) deleteKeysFromUnsortedCache(unsorted []*kv.Pair) {
324320
for _, kv := range unsorted {
325321
keyStr := conv.UnsafeBytesToStr(kv.Key)
326-
store.unsortedCache.Delete(keyStr)
322+
delete(store.unsortedCache, keyStr)
327323
}
328324
}
329325

@@ -335,19 +331,19 @@ func (store *Store) setCacheValue(key, value []byte, deleted bool, dirty bool) {
335331
types.AssertValidKey(key)
336332

337333
keyStr := conv.UnsafeBytesToStr(key)
338-
store.cache.Store(keyStr, types.NewCValue(value, dirty))
334+
store.cache[keyStr] = types.NewCValue(value, dirty)
339335
if deleted {
340-
store.deleted.Store(keyStr, struct{}{})
336+
store.deleted[keyStr] = struct{}{}
341337
} else {
342-
store.deleted.Delete(keyStr)
338+
delete(store.deleted, keyStr)
343339
}
344340
if dirty {
345-
store.unsortedCache.Store(keyStr, struct{}{})
341+
store.unsortedCache[keyStr] = struct{}{}
346342
}
347343
}
348344

349345
func (store *Store) isDeleted(key string) bool {
350-
_, ok := store.deleted.Load(key)
346+
_, ok := store.deleted[key]
351347
return ok
352348
}
353349

@@ -367,20 +363,18 @@ func (store *Store) GetAllKeyStrsInRange(start, end []byte) (res []string) {
367363
for _, pk := range store.parent.GetAllKeyStrsInRange(start, end) {
368364
keyStrs[pk] = struct{}{}
369365
}
370-
store.cache.Range(func(key, value any) bool {
371-
kbz := []byte(key.(string))
366+
for key, cv := range store.cache {
367+
kbz := []byte(key)
372368
if bytes.Compare(kbz, start) < 0 || bytes.Compare(kbz, end) >= 0 {
373369
// we don't want to break out of the iteration since cache isn't sorted
374-
return true
370+
continue
375371
}
376-
cv := value.(*types.CValue)
377372
if cv.Value() == nil {
378-
delete(keyStrs, key.(string))
373+
delete(keyStrs, key)
379374
} else {
380-
keyStrs[key.(string)] = struct{}{}
375+
keyStrs[key] = struct{}{}
381376
}
382-
return true
383-
})
377+
}
384378
for k := range keyStrs {
385379
res = append(res, k)
386380
}

0 commit comments

Comments
 (0)