Embedded Key Value Stores in Go Bolt Badger and Pebble

Not every program needs a database server. A CLI that remembers state, a service with a local index, a node in a cluster storing its own log, a cache that survives restarts: all want a place to put bytes by key, on local disk, without running anything. Go has three well known embedded key value stores that need no cgo and no separate process: bbolt, Badger and Pebble. They look similar from outside and are built on different ideas, which decide what each is good at. This roundup measures all three on the same load and gives a verdict by use case.

It belongs to the database cornerstone. Each store has its own article: bbolt in practice and Badger. For SQL in the same embedded role, see SQLite in Go.

What they are

An LSM tree (log structured merge tree) turns random writes into sequential ones: new data goes to a memory table and a log, and is merged into sorted files in the background. Writes are cheap; reads may check several files; background compaction costs CPU and disk. A B+tree updates pages in place: reads are direct, writes touch pages and must be made durable, so they cost more.

The measurement

package main

import (
	"encoding/binary"
	"fmt"
	"log"
	"math/rand/v2"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cockroachdb/pebble/v2"
	"github.com/cockroachdb/pebble/v2/bloom"
	"github.com/dgraph-io/badger/v4"
	bolt "go.etcd.io/bbolt"
)

const n = 100000

func key(i int) []byte {
	k := make([]byte, 8)
	binary.BigEndian.PutUint64(k, uint64(i))
	return k
}

// val returns 100 pseudo random bytes that differ for every key, so compression cannot flatter a store.
func val(i int) []byte {
	v := make([]byte, 100)
	r := rand.New(rand.NewPCG(uint64(i), 99))
	for j := range v {
		v[j] = byte(r.IntN(256))
	}
	return v
}

type store interface {
	Load(n int) error
	Get(i int) error
	Scan() (int, error)
	Close() error
}

// bbolt: one B+tree file, a single writer, fsync on every commit.
type boltStore struct{ db *bolt.DB }

func (s boltStore) Load(n int) error {
	for lo := 0; lo < n; lo += 1000 {
		if err := s.db.Update(func(tx *bolt.Tx) error {
			b, err := tx.CreateBucketIfNotExists([]byte("kv"))
			if err != nil {
				return err
			}
			for i := lo; i < lo+1000 && i < n; i++ {
				if err := b.Put(key(i), val(i)); err != nil {
					return err
				}
			}
			return nil
		}); err != nil {
			return err
		}
	}
	return nil
}

func (s boltStore) Get(i int) error {
	return s.db.View(func(tx *bolt.Tx) error {
		if tx.Bucket([]byte("kv")).Get(key(i)) == nil {
			return fmt.Errorf("missing %d", i)
		}
		return nil
	})
}

func (s boltStore) Scan() (c int, err error) {
	err = s.db.View(func(tx *bolt.Tx) error {
		cur := tx.Bucket([]byte("kv")).Cursor()
		for k, _ := cur.First(); k != nil; k, _ = cur.Next() {
			c++
		}
		return nil
	})
	return
}
func (s boltStore) Close() error { return s.db.Close() }

// Badger: an LSM tree with a value log; writes are buffered and the default does not fsync each one.
type badgerStore struct{ db *badger.DB }

func (s badgerStore) Load(n int) error {
	wb := s.db.NewWriteBatch()
	defer wb.Cancel()
	for i := 0; i < n; i++ {
		if err := wb.Set(key(i), val(i)); err != nil {
			return err
		}
	}
	return wb.Flush()
}

func (s badgerStore) Get(i int) error {
	return s.db.View(func(tx *badger.Txn) error {
		_, err := tx.Get(key(i))
		return err
	})
}

func (s badgerStore) Scan() (c int, err error) {
	err = s.db.View(func(tx *badger.Txn) error {
		opts := badger.DefaultIteratorOptions
		opts.PrefetchValues = false
		it := tx.NewIterator(opts)
		defer it.Close()
		for it.Rewind(); it.Valid(); it.Next() {
			c++
		}
		return nil
	})
	return
}
func (s badgerStore) Close() error { return s.db.Close() }

// Pebble: the LSM tree used by CockroachDB; batches committed without an fsync here, as with Badger's default.
type pebbleStore struct{ db *pebble.DB }

func (s pebbleStore) Load(n int) error {
	for lo := 0; lo < n; lo += 1000 {
		b := s.db.NewBatch()
		for i := lo; i < lo+1000 && i < n; i++ {
			if err := b.Set(key(i), val(i), nil); err != nil {
				return err
			}
		}
		if err := b.Commit(pebble.NoSync); err != nil {
			return err
		}
	}
	return s.db.Flush()
}

func (s pebbleStore) Get(i int) error {
	v, closer, err := s.db.Get(key(i))
	if err != nil {
		return err
	}
	_ = v
	return closer.Close()
}

func (s pebbleStore) Scan() (c int, err error) {
	it, err := s.db.NewIter(nil)
	if err != nil {
		return 0, err
	}
	defer it.Close()
	for it.First(); it.Valid(); it.Next() {
		c++
	}
	return c, it.Error()
}
func (s pebbleStore) Close() error { return s.db.Close() }

type quiet struct{}

func (quiet) Infof(string, ...any)              {}
func (quiet) Errorf(string, ...any)             {}
func (quiet) Fatalf(format string, args ...any) { log.Fatalf(format, args...) }

func dirSize(dir string) int64 {
	var total int64
	_ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
		if err == nil && !info.IsDir() {
			total += info.Size()
		}
		return nil
	})
	return total
}

func main() {
	root, err := os.MkdirTemp("", "kv640")
	if err != nil {
		log.Fatal(err)
	}
	defer os.RemoveAll(root)

	open := map[string]func(dir string) store{
		"bbolt": func(dir string) store {
			db, err := bolt.Open(filepath.Join(dir, "kv.db"), 0o600, nil)
			if err != nil {
				log.Fatal(err)
			}
			return boltStore{db}
		},
		"badger": func(dir string) store {
			db, err := badger.Open(badger.DefaultOptions(dir).WithLoggingLevel(badger.ERROR))
			if err != nil {
				log.Fatal(err)
			}
			return badgerStore{db}
		},
		"pebble": func(dir string) store {
			db, err := pebble.Open(dir, &pebble.Options{Logger: quiet{}})
			if err != nil {
				log.Fatal(err)
			}
			return pebbleStore{db}
		},
		"pebble+bloom": func(dir string) store {
			opts := &pebble.Options{Logger: quiet{}}
			for i := range opts.Levels {
				opts.Levels[i].FilterPolicy = bloom.FilterPolicy(10)
			}
			db, err := pebble.Open(dir, opts)
			if err != nil {
				log.Fatal(err)
			}
			return pebbleStore{db}
		},
	}
	fmt.Printf("%d keys of 8 bytes with 100 byte values, then %d random reads and one full scan\n", n, n)
	fmt.Printf("%-13s %10s %10s %10s %10s\n", "store", "load", "reads", "scan", "on disk")
	for _, name := range []string{"bbolt", "badger", "pebble", "pebble+bloom"} {
		dir := filepath.Join(root, strings.ReplaceAll(name, "+", "-"))
		_ = os.MkdirAll(dir, 0o755)
		s := open[name](dir)

		start := time.Now()
		if err := s.Load(n); err != nil {
			log.Fatal(name, err)
		}
		load := time.Since(start)

		r := rand.New(rand.NewPCG(1, 2))
		start = time.Now()
		for i := 0; i < n; i++ {
			if err := s.Get(r.IntN(n)); err != nil {
				log.Fatal(name, err)
			}
		}
		reads := time.Since(start)

		start = time.Now()
		c, err := s.Scan()
		if err != nil || c != n {
			log.Fatal(name, c, err)
		}
		scan := time.Since(start)
		_ = s.Close()
		fmt.Printf("%-13s %10v %10v %10v %8.1f MB\n", name, load.Round(time.Millisecond), reads.Round(time.Millisecond), scan.Round(time.Millisecond), float64(dirSize(dir))/1e6)
	}
}

100000 keys of 8 bytes with 100 byte values, then 100000 random reads and one full scan
store               load      reads       scan    on disk
bbolt             1.357s       96ms        1ms     33.6 MB
badger             173ms      167ms        9ms     13.3 MB
pebble             256ms      709ms       10ms     18.0 MB
pebble+bloom       250ms     1.197s        9ms     20.9 MB

The program loads 100,000 keys of 8 bytes with 100 byte values, each value different pseudo random data so that compression cannot flatter any store, then does 100,000 random point reads and one full ordered scan. It prints wall time and the size of the store on disk afterwards. The settings are the defaults of each library, except where the table notes them: bbolt commits 1,000 keys per transaction with an fsync each; Badger uses a write batch and its default of not syncing every write; Pebble commits batches of 1,000 with NoSync and flushes at the end. That makes load times a comparison of durability policy as much as of design, which is the honest way to read them.

The raw data is 10.8 MB (100,000 times 108 bytes), so the on disk figure is also the overhead of each design. Read the numbers in four ways.

Writes favour the LSM stores. Badger and Pebble loaded 5 to 8 times faster than bbolt, which spent most of the time on 100 fsyncs. If bbolt were given NoSync it would be much faster and would lose durability. Compare like with like before you choose: a store configured to lose the last second of writes on a crash is not doing the same job.

Reads favour the B+tree for a memory resident data set. bbolt answered 100,000 random reads in 96ms, about a microsecond each, since the whole 33 MB file is memory mapped and a lookup is a tree walk over cached pages. Badger took 167ms. Pebble took 709ms. The LSM stores may consult several files, and Pebble's default block cache and read path are tuned for data much larger than this test. At sizes beyond memory, LSM reads depend more on caching and filters, and the ranking can change. Do not take 100,000 keys as a model of 100 million.

Bloom filters are for misses. Pebble with a Bloom filter policy read slower, not faster. Every key in the test existed, so the filter was consulted and never saved a file read. Filters pay off on workloads with many lookups of absent keys. The lab shows that a tuning knob needs the right workload.

Space. bbolt's file was 33.6 MB for 10.8 MB of data, because pages are filled partially, and it never shrinks on its own (see the compaction step in the bbolt article). Badger was most compact in this run. Pebble's figure includes its write ahead log and manifest files.

Things the numbers hide

A benchmark of a hundred thousand keys on a laptop is a sketch. Four things it cannot show matter when you pick.

Crash behaviour. The test measures speed, and the reason to use a database is what happens when the power fails. bbolt's commit is durable when it returns. Badger and Pebble, as configured here, are not: a crash can lose the writes of the last moments, and both offer options to sync (SyncWrites in Badger, pebble.Sync on commit), which brings their load times much closer to bbolt's. Decide your durability requirement first, set each store to it, and only then compare speed.

Size over time. A store that is fast at 100 MB can behave differently at 100 GB. LSM stores need compaction, which uses disk bandwidth and can cause latency spikes under sustained writes; bbolt's file grows with data and reuses freed pages, but does not shrink. Test with data larger than memory and a workload shaped like yours, including deletes and overwrites.

Concurrency. The lab ran one goroutine. Under many writers, bbolt serialises them and the LSM stores overlap them, and under many readers all three scale well. The bbolt article shows how Batch changes the picture for small concurrent writes.

Operational surface. bbolt is one file you can copy while a read transaction is open (tx.WriteTo), which makes backup simple. Badger and Pebble are directories with several files and a defined procedure for consistent backup. All three need you to handle the file lock, disk full errors and a clean Close on shutdown, because a store that is not closed properly recovers from its log on the next start and takes longer to open.

Verdicts

bbolt: small to medium data, many reads, strict durability, simple needs. Config stores, indexes, metadata, state for tools. Its model is the easiest to reason about, since transactions are serialisable and a read never blocks. Its limits: a single writer, so write heavy workloads queue (the lab's 200 concurrent small writes took 1.87s with Update and 20ms with Batch), file growth that needs compaction, and a long read transaction that can stall a writer that needs to grow the file. It is archived in its original repository and maintained by the etcd team as go.etcd.io/bbolt; use that path.

Badger: write heavy, large values, TTLs, and many concurrent writers. It has transactions with optimistic conflict detection, TTLs, key only iteration and a design that keeps large values out of the LSM tree so compaction stays cheap. The costs are operational: a value log that needs garbage collection, several background goroutines, memory settings to tune and a transaction model where concurrent writers to the same key get ErrConflict and must retry (1,024 retries for 200 increments in the lab's contention test).

Pebble: when you are building a database or need to tune an LSM tree. It is the storage engine of CockroachDB, written in Go to replace RocksDB, with fine control over levels, filters, compression, block caches and snapshots, and iterators with bounds. It gives you less out of the box: no multi key transactions, no TTL and a tuning surface that rewards study. Pick it when you can use that control, and when you want a well supported LSM implementation that is not cgo.

What none of them is. They are key value stores, so there is no query language, no secondary indexes and no schema. Anything beyond get, put, delete and ordered range scan is code you write: key encoding, index maintenance, serialisation. Encode keys so that byte order is the order you want, with big endian integers as the lab does, and fixed prefixes for each kind of record. If you need a relational model in an embedded database, SQLite, pure Go through modernc.org/sqlite or with cgo, as in using SQLite in Go, is usually the better tool and saves you writing a query engine.

Choosing quickly

All three open a directory or file exclusively, so only one process can use it. If a second process must read the data, run a server in front, or use SQLite in WAL mode. The projects are at bbolt, Badger and Pebble, and each repository documents its tuning options; pin the module version, as the APIs changed between major versions.