Badger in Go as a Fast Embedded Key Value Store Option

Badger is a key value database written in pure Go, built by Dgraph as the storage layer for their graph database. It is an LSM tree with a twist: keys and small values live in the tree, while large values are written to a separate value log, so compaction moves keys and not megabytes of data. It gives you transactions, TTLs, iteration and backup, and it is fast at writes. This guide covers using Badger v4 on Go 1.25.6 with a lab that prints what each feature does, including three behaviours that surprised me while writing it.

It belongs to the database cornerstone and is compared with bbolt and Pebble. The B+tree alternative, with its single writer, is in bbolt in practice.

Opening it

package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"sync"
	"time"

	"github.com/dgraph-io/badger/v4"
)

func must(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	dir, err := os.MkdirTemp("", "badger642")
	must(err)
	defer os.RemoveAll(dir)
	db, err := badger.Open(badger.DefaultOptions(dir).WithLoggingLevel(badger.ERROR))
	must(err)
	defer db.Close()

	// 1. Set, get and a missing key.
	must(db.Update(func(tx *badger.Txn) error { return tx.Set([]byte("user:ada"), []byte("pro")) }))
	must(db.View(func(tx *badger.Txn) error {
		item, err := tx.Get([]byte("user:ada"))
		if err != nil {
			return err
		}
		val, _ := item.ValueCopy(nil)
		_, miss := tx.Get([]byte("user:nobody"))
		fmt.Printf("1. user:ada = %s, a missing key: %v (ErrKeyNotFound: %v)\n", val, miss, errors.Is(miss, badger.ErrKeyNotFound))
		return nil
	}))

	// 2. TTL: entries that expire on their own. The expiry time is stored in whole seconds.
	must(db.Update(func(tx *badger.Txn) error {
		if err := tx.SetEntry(badger.NewEntry([]byte("session:short"), []byte("abc")).WithTTL(300 * time.Millisecond)); err != nil {
			return err
		}
		return tx.SetEntry(badger.NewEntry([]byte("session:long"), []byte("abc")).WithTTL(2 * time.Second))
	}))
	exists := func(k string) bool {
		return db.View(func(tx *badger.Txn) error { _, err := tx.Get([]byte(k)); return err }) == nil
	}
	fmt.Printf("2. right after writing: 300ms TTL entry present: %v, 2s TTL entry present: %v\n", exists("session:short"), exists("session:long"))
	time.Sleep(2500 * time.Millisecond)
	fmt.Printf("   after 2.5 seconds: 300ms TTL entry present: %v, 2s TTL entry present: %v\n", exists("session:short"), exists("session:long"))

	// 3. A prefix scan over keys only, which is cheap because values are not read.
	must(db.Update(func(tx *badger.Txn) error {
		for i := 0; i < 5; i++ {
			if err := tx.Set([]byte(fmt.Sprintf("order:%03d", i)), []byte("x")); err != nil {
				return err
			}
		}
		return tx.Set([]byte("zzz"), []byte("x"))
	}))
	var keys []string
	must(db.View(func(tx *badger.Txn) error {
		opts := badger.DefaultIteratorOptions
		opts.PrefetchValues = false
		opts.Prefix = []byte("order:")
		it := tx.NewIterator(opts)
		defer it.Close()
		for it.Rewind(); it.Valid(); it.Next() {
			keys = append(keys, string(it.Item().Key()))
		}
		return nil
	}))
	fmt.Println("3. keys with the prefix order::", keys)

	// 4. Two transactions that read and write the same key: the second to commit conflicts.
	must(db.Update(func(tx *badger.Txn) error { return tx.Set([]byte("counter"), []byte{0}) }))
	t1 := db.NewTransaction(true)
	t2 := db.NewTransaction(true)
	for _, t := range []*badger.Txn{t1, t2} {
		item, _ := t.Get([]byte("counter"))
		v, _ := item.ValueCopy(nil)
		_ = t.Set([]byte("counter"), []byte{v[0] + 1})
	}
	e1 := t1.Commit()
	e2 := t2.Commit()
	fmt.Println("4. two read-modify-write transactions on one key: first commit", e1, "| second commit", e2, "| ErrConflict:", errors.Is(e2, badger.ErrConflict))

	// 5. Retrying on a conflict makes concurrent increments exact.
	var wg sync.WaitGroup
	retries := 0
	var mu sync.Mutex
	for w := 0; w < 8; w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for i := 0; i < 25; i++ {
				for {
					err := db.Update(func(tx *badger.Txn) error {
						item, err := tx.Get([]byte("hits"))
						n := 0
						if err == nil {
							_ = item.Value(func(v []byte) error { n = int(v[0]); return nil })
						}
						return tx.Set([]byte("hits"), []byte{byte(n + 1)})
					})
					if errors.Is(err, badger.ErrConflict) {
						mu.Lock()
						retries++
						mu.Unlock()
						continue
					}
					must(err)
					break
				}
			}
		}()
	}
	wg.Wait()
	var hits int
	must(db.View(func(tx *badger.Txn) error {
		item, _ := tx.Get([]byte("hits"))
		return item.Value(func(v []byte) error { hits = int(v[0]); return nil })
	}))
	fmt.Printf("5. eight goroutines, 25 increments each, retrying on ErrConflict: %d (want 200), %d retries\n", hits, retries)

	// 6. Deleted data stays until compaction: the value log needs garbage collection.
	must(db.Update(func(tx *badger.Txn) error { return tx.Delete([]byte("user:ada")) }))
	err = db.RunValueLogGC(0.5)
	fmt.Println("6. RunValueLogGC on a tiny database:", err, "(ErrNoRewrite means there was nothing worth rewriting)")
	lsm, vlog := db.Size()
	fmt.Println("   sizes reported by db.Size() are non-negative:", lsm >= 0 && vlog >= 0)
}

1. user:ada = pro, a missing key: Key not found (ErrKeyNotFound: true)
2. right after writing: 300ms TTL entry present: false, 2s TTL entry present: true
   after 2.5 seconds: 300ms TTL entry present: false, 2s TTL entry present: false
3. keys with the prefix order:: [order:000 order:001 order:002 order:003 order:004]
4. two read-modify-write transactions on one key: first commit <nil> | second commit Transaction Conflict. Please retry | ErrConflict: true
5. eight goroutines, 25 increments each, retrying on ErrConflict: 200 (want 200), 1024 retries
6. RunValueLogGC on a tiny database: Value log GC attempt didn't result in any cleanup (ErrNoRewrite means there was nothing worth rewriting)
   sizes reported by db.Size() are non-negative: true

badger.Open(badger.DefaultOptions(dir)) opens or creates a database in a directory. Options are set by chained With... methods: the lab sets WithLoggingLevel(badger.ERROR) to keep the output quiet. Defaults worth knowing: SyncWrites is false, so a commit is acknowledged before it is flushed to disk, and a crash can lose the last moments of writes; set it to true if you cannot accept that, and expect writes to slow down to the speed of your disk's fsync. Badger holds an exclusive lock on the directory, so one process at a time can use it, and it starts several background goroutines, so always call db.Close(); closing flushes memory tables and the next start is faster for it.

Transactions

Every read and write happens in a transaction. db.View runs a read only transaction, db.Update a read write one, and both take a function, with the transaction committed or discarded for you. A transaction sees a snapshot of the database from the moment it began, so reads inside it are consistent, and writes are buffered until commit.

The first block of the lab sets a key, reads it back and looks up a key that does not exist. tx.Get returns badger.ErrKeyNotFound for a missing key, which you test with errors.Is. The value of an item is valid only inside the transaction: item.ValueCopy(nil) makes a copy you may keep, and item.Value(func(v []byte) error) gives a view that must not escape the callback. Using the view after the transaction ends reads memory that has been reused, one of the common sources of corrupted data in Badger programs.

Conflicts and retries

Badger's transactions are optimistic. They do not lock; at commit, Badger checks whether any key your transaction read was written by another transaction that committed after yours began, and if so your commit fails with badger.ErrConflict. The fourth block shows it: two transactions each read counter, each added one, and the first commit returned nil while the second returned Transaction Conflict. Please retry, and errors.Is(err, badger.ErrConflict) was true. Without the check the second would have overwritten the first and one increment would have been lost.

So any read modify write must be in a retry loop. The fifth block does it: eight goroutines each increment a shared counter 25 times, and on ErrConflict they retry the whole function. The result was exactly 200, with 1,024 retries. The correctness is the point; the retry count is the cost. All the goroutines were fighting for one key, which is the worst case for optimistic concurrency, and each conflict throws away the work of a transaction. Keys that are updated by many writers at once are a design problem: shard the counter into several keys and sum them, or serialise those writes through one goroutine.

Two practical rules follow. Keep the retried function free of side effects, since it may run several times. And only read what you need inside a write transaction, because every key read joins the conflict set; a transaction that scans a large range and writes one key conflicts with any change in that range.

For many independent writes there is a faster path. db.NewWriteBatch() collects Set calls with no reads, splits them into transactions and commits them concurrently, which is how bulk loading should be done. It is what the roundup uses to load 100,000 keys in 165ms.

TTLs and their granularity

Badger can expire entries: badger.NewEntry(key, value).WithTTL(d) set with tx.SetEntry. Expired entries are invisible to reads and removed during compaction. The second block of the lab wrote two entries, one with a 300 millisecond TTL and one with 2 seconds, and checked for them right away:

• the 300ms entry was not present, even immediately after it was written;
• the 2s entry was present, and gone after 2.5 seconds.

Badger stores the expiry as a Unix timestamp in whole seconds. WithTTL adds the duration to the current time and truncates to seconds, so a sub second TTL expires at the start of the current second, which has usually already passed. The sensible reading is that TTLs have a resolution of one second and anything shorter than that is unreliable, with the exact error up to a second shorter than asked. Use TTLs of several seconds or more, and do not use them for precise timing. For a rate limiter or lock that needs 100ms accuracy, keep the deadline in the value and compare it yourself.

Iteration and prefixes

Badger keeps keys sorted, so ranges and prefixes are cheap. The third block writes five order: keys and one zzz key and iterates with opts.Prefix = []byte("order:"), which returned only the five. Two settings decide the cost:

• PrefetchValues: when true (the default) the iterator reads values ahead for a number of items, which helps when you will use the values and wastes work when you will not. The lab sets it to false because it only collects keys, and a keys only scan never touches the value log.
• Reverse: iterate in descending order, useful for "the latest N" under a time ordered key. When reversing with a prefix, seek to the prefix followed by 0xFF rather than to Rewind.

Always defer it.Close(): an iterator holds resources, and Badger complains if a transaction is discarded with an open iterator. Key design is as in any sorted store: use big endian numbers, fixed prefixes per record type and separators that cannot appear in your ids.

Deletes, compaction and the value log

Deleting a key writes a marker, and the space comes back only when compaction rewrites the files, in the LSM tree. Large values live in the value log, which is cleaned by a different mechanism: you must call db.RunValueLogGC(discardRatio) yourself. It rewrites a value log file if at least that fraction of it is garbage, and returns badger.ErrNoRewrite when no file qualifies. The last block of the lab calls it on a tiny database and gets exactly that, which is the normal result, not a failure.

In a service that overwrites or deletes large values, run it periodically from a goroutine, for example every five minutes with a ratio of 0.5, and loop while it keeps returning nil, because each call rewrites at most one file. Without that, the value log grows without bound. The ValueThreshold option sets the size at which a value moves from the LSM tree to the value log; a higher threshold keeps more in the tree, speeding reads of mid sized values and raising compaction cost.

Tuning and operating

Badger has many options, and a few matter most.

• Memory. MemTableSize, BlockCacheSize and IndexCacheSize set how much RAM it uses. The defaults assume a machine with room to spare; in a container with a small memory limit, reduce them or the process will be killed.
• Compression. Blocks are compressed with Snappy or Zstd depending on options, which helps with repetitive values and wastes CPU on random data.
• Concurrency. NumCompactors and NumGoroutines bound the background work.
• Backup. db.Backup(w, since) streams a consistent snapshot, incrementally if you pass the previous version, and db.Load restores it. Copying the directory of a running database is not a safe backup.
• Metrics. db.Size() reports the LSM and value log sizes in bytes. Export them, with the number of retries you see on ErrConflict.

A worked design: a session store

Take a concrete job: store web sessions with an expiry, look one up by token, and list all sessions of a user to revoke them. Badger gives key lookups and prefix scans, so the design is two key families. s:<token> holds the session data with a TTL. u:<user id>:<token> holds nothing, with the same TTL, and serves the "all sessions of this user" scan. Creating a session writes both in one Update, so they appear and disappear together. Looking one up is a Get on the first key. Revoking everything for a user is a prefix scan over u:<user id>: that collects tokens (keys only, no value reads) and deletes both keys for each, in a write batch.

Three consequences follow from what the lab showed. The TTL has whole second resolution, so a session set to expire in a second may live from zero to one second; use durations of minutes and treat the TTL as a clean up mechanism, with an expires field in the value checked on read for exactness. Both keys are written in a single transaction, and writes by different users touch different keys, so conflicts are rare. And expired entries vanish from reads at once and from disk only after compaction, so the database may be larger than the live data, which matters when you size the volume.

Streaming and bulk work

Two APIs exist for work on many keys at once. db.NewStream() iterates the whole database in parallel across key ranges, which is the right tool for exports, rebuilds and full scans: you set a Prefix, a NumGo for the number of goroutines, and a Send callback that receives batches. It is far faster than a single iterator on a large database, and the callback must be safe to call from one goroutine at a time as documented. The matching tool for loading is db.NewStreamWriter(), which writes sorted data directly into the tree and is how a database is restored or bulk imported much faster than ordinary writes; it needs the data in key order and an empty database or range.

For ordinary bulk writes, NewWriteBatch is enough. Remember that its writes do not read, so they never conflict, and that a failure partway leaves part of the data written, because batches are split into transactions. If you need all or nothing, use one Update, and respect the transaction size limit: a transaction that holds too many writes fails with badger.ErrTxnTooBig, which you handle by committing the transaction and starting another.

Errors and what they mean

A short list covers most of what you will see:

• ErrKeyNotFound: the key does not exist or has expired. Expected, not a failure.
• ErrConflict: an optimistic conflict. Retry the whole transaction function.
• ErrTxnTooBig: the transaction holds more writes than the memtable allows. Commit and continue in a new transaction.
• ErrNoRewrite: from value log GC, meaning nothing to clean up now.
• ErrDBClosed and ErrReadOnlyTxn: use after close, and a write attempted in a View.
• A lock error on Open: another process, or a copy of your program that was not shut down, has the directory.

Treat ErrConflict as normal under load and count it. A rising rate of conflicts is the signal that a key has become a hotspot. Treat ErrTxnTooBig as a bug in the code that built the transaction. Log everything else with the database path, since a store that fails to open usually points at permissions, a full disk or a second process.

Upgrades and compatibility

Major versions of Badger can change the on disk format, so check the release notes before moving a data directory across one: the safe path when a format changes is to back up with the old version and restore with the new one. Import path matters as well, since github.com/dgraph-io/badger/v4 is separate from earlier majors and the project's ownership and documentation locations have moved over the years. Pin the version in go.mod, read the release notes before upgrading, and test an upgrade on a copy of real data, including a restore from a backup, before you rely on either.

Testing

Test with real databases in a temporary directory, as the lab does. Badger also has an in-memory mode, badger.DefaultOptions("").WithInMemory(true), which is fast and good for unit tests, though it behaves differently on durability and recovery, so keep a few tests on disk. Test your retry loop by running many goroutines against one key under the race detector and asserting the exact final count, which is how the lab found that correctness survives a thousand conflicts. Test TTL behaviour with seconds, never milliseconds.

When to choose Badger

Choose it for write heavy embedded storage, for values large enough that keeping them out of the tree matters, for TTLs on entries, and when you want transactions without a server. Be ready to operate it: value log garbage collection, memory tuning, retries on conflict and a clean shutdown. Prefer bbolt when the data is modest, reads dominate and you want the simplest model, and prefer SQLite when you need queries. Badger's design is described in its documentation and the code is on GitHub, with the options listed on pkg.go.dev.