BoltDB and bbolt in Go With Practical Patterns That Work

bbolt is the database you use when you want a file of data with transactions and nothing else: no server, no query language, no configuration to speak of. It is a pure Go key value store built on a copy on write B+tree in a single memory mapped file. It descends from Bolt, the project that Ben Johnson wrote after LMDB and later archived, and the maintained fork lives under the etcd organisation as go.etcd.io/bbolt; etcd itself stores its data in it. Its appeal is that it is small enough to understand completely. This guide goes through the patterns that work, and the parts of its behaviour that catch people, on Go 1.25.6 with bbolt v1.4.3, with a lab that prints what each does.

It belongs to the database cornerstone. It is one of the three stores compared in embedded key value stores, next to Badger.

The model

A bbolt database is a file. Inside it live buckets, named collections of key and value pairs, which can nest. Keys and values are byte slices, kept sorted by key in byte order. Everything happens in a transaction: db.Update runs a read write transaction, db.View a read only one, and both take a function. If the function returns an error, an update is rolled back; if it returns nil, it is committed with an fsync, so a committed transaction survives a crash.

The concurrency rule is the one to remember: one writer at a time, any number of readers, and readers never block the writer or each other. Each read transaction sees a consistent snapshot of the database as it was when it started.

package main

import (
	"bytes"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	bolt "go.etcd.io/bbolt"
)

func must(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func size(path string) string {
	st, _ := os.Stat(path)
	return fmt.Sprintf("%.1f MB", float64(st.Size())/1e6)
}

func main() {
	dir, err := os.MkdirTemp("", "bolt641")
	must(err)
	defer os.RemoveAll(dir)
	path := filepath.Join(dir, "app.db")

	db, err := bolt.Open(path, 0o600, &bolt.Options{Timeout: 500 * time.Millisecond, InitialMmapSize: 64 << 20})
	must(err)
	defer db.Close()

	// 1. Buckets, nested buckets and values.
	must(db.Update(func(tx *bolt.Tx) error {
		users, err := tx.CreateBucketIfNotExists([]byte("users"))
		if err != nil {
			return err
		}
		for _, u := range []string{"ada", "alan", "grace", "linus"} {
			if err := users.Put([]byte(u), []byte(`{"plan":"free"}`)); err != nil {
				return err
			}
		}
		prefs, err := users.CreateBucketIfNotExists([]byte("ada:prefs"))
		if err != nil {
			return err
		}
		return prefs.Put([]byte("theme"), []byte("dark"))
	}))
	must(db.View(func(tx *bolt.Tx) error {
		v := tx.Bucket([]byte("users")).Get([]byte("ada"))
		missing := tx.Bucket([]byte("users")).Get([]byte("nobody"))
		fmt.Printf("1. ada = %s, a missing key returns nil: %v\n", v, missing == nil)
		return nil
	}))

	// 2. Keys come back in byte order, so a prefix scan is a seek and a loop.
	must(db.View(func(tx *bolt.Tx) error {
		c := tx.Bucket([]byte("users")).Cursor()
		var found []string
		for k, v := c.Seek([]byte("al")); k != nil && bytes.HasPrefix(k, []byte("al")); k, v = c.Next() {
			if v == nil { // a nil value marks a nested bucket
				continue
			}
			found = append(found, string(k))
		}
		fmt.Println("2. keys starting with al:", found)
		return nil
	}))

	// 3. An error returned from Update rolls the whole transaction back.
	err = db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("users"))
		_ = b.Put([]byte("temp"), []byte("x"))
		return errors.New("something failed after the first write")
	})
	_ = db.View(func(tx *bolt.Tx) error {
		fmt.Printf("3. Update returned %q, and the write is gone: %v\n", err, tx.Bucket([]byte("users")).Get([]byte("temp")) == nil)
		return nil
	})

	// 4. A read only transaction cannot write.
	err = db.View(func(tx *bolt.Tx) error {
		return tx.Bucket([]byte("users")).Put([]byte("x"), []byte("y"))
	})
	fmt.Println("4. writing inside View:", err)

	// 5. Readers do not block the writer, and a reader keeps its own snapshot.
	readerStarted := make(chan struct{})
	writerDone := make(chan struct{})
	var seen string
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		_ = db.View(func(tx *bolt.Tx) error {
			close(readerStarted)
			<-writerDone
			seen = string(tx.Bucket([]byte("users")).Get([]byte("ada")))
			return nil
		})
	}()
	<-readerStarted
	start := time.Now()
	must(db.Update(func(tx *bolt.Tx) error {
		return tx.Bucket([]byte("users")).Put([]byte("ada"), []byte(`{"plan":"pro"}`))
	}))
	took := time.Since(start)
	close(writerDone)
	wg.Wait()
	_ = db.View(func(tx *bolt.Tx) error {
		fmt.Printf("5. writer committed in %v while a read transaction was open; that reader still saw %s, a new one sees %s\n", took.Round(time.Millisecond), seen, tx.Bucket([]byte("users")).Get([]byte("ada")))
		return nil
	})

	// 6. Writers take turns: Batch combines concurrent small writes into one commit.
	count := func(label string, write func(i int) error) {
		start := time.Now()
		var wg sync.WaitGroup
		for i := 0; i < 200; i++ {
			wg.Add(1)
			go func() { defer wg.Done(); must(write(i)) }()
		}
		wg.Wait()
		fmt.Printf("6. 200 concurrent small writes with %-8s %v\n", label, time.Since(start).Round(time.Millisecond))
	}
	count("Update:", func(i int) error {
		return db.Update(func(tx *bolt.Tx) error {
			b, _ := tx.CreateBucketIfNotExists([]byte("events"))
			return b.Put([]byte(fmt.Sprintf("u%04d", i)), []byte("x"))
		})
	})
	count("Batch:", func(i int) error {
		return db.Batch(func(tx *bolt.Tx) error {
			b, _ := tx.CreateBucketIfNotExists([]byte("events"))
			return b.Put([]byte(fmt.Sprintf("b%04d", i)), []byte("x"))
		})
	})

	// 7. The file is locked: a second open gives up after its timeout.
	_, err = bolt.Open(path, 0o600, &bolt.Options{Timeout: 300 * time.Millisecond})
	fmt.Println("7. opening the same file again:", err)

	// 8. The file does not shrink when data is deleted; compaction copies the live data to a new file.
	must(db.Update(func(tx *bolt.Tx) error {
		b, _ := tx.CreateBucketIfNotExists([]byte("bulk"))
		for i := 0; i < 50000; i++ {
			if err := b.Put([]byte(fmt.Sprintf("k%06d", i)), make([]byte, 200)); err != nil {
				return err
			}
		}
		return nil
	}))
	grown := size(path)
	must(db.Update(func(tx *bolt.Tx) error { return tx.DeleteBucket([]byte("bulk")) }))
	afterDelete := size(path)
	dst, err := bolt.Open(filepath.Join(dir, "compact.db"), 0o600, nil)
	must(err)
	must(bolt.Compact(dst, db, 65536))
	must(dst.Close())
	fmt.Printf("8. file after 50,000 inserts %s, after deleting them %s, a compacted copy %s\n", grown, afterDelete, size(filepath.Join(dir, "compact.db")))
}

1. ada = {"plan":"free"}, a missing key returns nil: true
2. keys starting with al: [alan]
3. Update returned "something failed after the first write", and the write is gone: true
4. writing inside View: tx not writable
5. writer committed in 10ms while a read transaction was open; that reader still saw {"plan":"free"}, a new one sees {"plan":"pro"}
6. 200 concurrent small writes with Update:  1.87s
6. 200 concurrent small writes with Batch:   20ms
7. opening the same file again: timeout
8. file after 50,000 inserts 39.7 MB, after deleting them 39.7 MB, a compacted copy 0.3 MB

Buckets, keys and values

The first block creates a bucket named users, puts four keys in it and a nested bucket, and reads one back. CreateBucketIfNotExists is the idempotent form to use at start-up. A Get of a key that does not exist returns nil, not an error, which is also how an empty value looks, so use Get for "is there something" and be careful storing empty values. Returned slices are valid only inside the transaction: copy them if you keep them, since the memory is the file's mapping and changes underneath you.

A nil value from a cursor marks a nested bucket, as the scan in the lab checks. Nested buckets are good for grouping, such as the preferences of one user under their key, and each level costs a page. Do not nest deeply.

Cursors and prefix scans

Because keys are sorted, a range or prefix scan is a seek followed by a loop. The lab's second block calls c.Seek([]byte("al")) and iterates while the key still has that prefix, finding alan. This is the main way to query a key value store: design keys so the things you need to scan together are neighbours. For time series, big endian timestamps sort chronologically; for a one to many relation, a key like user:ada:order:0042 lets a prefix scan of user:ada:order: return all of a user's orders in order. Write numbers with binary.BigEndian for keys, never as decimal text, or 10 sorts before 9.

bucket.ForEach is simpler for a full scan, and cursor.First, Last, Next and Prev give control. bucket.Stats() and NextSequence() help with sizes and auto-incrementing ids.

Transactions

Three behaviours in the lab matter in practice. An error returned from the Update function rolls back everything the function wrote, shown by the third block: the Put of temp was undone and the error was returned. That makes a multi step change atomic by construction, as with a SQL transaction. A write inside View fails with tx not writable (block 4), so a function you pass to View cannot modify data by mistake. And a panic inside the function also rolls back.

Never start a transaction inside another from the same goroutine. A read transaction followed by a write transaction in the same goroutine can deadlock, as the next section shows.

Readers, the writer and the deadlock

The fifth block holds a read transaction open in one goroutine, commits a write from another, and then reads. The writer committed in 10ms without waiting for the reader, and the reader still saw {"plan":"free"} after the commit, while a new read transaction saw {"plan":"pro"}. That is snapshot isolation: the open reader has its own consistent view, and the copy on write design keeps the old pages alive until it finishes.

The first version of this lab failed in a way worth recording. It opened the database with default options, held a read transaction open and tried the same write. The program deadlocked: the writer blocked and nothing could proceed. The cause is documented in bbolt: when the file has to grow, the writer must remap the file into memory, and it cannot remap while any read transaction holds the old mapping. With the default initial map size of 32 KB, even a tiny write can need a remap, and it waited for a reader that was waiting on the writer. The lab now sets InitialMmapSize: 64 << 20, so the map is large enough that small writes need no remap.

Two lessons. Do not hold read transactions open for long, and never wait on a write from inside a View. A long read transaction also pins old pages, so the file grows while it lives. And for any database whose size you can estimate, set InitialMmapSize above the expected size, which removes remaps and the stall. Open with a Timeout as well, so that opening an already locked file fails instead of hanging.

One writer, and Batch

Writers take turns, and each Update commits with an fsync, so its cost is dominated by disk latency, a few milliseconds each. The sixth block runs 200 small writes from 200 goroutines two ways. With db.Update they took 1.87s: each waited for its turn and paid its own fsync. With db.Batch they took 20ms. Batch collects calls that arrive close together and runs them in one transaction with one fsync, up to a limit of size or delay, which you can tune with MaxBatchSize and MaxBatchDelay.

Use Batch when many goroutines each do a small independent write, as in a request handler recording an event. Two conditions apply. The function may run more than once: if one function in a batch returns an error, bbolt re-runs the others in a new transaction, so the function must be idempotent and free of side effects other than the database. And Batch helps only with concurrent callers: a single goroutine calling it in a loop gets nothing.

For bulk loads from one goroutine, put many writes in one Update, as the comparison in the roundup does with 1,000 keys per transaction. Setting db.NoSync = true during a bulk load and calling db.Sync() at the end is faster and risks the loaded data on a crash, so use it only when you can redo the load.

The file lock

bbolt takes an exclusive lock on the file, so one process at a time can have it open, and Open blocks until it gets the lock. The seventh block opens the same file again with a 300 millisecond timeout and receives timeout. Always set Options.Timeout: with the default of zero, a second instance of your program waits forever, which looks like a hang. For a read only inspection of a database another process holds, copy the file, or use ReadOnly: true when no writer is running. If several processes need the data, use a server.

The file never shrinks

The eighth block inserts 50,000 records of 200 bytes, a file of 39.7 MB, then deletes the bucket. The file is still 39.7 MB. bbolt keeps freed pages on a free list and reuses them for later writes, and it never gives space back to the operating system. A database that has grown and been pruned stays big. The remedy is compaction: bolt.Compact(dst, src, txMaxSize) copies all live data into a fresh file, and the result here was 0.3 MB. In a running service, compact during a quiet period or at start-up by writing a new file and replacing the old one. The bbolt compact command line tool does the same offline.

Practical patterns

• Keep values small and encode them once. A bucket of JSON or Protocol Buffers values is fine. For large blobs store files on disk and keep the path.
• Wrap the database in your own type with methods that take and return Go values, so no transaction escapes and no byte slices leak.
• Use transactions to maintain secondary indexes. An index is another bucket whose keys are the indexed value plus the primary key, and updating both inside one Update keeps them consistent.
• Back up with tx.WriteTo(w) inside a View: it streams a consistent snapshot while writers continue.
• Check the error from Close and close on shutdown, so the file is left clean.
• Test with a temporary directory per test and a real file, as the lab does; there is nothing to mock.

Designing keys and a secondary index

Everything you can ask of bbolt is "the value for this key" and "the keys in this range", so the data model is a set of key layouts. Suppose you store orders and want to list a customer's orders and find orders by status. The primary bucket orders is keyed by the order id. A second bucket, orders_by_customer, holds keys made of the customer id, a separator and the order id, with empty values. Listing a customer's orders is a prefix scan of that bucket followed by a lookup in orders for each id. A bucket orders_by_status works the same way for the status. All three writes happen inside one Update, so the index can never disagree with the data: a crash leaves either all of them or none.

Two details make or break this. The separator must not appear in the ids, or ada and ada:2 can collide; use a byte such as 0x00 or fixed width ids. And changing an indexed field means deleting the old index key and inserting the new one in the same transaction, so wrap the whole change in one function so no caller can update the order and forget the index. This is what a SQL database does for you with create index, and it is the price of an embedded key value store: you write the index maintenance, and you test it.

Encoding values and changing them later

Values are bytes, so you choose a format. JSON is easy to inspect and costs space and time; Protocol Buffers or gob are smaller and faster; a hand written binary layout is smallest and hardest to evolve. Whatever you pick, put a version byte at the start of each value or a version field inside it. Data written today will be read by code from next year, and a database file has no migration tool. When the format changes, readers accept both versions and a background pass rewrites old records, or a one time migration runs at start-up inside a single Update and records the new schema version in a meta bucket. Never assume that every record has the newest shape.

Testing code that uses bbolt

bbolt makes tests easy, since a database is a file in a temporary directory. Open a fresh one per test with t.TempDir(), close it with t.Cleanup, and run the real code against it. Test the three behaviours that decide correctness: that a failed Update leaves no partial state, that an index agrees with the data after a series of changes, and that your encoding round trips values from every version you still support. Add a test that opens the file a second time with a short Timeout to confirm your program fails cleanly and does not hang when another instance holds the lock. For concurrency, run the writers under the race detector with several goroutines calling your wrapper, because the library is safe and your own caches around it may not be.

When bbolt is the wrong tool

Write heavy workloads suffer from the single writer and a fsync per commit, even with Batch. Data larger than memory works, since the file is mapped and paged by the operating system, and performance falls off as the working set outgrows RAM. Anything needing queries beyond key order wants SQL, and several processes need a server. For those cases, see SQLite in Go, or Badger for a write heavy LSM design. The project is on GitHub and documented on pkg.go.dev, including the notes on read transactions and remapping quoted above.