gRPC Performance Tuning in Go and Common Bottlenecks

Performance advice for gRPC is full of folklore: use more connections, turn on compression, tune the window sizes, avoid large messages. Some of it is true in some conditions. The way to know which conditions are yours is to measure. This article measures a few of the common claims in grpc-go with a benchmark, reports what happened (including a claim that did not hold), and then lists the bottlenecks that, in practice, are far more likely to be the problem than the transport.

All numbers come from a loopback benchmark on an Apple M1 Max running Go 1.25.6 and grpc-go v1.75.0. Loopback has no latency, no bandwidth limit and no TLS. That makes it good for comparing the CPU cost of choices and useless for predicting what a user in another country sees. For benchmarking technique in general, see benchmarking a handler, and for the related comparison with Connect, Connect RPC versus gRPC.

The setup

The benchmark registers a unary echo service by hand, with protobuf's BytesValue as the message so that no code generation is needed, and calls it from many goroutines in parallel (eight per CPU). It varies one thing at a time.

package main

func main() {}

package main

import (
	"bytes"
	"context"
	"crypto/rand"
	"fmt"
	"net"
	"sync/atomic"
	"testing"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/encoding/gzip"
	"google.golang.org/protobuf/types/known/wrapperspb"
)

const method = "/perf.v1.Echo/Echo"

var echoDesc = grpc.ServiceDesc{
	ServiceName: "perf.v1.Echo",
	HandlerType: (*any)(nil),
	Methods: []grpc.MethodDesc{{
		MethodName: "Echo",
		Handler: func(_ any, ctx context.Context, dec func(any) error, _ grpc.UnaryServerInterceptor) (any, error) {
			in := new(wrapperspb.BytesValue)
			if err := dec(in); err != nil {
				return nil, err
			}
			return in, nil
		},
	}},
}

func server(b *testing.B) string {
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		b.Fatal(err)
	}
	srv := grpc.NewServer()
	srv.RegisterService(&echoDesc, struct{}{})
	go srv.Serve(lis)
	b.Cleanup(srv.Stop)
	return lis.Addr().String()
}

func dial(b *testing.B, addr string, opts ...grpc.DialOption) *grpc.ClientConn {
	conn, err := grpc.NewClient(addr, append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, opts...)...)
	if err != nil {
		b.Fatal(err)
	}
	b.Cleanup(func() { conn.Close() })
	return conn
}

// run drives the echo with the given connections, round-robining goroutines over them.
func run(b *testing.B, conns []*grpc.ClientConn, msg *wrapperspb.BytesValue, opts ...grpc.CallOption) {
	var next atomic.Uint64
	b.SetBytes(int64(len(msg.Value)))
	b.ReportAllocs()
	b.ResetTimer()
	b.SetParallelism(8) // 8 goroutines per CPU, so there is real concurrency to contend on a connection
	b.RunParallel(func(pb *testing.PB) {
		conn := conns[int(next.Add(1))%len(conns)]
		out := new(wrapperspb.BytesValue)
		for pb.Next() {
			if err := conn.Invoke(context.Background(), method, msg, out, opts...); err != nil {
				b.Fatal(err)
			}
		}
	})
}

func BenchmarkMessageSize(b *testing.B) {
	for _, size := range []int{64, 4 << 10, 256 << 10} {
		b.Run(fmt.Sprintf("%dB", size), func(b *testing.B) {
			conn := dial(b, server(b))
			run(b, []*grpc.ClientConn{conn}, wrapperspb.Bytes(make([]byte, size)))
		})
	}
}

func BenchmarkConnections(b *testing.B) {
	for _, n := range []int{1, 4} {
		b.Run(fmt.Sprintf("%dconn", n), func(b *testing.B) {
			addr := server(b)
			var conns []*grpc.ClientConn
			for i := 0; i < n; i++ {
				conns = append(conns, dial(b, addr))
			}
			run(b, conns, wrapperspb.Bytes(make([]byte, 256)))
		})
	}
}

func BenchmarkCompression(b *testing.B) {
	random := make([]byte, 16<<10)
	rand.Read(random)
	for _, tc := range []struct {
		name string
		data []byte
	}{{"repetitive16KB", bytes.Repeat([]byte("abcdefgh"), 2<<10)}, {"random16KB", random}} {
		for _, compress := range []bool{false, true} {
			name := tc.name + "/plain"
			var opts []grpc.CallOption
			if compress {
				name = tc.name + "/gzip"
				opts = []grpc.CallOption{grpc.UseCompressor(gzip.Name)}
			}
			b.Run(name, func(b *testing.B) {
				conn := dial(b, server(b))
				run(b, []*grpc.ClientConn{conn}, wrapperspb.Bytes(tc.data), opts...)
			})
		}
	}
}

go test -run='^$' -bench=. -benchtime=1s -count=3 . | grep -E '^Benchmark' | sed -E 's/-10 /    /' | awk '{printf "%-48s %10s ns/op %6s allocs/op\n", $1, $3, $9}'

BenchmarkMessageSize/64B                              14234 ns/op    155 allocs/op
BenchmarkMessageSize/64B                              14876 ns/op    155 allocs/op
BenchmarkMessageSize/64B                              14376 ns/op    155 allocs/op
BenchmarkMessageSize/4096B                            15838 ns/op    146 allocs/op
BenchmarkMessageSize/4096B                            22635 ns/op    146 allocs/op
BenchmarkMessageSize/4096B                            14928 ns/op    146 allocs/op
BenchmarkMessageSize/262144B                         161530 ns/op    166 allocs/op
BenchmarkMessageSize/262144B                         162872 ns/op    169 allocs/op
BenchmarkMessageSize/262144B                         155816 ns/op    168 allocs/op
BenchmarkConnections/1conn                            14890 ns/op    155 allocs/op
BenchmarkConnections/1conn                            14390 ns/op    155 allocs/op
BenchmarkConnections/1conn                            14981 ns/op    155 allocs/op
BenchmarkConnections/4conn                            12970 ns/op    156 allocs/op
BenchmarkConnections/4conn                            13310 ns/op    156 allocs/op
BenchmarkConnections/4conn                            12852 ns/op    156 allocs/op
BenchmarkCompression/repetitive16KB/plain             18482 ns/op    154 allocs/op
BenchmarkCompression/repetitive16KB/plain             20396 ns/op    155 allocs/op
BenchmarkCompression/repetitive16KB/plain             18359 ns/op    154 allocs/op
BenchmarkCompression/repetitive16KB/gzip              27776 ns/op    192 allocs/op
BenchmarkCompression/repetitive16KB/gzip              27903 ns/op    192 allocs/op
BenchmarkCompression/repetitive16KB/gzip              26705 ns/op    192 allocs/op
BenchmarkCompression/random16KB/plain                 18636 ns/op    155 allocs/op
BenchmarkCompression/random16KB/plain                 18795 ns/op    155 allocs/op
BenchmarkCompression/random16KB/plain                 19154 ns/op    155 allocs/op
BenchmarkCompression/random16KB/gzip                  65536 ns/op    205 allocs/op
BenchmarkCompression/random16KB/gzip                  74945 ns/op    205 allocs/op
BenchmarkCompression/random16KB/gzip                  72135 ns/op    205 allocs/op

Message size

From 64 bytes to 4 KiB the time barely moves: about a microsecond or two for 64 times as much data. Small messages are dominated by fixed per-call cost (the stream, headers, the framework, 150-odd allocations), not by bytes. At 256 KiB the cost is about 161 microseconds, 11 times the small call, which is a payload rate of about 1.6 GB/s one way (the echo moves the data both ways). Large messages are where copying and allocation show up: 546 KB allocated for a 256 KB call, roughly two copies each way.

What to take from it. Do not shrink 2 KB messages to 1 KB to save time. Do worry about messages in the hundreds of kilobytes, where memory, GC pressure and the 4 MiB default receive limit all arrive together; stream them in chunks or move the bulk data out of band (message size limits, presigned uploads).

One connection or several

The folklore says a single HTTP/2 connection limits throughput, because streams share one TCP connection, one flow-control window and, in some servers, a cap on concurrent streams, and that you should open a pool of connections. The benchmark runs 80 goroutines through one ClientConn and through four. Numbers in the tables are the median of three runs of one second each, taken from the recorded output; single runs varied by 10 to 40 percent.

A small difference, in favour of more connections, that was larger in this run (about 13 percent) than in an earlier run (under 5 percent). grpc-go's server allows an effectively unlimited number of concurrent streams by default, and on a loopback with no bandwidth limit one connection kept up with 80 goroutines. Do not add a connection pool because someone said so; the gain here is small. It becomes relevant when (a) the server or a proxy caps concurrent streams per connection (Envoy's default is 100), (b) a single connection's TCP window is the limit on a high-latency link, or (c) one connection's I/O goroutine becomes a CPU bottleneck at very high request rates. Test in the real topology before adding the complexity, and note that more connections also interact with load balancing, since each can land on a different backend (load balancing).

Compression

gRPC supports per-call compression; the benchmark enables gzip with grpc.UseCompressor, on 16 KiB messages that are either very repetitive or random.

Gzip made both slower: about 50 percent more for compressible data and nearly four times as much for incompressible data, because the CPU spent on compression is wasted when nothing shrinks. On loopback, where bytes are free, there is no benefit to set against the cost.

That does not mean never compress. Compression pays when the network is the bottleneck: a slow or metered link, large text-heavy messages, WAN traffic between regions. A rule that works: do not enable it by default on internal, high-bandwidth links; enable it for specific calls whose messages are large and compressible, measured end to end; and never compress small messages, where the framing costs more than the saving. Binary protobuf with numbers and short strings often compresses poorly.

What matters more than the transport

A call costs about 14 to 16 microseconds of gRPC machinery on this machine. A typical handler costs hundreds of microseconds to milliseconds. Before touching window sizes or buffers, look here.

The handler's own work. Database queries dominate: N+1 patterns, missing indexes, unbounded result sets (database patterns help find them). Profile the server with pprof and look at where CPU and wall time go; a gRPC call is just a goroutine.

Interceptors. Every interceptor runs for every call. A logging interceptor that formats JSON for every request, an auth interceptor that verifies a JWT signature without caching, or a metrics interceptor that takes a global lock can each cost more than the rest of the call. Benchmark the full chain, not the bare handler (interceptors).

Allocation and GC. About 155 allocations per tiny call, and many more once you convert messages to domain types and back. Under high rates, GC work grows. Reuse buffers where it is safe, avoid copying large byte slices into new messages, and watch the allocation rate in your profiles. GOMEMLIMIT and GC tuning come after you reduce the allocations themselves.

Serialisation of your own data. Converting between protobuf messages and internal structs, building large repeated fields element by element and converting to JSON for logging all show up in profiles of chatty services.

Fan-out and serial calls. A handler that makes five downstream calls in sequence costs five round trips. Make independent calls concurrently with an errgroup (errgroup) and pass the deadline along (deadlines).

Connection churn. Creating a ClientConn per request pays a TCP and TLS handshake every time. Keep one per target for the life of the process.

Retries and timeouts. A missing deadline lets slow calls pile up, and a retry policy without throttling multiplies load during trouble (retries). These show up as latency in the tail, not as CPU in the profile.

Server resources. GOMAXPROCS wrong for the container's CPU limit means throttling that looks like random latency spikes (GOMAXPROCS in containers).

The settings that exist, and when to touch them

grpc-go exposes tuning options: WithInitialWindowSize and WithInitialConnWindowSize (flow-control windows, which matter for high bandwidth-delay links), WithReadBufferSize and WithWriteBufferSize (per-connection buffers), MaxConcurrentStreams on the server, NumStreamWorkers (a worker pool for streams) and SharedWriteBuffer. Defaults are reasonable and grpc-go adapts window sizes dynamically using a bandwidth estimate. Reach for them when you have a measured problem, such as a cross-region bulk transfer limited by the window, and verify the change with the same benchmark that showed the problem. Changing buffer sizes on a hunch mostly changes memory use.

How to run your own measurements

Build the benchmark around your real messages and your real interceptor chain, on hardware like production, with a network that has realistic latency. Use -benchmem and -count large enough to see the noise, and compare runs with benchstat (benchstat). Complement it with a load test that holds a fixed arrival rate and looks at tail latency, since the average hides what users feel (load testing). The grpc-go project ships its own benchmark tools and documents performance practices in the gRPC performance best practices guide, and the Go team's diagnostics documentation covers the profilers used for the server side.