gRPC Mistakes in Go Services That Cause Real Outages

gRPC is reliable. The incidents come from how it is used, and they repeat. Looking back over what goes wrong in Go services that use it, the same dozen mistakes account for most of the pages: a missing deadline, a connection created in the wrong place, a stream nobody closed, an interceptor that guards some methods and not others. None is exotic, and each is invisible in tests and obvious in production.

This is a list of them, ordered roughly by how often they cause trouble. Three are reproduced in one program so you can see the cost, and the rest are described with the fix and a pointer to the article that goes deeper. It assumes the basics from the gRPC tutorial. The measurements ran on Go 1.25.6.

package main

import (
	"context"
	"fmt"
	"log"
	"net"
	"runtime"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/types/known/wrapperspb"
)

var (
	activeStreams atomic.Int64
	hang          = make(chan struct{})
)

// A hand-registered service so the lab needs no code generation.
var desc = grpc.ServiceDesc{
	ServiceName: "lab.v1.Lab",
	HandlerType: (*any)(nil),
	Methods: []grpc.MethodDesc{
		{MethodName: "Fast", Handler: func(_ any, _ context.Context, dec func(any) error, _ grpc.UnaryServerInterceptor) (any, error) {
			return wrapperspb.String("ok"), nil
		}},
		{MethodName: "Hang", Handler: func(_ any, ctx context.Context, dec func(any) error, _ grpc.UnaryServerInterceptor) (any, error) {
			select {
			case <-hang:
			case <-ctx.Done():
			}
			return wrapperspb.String("late"), nil
		}},
	},
	Streams: []grpc.StreamDesc{{
		StreamName: "Ticks", ServerStreams: true,
		Handler: func(_ any, s grpc.ServerStream) error {
			activeStreams.Add(1)
			defer activeStreams.Add(-1)
			s.RecvMsg(new(wrapperspb.StringValue))
			t := time.NewTicker(10 * time.Millisecond)
			defer t.Stop()
			for {
				select {
				case <-t.C:
					if err := s.SendMsg(wrapperspb.String("tick")); err != nil {
						return err
					}
				case <-s.Context().Done():
					return s.Context().Err()
				}
			}
		},
	}},
}

func dial(addr string) *grpc.ClientConn {
	conn, err := grpc.NewClient(addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
	if err != nil {
		log.Fatal(err)
	}
	return conn
}

func main() {
	lis, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		log.Fatal(err)
	}
	srv := grpc.NewServer()
	srv.RegisterService(&desc, struct{}{})
	go srv.Serve(lis)
	defer srv.Stop()
	addr := lis.Addr().String()
	ctx := context.Background()
	call := func(c *grpc.ClientConn, m string) error {
		return c.Invoke(ctx, "/lab.v1.Lab/"+m, wrapperspb.String(""), new(wrapperspb.StringValue))
	}

	// Mistake 1: a new connection for every call.
	const n = 200
	start := time.Now()
	for i := 0; i < n; i++ {
		c := dial(addr)
		call(c, "Fast")
		c.Close()
	}
	perCall := time.Since(start)
	shared := dial(addr)
	call(shared, "Fast") // connect once
	start = time.Now()
	for i := 0; i < n; i++ {
		call(shared, "Fast")
	}
	sharedTime := time.Since(start)
	fmt.Printf("1. %d calls, a new connection each: %v; one shared connection: %v (about %.0fx faster)\n",
		n, perCall.Round(time.Millisecond), sharedTime.Round(time.Millisecond), float64(perCall)/float64(sharedTime))

	// Mistake 2: no deadline. The call waits for as long as the server does.
	done := make(chan error, 1)
	go func() { done <- call(shared, "Hang") }()
	select {
	case <-done:
		fmt.Println("2. no deadline: returned")
	case <-time.After(300 * time.Millisecond):
		fmt.Println("2. no deadline: still blocked after 300ms; nothing will ever end it unless the server answers")
	}
	short, cancel := context.WithTimeout(ctx, 100*time.Millisecond)
	err = shared.Invoke(short, "/lab.v1.Lab/Hang", wrapperspb.String(""), new(wrapperspb.StringValue))
	cancel()
	fmt.Println("   with a 100ms deadline:", status.Code(err) == codes.DeadlineExceeded, status.Code(err))
	close(hang) // let the first call finish
	<-done

	// Mistake 3: abandoning streams without cancelling them.
	before := runtime.NumGoroutine()
	var leaked []grpc.ClientStream
	for i := 0; i < 50; i++ {
		s, _ := shared.NewStream(ctx, &grpc.StreamDesc{ServerStreams: true}, "/lab.v1.Lab/Ticks")
		s.SendMsg(wrapperspb.String("go"))
		s.CloseSend()
		s.RecvMsg(new(wrapperspb.StringValue)) // read one message, then forget the stream
		leaked = append(leaked, s)
	}
	time.Sleep(100 * time.Millisecond)
	fmt.Printf("3. 50 streams read once and dropped: %d server handlers still running, goroutines +%d\n", activeStreams.Load(), runtime.NumGoroutine()-before)
	var cancels []context.CancelFunc
	for i := 0; i < 50; i++ {
		cctx, cancel := context.WithCancel(ctx)
		cancels = append(cancels, cancel)
		s, _ := shared.NewStream(cctx, &grpc.StreamDesc{ServerStreams: true}, "/lab.v1.Lab/Ticks")
		s.SendMsg(wrapperspb.String("go"))
		s.CloseSend()
		s.RecvMsg(new(wrapperspb.StringValue))
	}
	time.Sleep(100 * time.Millisecond)
	fmt.Println("   50 more streams with cancellable contexts, before cancelling: handlers running =", activeStreams.Load())
	for _, c := range cancels {
		c()
	}
	time.Sleep(200 * time.Millisecond)
	fmt.Println("   after cancelling those 50: handlers running =", activeStreams.Load(), "(the first 50 are still leaked)")
	_ = leaked
}

1. 200 calls, a new connection each: 69ms; one shared connection: 13ms (about 5x faster)
2. no deadline: still blocked after 300ms; nothing will ever end it unless the server answers
   with a 100ms deadline: true DeadlineExceeded
3. 50 streams read once and dropped: 50 server handlers still running, goroutines +100
   50 more streams with cancellable contexts, before cancelling: handlers running = 100
   after cancelling those 50: handlers running = 50 (the first 50 are still leaked)

Calls and connections

1. No deadline on a call. The most common cause of a hung service. A call without a deadline waits as long as the server does. The program's second block shows it: the call to a server that never answers was still blocked after 300 milliseconds, and nothing will end it. With a 100 ms deadline the same call returned DeadlineExceeded and freed the caller. Give every outgoing call a deadline, from the incoming one when you are a handler, and add a client interceptor that supplies a default when none is set (deadlines).

2. A new connection for every call. A grpc.ClientConn is built to be long-lived and shared: it holds the HTTP/2 connection, reconnects on failure and balances calls. Creating one per request pays a TCP handshake (and TLS, in production) every time. The first measurement: 200 calls with a fresh connection each took about five times as long as 200 calls over one shared connection, on loopback with no TLS; with real latency and TLS the gap is much larger. Create the connection once at start-up, pass it to your clients, close it at shutdown.

3. Abandoned streams. A stream is a live resource on the server: a goroutine, buffers and often a database cursor. The third block opened 50 server streams, read one message from each, and dropped them. The server still had 50 handlers running, and the process had 100 goroutines more than before (50 on each side). Dropping the variable does not end a stream. Only reading to the end, cancelling the context or closing the connection does. Tie every stream to a context with WithCancel or a timeout, and call cancel in a defer (streaming). The second set of 50 streams in the output, started with cancellable contexts, ended their handlers as soon as the contexts were cancelled.

4. Treating io.EOF as a failure, or ignoring other stream errors. Recv returning io.EOF is the normal end. Any other error is a failure. Logging every io.EOF as an error floods the logs; breaking out of a loop on any error and discarding it hides real failures. And when Send returns an error, the real status is on Recv (bidirectional streaming).

5. The WithBlock and Dial habit. Older code used grpc.Dial with WithBlock() to wait for the connection, which hangs start-up when a dependency is down. Use grpc.NewClient, which connects lazily, and let calls carry deadlines and WaitForReady when you want them to queue. Make start-up not depend on every dependency being up.

Errors and retries

6. Returning raw errors from handlers. A plain Go error becomes Unknown with its text sent to the client, including file names and SQL fragments. Map domain errors to status codes at the edge, with a generic Internal fallback (error codes).

7. Wrong codes. Internal for everything, or Unavailable for a bad request, breaks retry policies and dashboards. InvalidArgument for bad input, FailedPrecondition for wrong state, Unavailable only for transient server-side failure.

8. Retrying calls that are not safe to repeat. A retry policy on Unavailable for a non-idempotent Charge can charge twice when the first attempt succeeded and the response was lost. Retry only idempotent methods or ones keyed by an idempotency key, and enable retry throttling (retries).

9. Retries at every layer. Three layers each making three attempts turn one failure into 27 calls on the bottom service. Retry in one layer and budget it (timeouts and retries).

Servers

10. Authentication only on unary methods. A UnaryInterceptor does not run for streams. Services that added auth as a unary interceptor have left every streaming method open. Register the stream interceptor too, and test that every method rejects a call with no credentials (interceptors).

11. No panic recovery. A panic in a handler is not recovered by gRPC, and it takes the whole process down. Add a recovery interceptor that returns Internal and logs the stack. It cannot catch panics in goroutines your handler starts, which need their own recover.

12. No graceful shutdown. Stop() drops in-flight calls. Mark the health service NOT_SERVING, wait for balancers to notice, call GracefulStop with a time budget and fall back to Stop (graceful shutdown).

13. Ignoring the context in handlers. A handler that never looks at ctx keeps working after the client leaves, holding connections and memory. Pass ctx to every database call and outgoing RPC, and select on ctx.Done() in loops.

14. Mutating the request or sharing messages unsafely. Generated messages are not safe for concurrent mutation. A handler that stores the request pointer in a cache and modifies it later races with whatever serialises it. Copy with proto.Clone when you keep a message.

Operations

15. Keepalive that disagrees with the server. A client that pings every 30 seconds against a server whose policy allows one ping per 5 minutes gets too_many_pings GOAWAYs and random Unavailable errors (keepalive). Configure both sides together.

16. Balancing connections instead of calls. A ClusterIP Service or a TCP load balancer sends all of one client's calls to one backend. Use a headless service with round_robin, or an HTTP/2-aware proxy (load balancing).

17. Plaintext in production, and reflection left on. insecure.NewCredentials() is for tests. Use TLS and, between services, mutual TLS. Reflection hands out your API surface; restrict it (authentication, health and reflection).

18. Huge messages. A list or blob in one message hits the 4 MiB limit and costs memory on both sides. Stream chunks or move the bytes out of band (message size limits).

19. Logging payloads. Interceptors that log requests and responses send tokens and personal data to the logs. Log method, code and duration; redact the rest (logging).

20. No schema compatibility checks. One innocent rename or reused field number breaks clients that deploy later. Run lint and breaking checks in CI (Buf, compatibility rules).

Order of attack

If you inherit a gRPC service and can fix only a few things, fix them in this order: add deadlines everywhere (1), check that authentication covers streams (10), add panic recovery and graceful shutdown (11, 12), make sure retries are safe and not layered (8, 9), then share connections and cancel streams (2, 3). The first four prevent the most expensive incidents, a hang, an open door, a crash and a double charge. The rest are about efficiency and tidiness, and are worth fixing once the service stops falling over.

Making these less likely

Most of the list can be turned into tests and defaults. A shared package that builds the server (recovery, auth for unary and stream, logging, tracing, keepalive, limits) and another that builds clients (deadline default, retry policy, TLS, propagation) means a new service starts with the right choices and a reviewer has less to remember. A test that walks every method on the server and asserts it rejects unauthenticated calls finds the open stream. A CI job that runs buf breaking against main removes the compatibility class of incident. And a load test that stalls a client and kills a server in the middle of a stream finds the resource leaks and the shutdown problems before users do. The Go team's guidance on context and cancellation explains the pattern behind mistakes 1, 3 and 13, and the grpc-go repository's own documentation folder has short notes on several of these topics, including keepalive, retry and encoding.