fix: Fixes to server shutdown stream
This commit is contained in:
@@ -228,10 +228,19 @@ func (s *vantageServer) CommandStream(stream pb.Vantage_CommandStreamServer) err
|
||||
}
|
||||
}
|
||||
|
||||
func StartGRPC(port int) error {
|
||||
// StartGRPC serves the agent API until stop is called.
|
||||
//
|
||||
// It returns a stop function rather than serving forever because an abrupt exit
|
||||
// is not a neutral act here: every CommandStream handler holds an agent's
|
||||
// presence claim, released by a deferred call that a killed process never runs.
|
||||
// The claim then outlives its owner for the remainder of its 30s TTL, during
|
||||
// which dispatch believes the agent is reachable, publishes to a channel with
|
||||
// no subscriber, and fails as "agent offline" — a pod that has already exited
|
||||
// still answering for an agent it can no longer reach.
|
||||
func StartGRPC(port int) (stop func(), err error) {
|
||||
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to listen: %w", err)
|
||||
return nil, fmt.Errorf("failed to listen: %w", err)
|
||||
}
|
||||
|
||||
s := grpc.NewServer(
|
||||
@@ -248,6 +257,39 @@ func StartGRPC(port int) error {
|
||||
)
|
||||
pb.RegisterVantageServer(s, &vantageServer{})
|
||||
|
||||
log.Printf("gRPC server listening on :%d", port)
|
||||
return s.Serve(lis)
|
||||
go func() {
|
||||
log.Printf("gRPC server listening on :%d", port)
|
||||
if err := s.Serve(lis); err != nil {
|
||||
log.Fatalf("gRPC server error: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// GracefulStop sends GOAWAY and waits for the handlers to return, which is
|
||||
// what runs those deferred releases and, on the agent's side, ends the
|
||||
// stream with a clean error it reconnects from immediately rather than
|
||||
// waiting out a TCP timeout.
|
||||
//
|
||||
// It is bounded: an idle CommandStream returns as soon as its context is
|
||||
// cancelled, but a console relay mid-transfer would otherwise hold the
|
||||
// process past the pod's grace period and earn a SIGKILL — which is the
|
||||
// abrupt exit this exists to avoid.
|
||||
return func() {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
s.GracefulStop()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
log.Println("gRPC server stopped gracefully")
|
||||
case <-time.After(grpcStopTimeout):
|
||||
log.Printf("gRPC server did not stop within %s, forcing", grpcStopTimeout)
|
||||
s.Stop()
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
// How long GracefulStop is given before outstanding streams are cut. Comfortably
|
||||
// inside the chart's termination grace period, so the forced stop below still
|
||||
// leaves time for the HTTP server to drain.
|
||||
const grpcStopTimeout = 10 * time.Second
|
||||
|
||||
Reference in New Issue
Block a user