Return NotFound for unknown pod names (#11540)

Fixes #11065

When an inbound proxy receives a request with a canonical name of the form `hostname.service.namespace.svc.cluster.domain`, we assume that `hostname` is the hostname of the pod as described in https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-hostname-and-subdomain-fields.  However, pods are also addressable with `pod-ip.service.namespace.svc.cluster.domain`.  When the destination controller gets a profile request of this form, we attempt to find a pod with hostname of `pod-ip` and return an error with gRPC status `Unknown` since this will not exist.

It is expected that this profile lookup will fail since we cannot have service profiles for individual pods.  However, returning a gRPC status `Unknown` for these requests brings the reported success rate of the destination controller down.  Instead we should return these as gRPC status `NotFound` so that these responses don't get reported as server errors.

Signed-off-by: Alex Leong <[email protected]>

controller/api/destination/watcher/pod_watcher.go

@@ -277,7 +277,7 @@ func (pw *PodWatcher) getOrNewPodPublisher(service *ServiceID, hostname, ip stri
 	if hostname != "" {
 		pod, err = pw.getEndpointByHostname(hostname, service)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get pod for hostname %s: %w", hostname, err)
+			return nil, err
 		}
 		ip = pod.Status.PodIP
 	} else {
@@ -399,7 +399,7 @@ func (pw *PodWatcher) getEndpointByHostname(hostname string, svcID *ServiceID) (
 		}
 	}
 
-	return nil, fmt.Errorf("no pod found in Endpoints %s/%s for hostname %s", svcID.Namespace, svcID.Name, hostname)
+	return nil, status.Errorf(codes.NotFound, "no pod found in Endpoints %s/%s for hostname %s", svcID.Namespace, svcID.Name, hostname)
 }
 
 func (pp *podPublisher) subscribe(listener PodUpdateListener) {