diff --git a/changelog/fragments/1738867024-Improve-message-when-audit-uninstall-returns-401.yaml b/changelog/fragments/1738867024-Improve-message-when-audit-uninstall-returns-401.yaml
new file mode 100644
index 00000000000..f4cbc7864bf
--- /dev/null
+++ b/changelog/fragments/1738867024-Improve-message-when-audit-uninstall-returns-401.yaml
@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: Improve message when audit/uninstall returns 401
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/6711
diff --git a/internal/pkg/agent/install/uninstall.go b/internal/pkg/agent/install/uninstall.go
index 3202abd56d7..7b2ec2c5fc3 100644
--- a/internal/pkg/agent/install/uninstall.go
+++ b/internal/pkg/agent/install/uninstall.go
@@ -200,6 +200,9 @@ func notifyFleetAuditUninstall(ctx context.Context, log *logp.Logger, pt *progre
 			// Do not retry if it was a context error, or an error with the request.
 			if errors.Is(err, context.Canceled) {
 				return ctx.Err()
+			} else if errors.Is(err, fleetclient.ErrInvalidAPIKey) {
+				pt.Describe("API key is invalid (normal if already unenrolled), notification dropped.")
+				return nil
 			} else if errors.As(err, &reqErr) {
 				pt.Describe(fmt.Sprintf("notify Fleet: encountered unretryable error: %v", err))
 				return err
diff --git a/internal/pkg/agent/install/uninstall_test.go b/internal/pkg/agent/install/uninstall_test.go
index 19b5afdee95..e074f6bbc31 100644
--- a/internal/pkg/agent/install/uninstall_test.go
+++ b/internal/pkg/agent/install/uninstall_test.go
@@ -26,7 +26,6 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/vault"
-	"github.com/elastic/elastic-agent/internal/pkg/fleetapi/client"
 	"github.com/elastic/elastic-agent/internal/pkg/remote"
 )
 
@@ -164,7 +163,7 @@ func TestNotifyFleetAuditUnenroll(t *testing.T) {
 				w.WriteHeader(http.StatusUnauthorized)
 			}))
 		},
-		err: client.ErrInvalidAPIKey,
+		err: nil,
 	}, {
 		name: "returns 409",
 		getServer: func() *httptest.Server {