Impeove the error handling to catch and notify of errors (#26)

* Impeove the error handling to catch and notify of errors * missing fixes
nabsul · Feb 27, 2022 · b845aaa · b845aaa
1 parent f4c17c2
commit b845aaa
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 35 deletions.
diff --git a/Services/EmailClient.cs b/Services/EmailClient.cs
@@ -1,5 +1,6 @@
 using KCert.Models;
 using Microsoft.Extensions.Logging;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
@@ -34,6 +35,13 @@ public async Task NotifyRenewalResultAsync(string secretNamespace, string secret
         await SendAsync(RenewalSubject(secretNamespace, secretName, ex), RenewalMessage(secretNamespace, secretName, ex));
     }
 
+    public async Task NotifyFailureAsync(string message, Exception ex)
+    {
+        var subject = "KCert encountered an unexpected error";
+        var body = $"{message}\n\n{ex.Message}\n\n{ex.StackTrace}";
+        await SendAsync(subject, body);
+    }
+
     private async Task SendAsync(string subject, string text)
     {
         if (!CanSendEmails())

diff --git a/Services/IngressMonitorService.cs b/Services/IngressMonitorService.cs
@@ -17,70 +17,116 @@ public class IngressMonitorService : IHostedService
     private readonly KCertClient _kcert;
     private readonly K8sClient _k8s;
     private readonly CertClient _cert;
+    private readonly EmailClient _email;
+    private readonly KCertConfig _cfg;
 
-    public IngressMonitorService(ILogger<IngressMonitorService> log, KCertClient kcert, K8sClient k8s, CertClient cert)
+    public IngressMonitorService(ILogger<IngressMonitorService> log, KCertClient kcert, K8sClient k8s, CertClient cert, EmailClient email, KCertConfig cfg)
     {
         _log = log;
         _kcert = kcert;
         _k8s = k8s;
         _cert = cert;
+        _email = email;
+        _cfg = cfg;
     }
 
     public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _ = WatchIngressesAsync(cancellationToken);
+        if (_cfg.WatchIngresses)
+        {
+            _ = WatchIngressesAsync(cancellationToken);
+        }
+
         return Task.CompletedTask;
     }
 
     private async Task WatchIngressesAsync(CancellationToken tok)
     {
-        try
-        {
-            _log.LogInformation("Watching for ingress changes");
-            await _k8s.WatchIngressesAsync(HandleIngressEventAsync, tok);
-        }
-        catch (Exception ex)
+        int numTries = 5;
+        while (numTries-- > 0)
         {
-            _log.LogError(ex, "Ingress watcher failed");
+            try
+            {
+                _log.LogInformation("Watching for ingress changes");
+                await _k8s.WatchIngressesAsync(HandleIngressEventAsync, tok);
+            }
+            catch (TaskCanceledException ex)
+            {
+                _log.LogError(ex, "Ingress watch service cancelled.");
+            }
+            catch (Exception ex)
+            {
+                _log.LogError(ex, "Ingress watcher failed");
+                try
+                {
+                    await _email.NotifyFailureAsync("Ingress watching failed unexpectedly", ex);
+                }
+                catch (Exception ex2)
+                {
+                    _log.LogError(ex2, "Failed to send error notification");
+                }
+            }
+
+            _log.LogError("Watch Ingresses failed. Sleeping for 10 seconds then trying {n} more times.", numTries);
+            await Task.Delay(TimeSpan.FromSeconds(10), tok);
         }
     }
 
     private async Task HandleIngressEventAsync(WatchEventType type, V1Ingress ingress, CancellationToken tok)
     {
-        _log.LogInformation("event [{type}] for {ns}-{name}", type, ingress.Namespace(), ingress.Name());
-        if (type != WatchEventType.Added && type != WatchEventType.Modified)
+        try
         {
-            return;
-        }
+            _log.LogInformation("event [{type}] for {ns}-{name}", type, ingress.Namespace(), ingress.Name());
+            if (type != WatchEventType.Added && type != WatchEventType.Modified)
+            {
+                return;
+            }
 
-        // fetch all ingresses to figure out which certs need have which hosts
-        var nsLookup = new Dictionary<(string, string), HashSet<string>>();
-        await foreach (var ing in _k8s.GetAllIngressesAsync())
-        {
-            _log.LogInformation("Processing ingress {ns}:{n}", ing.Namespace(), ing.Name());
-            foreach (var tls in ing?.Spec?.Tls ?? new List<V1IngressTLS>())
+            // fetch all ingresses to figure out which certs need have which hosts
+            var nsLookup = new Dictionary<(string, string), HashSet<string>>();
+            await foreach (var ing in _k8s.GetAllIngressesAsync())
             {
-                _log.LogInformation("Processing secret {s}", tls.SecretName);
-                var key = (ing.Namespace(), tls.SecretName);
-                if (!nsLookup.TryGetValue(key, out var hosts))
+                _log.LogInformation("Processing ingress {ns}:{n}", ing.Namespace(), ing.Name());
+                foreach (var tls in ing?.Spec?.Tls ?? new List<V1IngressTLS>())
                 {
-                    hosts = new HashSet<string>();
-                    nsLookup.Add(key, hosts);
-                }
+                    _log.LogInformation("Processing secret {s}", tls.SecretName);
+                    var key = (ing.Namespace(), tls.SecretName);
+                    if (!nsLookup.TryGetValue(key, out var hosts))
+                    {
+                        hosts = new HashSet<string>();
+                        nsLookup.Add(key, hosts);
+                    }
 
-                foreach (var h in tls.Hosts)
-                {
-                    hosts.Add(h);
+                    foreach (var h in tls.Hosts)
+                    {
+                        hosts.Add(h);
+                    }
                 }
             }
-        }
 
-        foreach (var ((ns, name), hosts) in nsLookup)
+            foreach (var ((ns, name), hosts) in nsLookup)
+            {
+                _log.LogInformation("Handling cert {ns} - {name} hosts: {h}", ns, name, string.Join(",", hosts));
+                await TryUpdateSecretAsync(ns, name, hosts, tok);
+            }
+        }
+        catch (TaskCanceledException ex)
+        {
+            _log.LogError(ex, "Ingress watch service cancelled.");
+        }
+        catch (Exception ex)
         {
-            _log.LogInformation("Handling cert {ns} - {name} hosts: {h}", ns, name, string.Join(",", hosts));
-            await TryUpdateSecretAsync(ns, name, hosts, tok);
+            _log.LogError(ex, "Ingress event handler failed unexpectedly");
+            try
+            {
+                await _email.NotifyFailureAsync("Ingress watching failed unexpectedly", ex);
+            }
+            catch (Exception ex2)
+            {
+                _log.LogError(ex2, "Failed to send error notification");
+            }
         }
     }
 

diff --git a/Services/KCertConfig.cs b/Services/KCertConfig.cs
@@ -15,6 +15,7 @@ public KCertConfig(IConfiguration cfg)
         _cfg = cfg;
     }
 
+    public bool WatchIngresses => GetBool("KCert:WatchIngresses");
     public string K8sConfigFile => _cfg["Config"];
     public bool AcceptAllChallenges => GetBool("KCert:AcceptAllChallenges");
     public string KCertNamespace => GetString("KCert:Namespace");

diff --git a/Services/RenewalService.cs b/Services/RenewalService.cs
@@ -34,7 +34,11 @@ public RenewalService(ILogger<RenewalService> log, KCertClient kcert, KCertConfi
 
     public Task StartAsync(CancellationToken cancellationToken)
     {
-        _ = StartInnerAsync(cancellationToken);
+        if (_cfg.EnableAutoRenew)
+        {
+            _ = StartInnerAsync(cancellationToken);
+        }
+
         return Task.CompletedTask;
     }
 
@@ -43,6 +47,7 @@ public async Task StartInnerAsync(CancellationToken cancellationToken)
         int numFailures = 0;
         while (numFailures < MaxServiceFailures && !cancellationToken.IsCancellationRequested)
         {
+            Exception error = null;
             _log.LogInformation("Starting up renewal service.");
             try
             {
@@ -51,12 +56,24 @@ public async Task StartInnerAsync(CancellationToken cancellationToken)
             catch (TaskCanceledException ex)
             {
                 _log.LogError(ex, "Renewal loop cancelled.");
-                break;
             }
             catch (Exception ex)
             {
                 numFailures++;
                 _log.LogError(ex, "Renewal Service encountered error {numFailures} of max {MaxServiceFailures}", numFailures, MaxServiceFailures);
+                error = ex;
+            }
+
+            if (error != null)
+            {
+                try
+                {
+                    await _email.NotifyFailureAsync("Certificate renewal failed unexpectedly", error);
+                }
+                catch (Exception ex)
+                {
+                    _log.LogError(ex, "Failed to send cert renewal failure email");
+                }
             }
         }
     }

diff --git a/appsettings.json b/appsettings.json
@@ -5,7 +5,8 @@
     "SecretName": "kcert",
     "ServiceName": "kcert",
     "ServicePort": 80,
-    "AcceptAllChallenges": false
+    "AcceptAllChallenges": false,
+    "WatchIngresses":  true
   },
   "Acme": {
     "DirUrl": "https://acme-staging-v02.api.letsencrypt.org/directory",