Add more error logging to Azure storage queue pushes

clearlydefined · Jan 16, 2025 · 3468d99 · 3468d99
1 parent c9cfd27
commit 3468d99
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 6 deletions.
diff --git a/ghcrawler/providers/queuing/storageQueue.js b/ghcrawler/providers/queuing/storageQueue.js
@@ -39,12 +39,19 @@ class StorageQueue {
         qlimit(this.options.parallelPush || 1)(async request => {
           try {
             const body = JSON.stringify(request)
+            this.logger.info(`Attempting to send message from storageQueue:`, {
+              bodyType: typeof body
+            })
             const queueMessageResult = await this.queueClient.sendMessage(body)
             this._log('Queued', request)
             return this._buildMessageReceipt(queueMessageResult, request)
           } catch (error) {
-            this.logger.error(`Failed to push message: ${error.message}`)
-            this.logger.error(`Request: ${JSON.stringify(request, null, 2)}`)
+            this.logger.error('Failed to push message from storageQueue:', {
+              error: error.stack,
+              request: request,
+              type: typeof request,
+              keys: Object.keys(request)
+            })
             throw error
           }
         })

diff --git a/ghcrawler/providers/storage/azureBlobFactory.js b/ghcrawler/providers/storage/azureBlobFactory.js
@@ -37,10 +37,10 @@ module.exports = options => {
     if (spnAuth) {
       const authParsed = JSON.parse(spnAuth)
       credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret)
-      options.logger.info('using service principal credentials')
+      options.logger.info('using service principal credentials in azureBlobFactory')
     } else {
       credential = new DefaultAzureCredential()
-      options.logger.info('using default credentials')
+      options.logger.info('using default credentials in azureBlobFactory')
     }
     blobServiceClient = new BlobServiceClient(`https://${account}.blob.core.windows.net`, credential, pipelineOptions)
   }

diff --git a/providers/store/azureQueueStore.js b/providers/store/azureQueueStore.js
@@ -23,13 +23,16 @@ class AzureStorageQueue {
     }
     if (connectionString) {
       this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions)
+      console.info('using connection string in azureQueueStore')
     } else {
       let credential
       if (spnAuth) {
         const authParsed = JSON.parse(spnAuth)
         credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret)
+        console.info('using SPN in azureQueueStore')
       } else {
         credential = new DefaultAzureCredential()
+        console.info('using default auth in azureQueueStore')
       }
       this.client = new QueueServiceClient(`https://${account}.queue.core.windows.net`, credential, pipelineOptions)
     }
@@ -41,8 +44,21 @@ class AzureStorageQueue {
   }
 
   async upsert(document) {
-    const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64')
-    return await this.queueService.sendMessage(message)
+    try {
+      this.logger.verbose(`Attempting to send message from azureQueueStore:`, {
+        bodyType: typeof body
+      })
+      const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64')
+      return await this.queueService.sendMessage(message)
+    } catch (error) {
+      this.logger.error('Failed to push message from azureQueueStore:', {
+        error: error.stack,
+        request: document,
+        type: typeof document,
+        keys: Object.keys(document)
+      })
+      throw error
+    }
   }
 
   get() {