diff --git a/ghcrawler/providers/queuing/storageQueue.js b/ghcrawler/providers/queuing/storageQueue.js index 982ff3f..72dec20 100644 --- a/ghcrawler/providers/queuing/storageQueue.js +++ b/ghcrawler/providers/queuing/storageQueue.js @@ -39,12 +39,19 @@ class StorageQueue { qlimit(this.options.parallelPush || 1)(async request => { try { const body = JSON.stringify(request) + this.logger.info(`Attempting to send message from storageQueue:`, { + bodyType: typeof body + }) const queueMessageResult = await this.queueClient.sendMessage(body) this._log('Queued', request) return this._buildMessageReceipt(queueMessageResult, request) } catch (error) { - this.logger.error(`Failed to push message: ${error.message}`) - this.logger.error(`Request: ${JSON.stringify(request, null, 2)}`) + this.logger.error('Failed to push message from storageQueue:', { + error: error.stack, + request: request, + type: typeof request, + keys: Object.keys(request) + }) throw error } }) diff --git a/ghcrawler/providers/storage/azureBlobFactory.js b/ghcrawler/providers/storage/azureBlobFactory.js index b08af8b..0885f38 100644 --- a/ghcrawler/providers/storage/azureBlobFactory.js +++ b/ghcrawler/providers/storage/azureBlobFactory.js @@ -37,10 +37,10 @@ module.exports = options => { if (spnAuth) { const authParsed = JSON.parse(spnAuth) credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret) - options.logger.info('using service principal credentials') + options.logger.info('using service principal credentials in azureBlobFactory') } else { credential = new DefaultAzureCredential() - options.logger.info('using default credentials') + options.logger.info('using default credentials in azureBlobFactory') } blobServiceClient = new BlobServiceClient(`https://${account}.blob.core.windows.net`, credential, pipelineOptions) } diff --git a/providers/store/azureQueueStore.js b/providers/store/azureQueueStore.js index 25c020b..17afff2 100644 --- a/providers/store/azureQueueStore.js +++ b/providers/store/azureQueueStore.js @@ -23,13 +23,16 @@ class AzureStorageQueue { } if (connectionString) { this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions) + console.info('using connection string in azureQueueStore') } else { let credential if (spnAuth) { const authParsed = JSON.parse(spnAuth) credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret) + console.info('using SPN in azureQueueStore') } else { credential = new DefaultAzureCredential() + console.info('using default auth in azureQueueStore') } this.client = new QueueServiceClient(`https://${account}.queue.core.windows.net`, credential, pipelineOptions) } @@ -41,8 +44,21 @@ class AzureStorageQueue { } async upsert(document) { - const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64') - return await this.queueService.sendMessage(message) + try { + this.logger.verbose(`Attempting to send message from azureQueueStore:`, { + bodyType: typeof body + }) + const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64') + return await this.queueService.sendMessage(message) + } catch (error) { + this.logger.error('Failed to push message from azureQueueStore:', { + error: error.stack, + request: document, + type: typeof document, + keys: Object.keys(document) + }) + throw error + } } get() {