chore(deps): migrate from azure-storage to @azure/storage-* SDK

package.json

@@ -25,6 +25,8 @@
     "url": "git+https://github.com/clearlydefined/service.git"
   },
   "dependencies": {
+    "@azure/storage-blob": "12.30.0",
+    "@azure/storage-queue": "12.29.0",
     "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.10",
     "@gitbeaker/rest": "^43.8.0",
     "@octokit/rest": "^22.0.0",
@@ -33,7 +35,6 @@
     "ajv-formats": "3.0.1",
     "applicationinsights": "^3.13.0",
     "axios": "^1.10.0",
-    "azure-storage": "^2.10.2",
     "base-64": "^1.0.0",
     "body-parser": "^2.2.0",
     "bottleneck": "^2.15.3",

providers/queueing/azureStorageQueue.js

@@ -1,9 +1,8 @@
 // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const azure = require('azure-storage')
+const { QueueServiceClient } = require('@azure/storage-queue')
 const logger = require('../logging/logger')
-const { promisify } = require('util')
 
 class AzureStorageQueue {
   constructor(options) {
@@ -12,10 +11,9 @@ class AzureStorageQueue {
   }
 
   async initialize() {
-    this.queueService = azure
-      .createQueueService(this.options.connectionString)
-      .withFilter(new azure.LinearRetryPolicyFilter())
-    await promisify(this.queueService.createQueueIfNotExists).bind(this.queueService)(this.options.queueName)
+    this.queueServiceClient = QueueServiceClient.fromConnectionString(this.options.connectionString)
+    this.queueClient = this.queueServiceClient.getQueueClient(this.options.queueName)
+    await this.queueClient.createIfNotExists()
   }
 
   /**
@@ -25,7 +23,7 @@ class AzureStorageQueue {
    * @param {string} message
    */
   async queue(message) {
-    await promisify(this.queueService.createMessage).bind(this.queueService)(this.options.queueName, message)
+    await this.queueClient.sendMessage(message)
   }
 
   /**
@@ -34,35 +32,45 @@ class AzureStorageQueue {
    * Returns null if the queue is empty
    * If DQ count exceeds 5 the message will be deleted and the next message will be returned
    *
-   * @returns {object} - { original: message, data: "JSON parsed, base64 decoded message" }
+   * @returns {object} - { original: message, data: "JSON parsed message" }
    */
   async dequeue() {
-    const message = await promisify(this.queueService.getMessage).bind(this.queueService)(this.options.queueName)
-    if (!message) return null
-    if (message.dequeueCount <= 5)
-      return { original: message, data: JSON.parse(Buffer.from(message.messageText, 'base64').toString('utf8')) }
+    const response = await this.queueClient.receiveMessages({ numberOfMessages: 1 })
+    if (!response.receivedMessageItems || response.receivedMessageItems.length === 0) return null
+
+    const message = response.receivedMessageItems[0]
+    if (message.dequeueCount <= 5) {
+      return {
+        original: message,
+        data: JSON.parse(message.messageText)
+      }
+    }
     await this.delete({ original: message })
     return this.dequeue()
   }
 
   /** Similar to dequeue() but returns multiple messages to improve performance */
   async dequeueMultiple() {
-    const messages = await promisify(this.queueService.getMessages).bind(this.queueService)(
-      this.options.queueName,
-      this.options.dequeueOptions
-    )
-    if (!messages || messages.length === 0) return []
-    for (const i in messages) {
-      if (messages[i].dequeueCount <= 5) {
-        messages[i] = {
-          original: messages[i],
-          data: JSON.parse(Buffer.from(messages[i].messageText, 'base64').toString('utf8'))
-        }
+    const options = this.options.dequeueOptions || {}
+    const response = await this.queueClient.receiveMessages({
+      numberOfMessages: options.numOfMessages || 32,
+      visibilityTimeout: options.visibilityTimeout
+    })
+
+    if (!response.receivedMessageItems || response.receivedMessageItems.length === 0) return []
+
+    const results = []
+    for (const message of response.receivedMessageItems) {
+      if (message.dequeueCount <= 5) {
+        results.push({
+          original: message,
+          data: JSON.parse(message.messageText)
+        })
       } else {
-        await this.delete({ original: messages[i] })
+        await this.delete({ original: message })
       }
     }
-    return messages
+    return results
   }
 
   /**
@@ -72,11 +80,7 @@ class AzureStorageQueue {
    * @param {object} message
    */
   async delete(message) {
-    await promisify(this.queueService.deleteMessage).bind(this.queueService)(
-      this.options.queueName,
-      message.original.messageId,
-      message.original.popReceipt
-    )
+    await this.queueClient.deleteMessage(message.original.messageId, message.original.popReceipt)
   }
 }
 

providers/stores/abstractAzblobStore.js

@@ -1,12 +1,10 @@
 // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const azure = require('azure-storage')
+const { BlobServiceClient } = require('@azure/storage-blob')
 const AbstractFileStore = require('./abstractFileStore')
 const logger = require('../logging/logger')
 
-const { promisify } = require('util')
-
 /**
  * @typedef {import('./abstractAzblobStore').AzBlobStoreOptions} AzBlobStoreOptions
  * @typedef {import('./abstractAzblobStore').BlobEntry} BlobEntry
@@ -40,10 +38,9 @@ class AbstractAzBlobStore {
    * @returns {Promise<void>} Promise that resolves when initialization is complete
    */
   async initialize() {
-    this.blobService = azure
-      .createBlobService(this.options.connectionString)
-      .withFilter(new azure.LinearRetryPolicyFilter())
-    return promisify(this.blobService.createContainerIfNotExists).bind(this.blobService)(this.containerName)
+    this.blobServiceClient = BlobServiceClient.fromConnectionString(this.options.connectionString)
+    this.containerClient = this.blobServiceClient.getContainerClient(this.containerName)
+    await this.containerClient.createIfNotExists()
   }
 
   /**
@@ -57,27 +54,20 @@ class AbstractAzBlobStore {
   async list(coordinates, visitor) {
     /** @type {any[]} */
     const list = []
-    let continuation = null
-    do {
-      const name = AbstractFileStore.toStoragePathFromCoordinates(coordinates)
-      // @ts-ignore - azure-storage promisify signature differs from standard promisify
-      const result = await promisify(this.blobService.listBlobsSegmentedWithPrefix).bind(this.blobService)(
-        this.containerName,
-        name,
-        continuation,
-        // @ts-ignore - azure-storage expects 4 args for this operation
-        {
-          include: azure.BlobUtilities.BlobListingDetails.METADATA
-        }
-      )
-      continuation = result.continuationToken
-      result.entries.forEach(
-        /** @param {BlobEntry} entry */ entry => {
-          const visitResult = visitor(entry)
-          if (visitResult !== null) list.push(visitResult)
-        }
-      )
-    } while (continuation)
+    const name = AbstractFileStore.toStoragePathFromCoordinates(coordinates)
+    const listOptions = {
+      prefix: name,
+      includeMetadata: true
+    }
+
+    for await (const blob of this.containerClient.listBlobsFlat(listOptions)) {
+      const entry = {
+        name: blob.name,
+        metadata: blob.metadata || {}
+      }
+      const visitResult = visitor(entry)
+      if (visitResult !== null) list.push(visitResult)
+    }
     return list
   }
 
@@ -91,8 +81,10 @@ class AbstractAzBlobStore {
     let name = AbstractFileStore.toStoragePathFromCoordinates(coordinates)
     if (!name.endsWith('.json')) name += '.json'
     try {
-      const result = await promisify(this.blobService.getBlobToText).bind(this.blobService)(this.containerName, name)
-      return JSON.parse(result)
+      const blobClient = this.containerClient.getBlobClient(name)
+      const downloadResponse = await blobClient.download()
+      const content = await this._streamToString(downloadResponse.readableStreamBody)
+      return JSON.parse(content)
     } catch (error) {
       const azureError = /** @type {{statusCode?: number}} */ (error)
       if (azureError.statusCode === 404) return null
@@ -130,6 +122,22 @@ class AbstractAzBlobStore {
   _toResultCoordinatesFromStoragePath(path) {
     return AbstractFileStore.toResultCoordinatesFromStoragePath(path)
   }
+
+  /**
+   * Helper to convert a readable stream to a string
+   *
+   * @protected
+   * @param {NodeJS.ReadableStream} readableStream - The stream to convert
+   * @returns {Promise<string>} The string content
+   */
+  async _streamToString(readableStream) {
+    /** @type {Buffer[]} */
+    const chunks = []
+    for await (const chunk of readableStream) {
+      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+    }
+    return Buffer.concat(chunks).toString('utf8')
+  }
 }
 
 module.exports = AbstractAzBlobStore

providers/stores/azblobAttachmentStore.js

@@ -1,8 +1,7 @@
 // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 
-const azure = require('azure-storage')
-const { promisify } = require('util')
+const { BlobServiceClient } = require('@azure/storage-blob')
 const Bottleneck = require('bottleneck').default
 const limiter = new Bottleneck({ maxConcurrent: 1000 })
 const logger = require('../logging/logger')
@@ -15,10 +14,9 @@ class AzBlobAttachmentStore {
   }
 
   async initialize() {
-    this.blobService = azure
-      .createBlobService(this.options.connectionString)
-      .withFilter(new azure.LinearRetryPolicyFilter())
-    return promisify(this.blobService.createContainerIfNotExists).bind(this.blobService)(this.containerName)
+    this.blobServiceClient = BlobServiceClient.fromConnectionString(this.options.connectionString)
+    this.containerClient = this.blobServiceClient.getContainerClient(this.containerName)
+    await this.containerClient.createIfNotExists()
   }
 
   /**
@@ -32,15 +30,33 @@ class AzBlobAttachmentStore {
       try {
         const name = 'attachment/' + key + '.json'
         this.logger.info('2:1:1:notice_generate:get_single_file:start', { ts: new Date().toISOString(), file: key })
-        const result = await promisify(this.blobService.getBlobToText).bind(this.blobService)(this.containerName, name)
+        const blobClient = this.containerClient.getBlobClient(name)
+        const downloadResponse = await blobClient.download()
+        const content = await this._streamToString(downloadResponse.readableStreamBody)
         this.logger.info('2:1:1:notice_generate:get_single_file:end', { ts: new Date().toISOString(), file: key })
-        return JSON.parse(result).attachment
+        return JSON.parse(content).attachment
       } catch (error) {
         if (error.statusCode === 404) return null
         throw error
       }
     })()
   }
+
+  /**
+   * Helper to convert a readable stream to a string
+   *
+   * @private
+   * @param {NodeJS.ReadableStream} readableStream - The stream to convert
+   * @returns {Promise<string>} The string content
+   */
+  async _streamToString(readableStream) {
+    /** @type {Buffer[]} */
+    const chunks = []
+    for await (const chunk of readableStream) {
+      chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+    }
+    return Buffer.concat(chunks).toString('utf8')
+  }
 }
 
 module.exports = options => new AzBlobAttachmentStore(options)

providers/stores/azblobDefinitionStore.js

@@ -5,7 +5,6 @@ const AbstractAzBlobStore = require('./abstractAzblobStore')
 const AbstractFileStore = require('./abstractFileStore')
 const EntityCoordinates = require('../../lib/entityCoordinates')
 const { sortedUniq } = require('lodash')
-const { promisify } = require('util')
 
 class AzBlobDefinitionStore extends AbstractAzBlobStore {
   /**
@@ -24,26 +23,23 @@ class AzBlobDefinitionStore extends AbstractAzBlobStore {
     return sortedUniq(list.filter(x => x))
   }
 
-  store(definition) {
+  async store(definition) {
     const blobName = this._toStoragePathFromCoordinates(definition.coordinates) + '.json'
-    return promisify(this.blobService.createBlockBlobFromText).bind(this.blobService)(
-      this.containerName,
-      blobName,
-      JSON.stringify(definition),
-      {
-        blockIdPrefix: 'block',
-        contentSettings: { contentType: 'application/json' },
-        metadata: { id: definition.coordinates.toString() }
-      }
-    )
+    const blockBlobClient = this.containerClient.getBlockBlobClient(blobName)
+    const content = JSON.stringify(definition)
+    await blockBlobClient.upload(content, Buffer.byteLength(content), {
+      blobHTTPHeaders: { blobContentType: 'application/json' },
+      metadata: { id: definition.coordinates.toString() }
+    })
   }
 
   async delete(coordinates) {
     const blobName = this._toStoragePathFromCoordinates(coordinates) + '.json'
+    const blobClient = this.containerClient.getBlobClient(blobName)
     try {
-      await promisify(this.blobService.deleteBlob).bind(this.blobService)(this.containerName, blobName)
+      await blobClient.delete()
     } catch (error) {
-      if (error.code !== 'BlobNotFound') throw error
+      if (error.statusCode !== 404) throw error
     }
   }
 }