diff --git a/package-lock.json b/package-lock.json index b865c5884..2653c217b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,8 @@ "version": "2.4.0", "license": "MIT", "dependencies": { + "@azure/storage-blob": "12.30.0", + "@azure/storage-queue": "12.29.0", "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.10", "@gitbeaker/rest": "^43.8.0", "@octokit/rest": "^22.0.0", @@ -17,7 +19,6 @@ "ajv-formats": "3.0.1", "applicationinsights": "^3.13.0", "axios": "^1.10.0", - "azure-storage": "^2.10.2", "base-64": "^1.0.0", "body-parser": "^2.2.0", "bottleneck": "^2.15.3", @@ -157,6 +158,47 @@ "node": ">=20.0.0" } }, + "node_modules/@azure/core-http-compat": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.3.1.tgz", + "integrity": "sha512-az9BkXND3/d5VgdRRQVkiJb2gOmDU8Qcq4GvjtBmDICNiQ9udFmDk4ZpSB5Qq1OmtDJGlQAfBaS4palFsazQ5g==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/core-rest-pipeline": { "version": "1.22.2", "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.22.2.tgz", @@ -201,6 +243,19 @@ "node": ">=20.0.0" } }, + "node_modules/@azure/core-xml": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.5.0.tgz", + "integrity": "sha512-D/sdlJBMJfx7gqoj66PKVmhDDaU6TKA49ptcolxdas29X7AfvLTmfAGLjAcIMBK7UZ2o4lygHIqVckOlQU3xWw==", + "license": "MIT", + "dependencies": { + "fast-xml-parser": "^5.0.7", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@azure/functions": { "version": "4.11.0", "resolved": "https://registry.npmjs.org/@azure/functions/-/functions-4.11.0.tgz", @@ -551,6 +606,74 @@ "node": ">=8.6.0" } }, + "node_modules/@azure/storage-blob": { + "version": "12.30.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.30.0.tgz", + "integrity": "sha512-peDCR8blSqhsAKDbpSP/o55S4sheNwSrblvCaHUZ5xUI73XA7ieUGGwrONgD/Fng0EoDe1VOa3fAQ7+WGB3Ocg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.5", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.2.0", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-common": { + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.2.0.tgz", + "integrity": "sha512-YZLxiJ3vBAAnFbG3TFuAMUlxZRexjQX5JDQxOkFGb6e2TpoxH3xyHI6idsMe/QrWtj41U/KoqBxlayzhS+LlwA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-queue": { + "version": "12.29.0", + "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.29.0.tgz", + "integrity": "sha512-p02H+TbPQWSI/SQ4CG+luoDvpenM+4837NARmOE4oPNOR5vAq7qRyeX72ffyYL2YLnkcyxETh28/bp/TiVIM+g==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.2.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.27.1", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", @@ -4513,21 +4636,6 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, - "node_modules/available-typed-arrays": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", - "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "license": "MIT", - "dependencies": { - "possible-typed-array-names": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -4554,53 +4662,6 @@ "proxy-from-env": "^1.1.0" } }, - "node_modules/azure-storage": { - "version": "2.10.7", - "resolved": "https://registry.npmjs.org/azure-storage/-/azure-storage-2.10.7.tgz", - "integrity": "sha512-4oeFGtn3Ziw/fGs/zkoIpKKtygnCVIcZwzJ7UQzKTxhkGQqVCByOFbYqMGYR3L+wOsunX9lNfD0jc51SQuKSSA==", - "deprecated": "Please note: newer packages @azure/storage-blob, @azure/storage-queue and @azure/storage-file are available as of November 2019 and @azure/data-tables is available as of June 2021. While the legacy azure-storage package will continue to receive critical bug fixes, we strongly encourage you to upgrade. Migration guide can be found: https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/MigrationGuide.md", - "license": "Apache-2.0", - "dependencies": { - "browserify-mime": "^1.2.9", - "extend": "^3.0.2", - "json-edm-parser": "~0.1.2", - "json-schema": "~0.4.0", - "md5.js": "^1.3.4", - "readable-stream": "^2.0.0", - "request": "^2.86.0", - "underscore": "^1.12.1", - "uuid": "^3.0.0", - "validator": "^13.7.0", - "xml2js": "~0.2.8", - "xmlbuilder": "^9.0.7" - }, - "engines": { - "node": ">= 0.8.26" - } - }, - "node_modules/azure-storage/node_modules/sax": { - "version": "0.5.8", - "resolved": "https://registry.npmjs.org/sax/-/sax-0.5.8.tgz", - "integrity": "sha512-c0YL9VcSfcdH3F1Qij9qpYJFpKFKMXNOkLWFssBL3RuF7ZS8oZhllR2rWlCRjDTJsfq3R6wbSsaRU6o0rkEdNw==", - "license": "BSD" - }, - "node_modules/azure-storage/node_modules/xml2js": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz", - "integrity": "sha512-ZHZBIAO55GHCn2jBYByVPHvHS+o3j8/a/qmpEe6kxO3cTnTCWC3Htq9RYJ5G4XMwMMClD2QkXA9SNdPadLyn3Q==", - "dependencies": { - "sax": "0.5.x" - } - }, - "node_modules/azure-storage/node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==", - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, "node_modules/b4a": { "version": "1.7.3", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz", @@ -4959,11 +5020,6 @@ "dev": true, "license": "ISC" }, - "node_modules/browserify-mime": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/browserify-mime/-/browserify-mime-1.2.9.tgz", - "integrity": "sha512-uz+ItyJXBLb6wgon1ELEiVowJBEsy03PUWGRQU7cxxx9S+DW2hujPp+DaMYEOClRPzsn7NB99NtJ6pGnt8y+CQ==" - }, "node_modules/browserslist": { "version": "4.28.1", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.1.tgz", @@ -5119,24 +5175,6 @@ "node": ">=8" } }, - "node_modules/call-bind": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", - "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.0", - "es-define-property": "^1.0.0", - "get-intrinsic": "^1.2.4", - "set-function-length": "^1.2.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -5978,23 +6016,6 @@ "abstract-leveldown": "~2.6.0" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", @@ -6695,7 +6716,6 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -6884,6 +6904,24 @@ ], "license": "BSD-3-Clause" }, + "node_modules/fast-xml-parser": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.3.3.tgz", + "integrity": "sha512-2O3dkPAAC6JavuMm8+4+pgTk+5hoAs+CjZ+sWcQLkX9+/tHRuTkQh/Oaifr8qDmZ8iEHb771Ea6G8CdwkrgvYA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^2.1.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -7053,21 +7091,6 @@ } } }, - "node_modules/for-each": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", - "integrity": "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==", - "license": "MIT", - "dependencies": { - "is-callable": "^1.2.7" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", @@ -7583,18 +7606,6 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -7622,21 +7633,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/hash-base": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.2.tgz", - "integrity": "sha512-Bb33KbowVTIj5s7Ked1OsqHUeCpz//tPwR+E2zJgJKo9Z5XolZ9b6bdUgjmYlwnWhoOQKoTd1TYToZGn5mAYOg==", - "license": "MIT", - "dependencies": { - "inherits": "^2.0.4", - "readable-stream": "^2.3.8", - "safe-buffer": "^5.2.1", - "to-buffer": "^1.2.1" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/hasha": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", @@ -7993,18 +7989,6 @@ "node": ">=8" } }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", @@ -8143,21 +8127,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/is-typed-array": { - "version": "1.1.15", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.15.tgz", - "integrity": "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ==", - "license": "MIT", - "dependencies": { - "which-typed-array": "^1.1.16" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -8206,6 +8175,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, "license": "MIT" }, "node_modules/isexe": { @@ -8427,15 +8397,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-edm-parser": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/json-edm-parser/-/json-edm-parser-0.1.2.tgz", - "integrity": "sha512-J1U9mk6lf8dPULcaMwALXB6yel3cJyyhk9Z8FQ4sMwiazNwjaUhegIcpZyZFNMvLRtnXwh+TkCjX9uYUObBBYA==", - "license": "MIT", - "dependencies": { - "jsonparse": "~1.2.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -8474,15 +8435,6 @@ "node": ">=6" } }, - "node_modules/jsonparse": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.2.0.tgz", - "integrity": "sha512-LkDEYtKnPFI9hQ/IURETe6F1dUH80cbRkaF6RaViSwoSNPwaxQpi6TgJGvJKyLQ2/9pQW+XCxK3hBoR44RAjkg==", - "engines": [ - "node >= 0.2.0" - ], - "license": "MIT" - }, "node_modules/jsonwebtoken": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", @@ -8859,17 +8811,6 @@ "node": ">= 0.4" } }, - "node_modules/md5.js": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", - "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", - "license": "MIT", - "dependencies": { - "hash-base": "^3.0.0", - "inherits": "^2.0.1", - "safe-buffer": "^5.1.2" - } - }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -10983,15 +10924,6 @@ "node": ">= 10.12" } }, - "node_modules/possible-typed-array-names": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", - "integrity": "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", @@ -11071,6 +11003,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, "license": "MIT" }, "node_modules/process-on-spawn": { @@ -11364,6 +11297,7 @@ "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, "license": "MIT", "dependencies": { "core-util-is": "~1.0.0", @@ -11379,6 +11313,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, "license": "MIT" }, "node_modules/readdir-glob": { @@ -11895,23 +11830,6 @@ "dev": true, "license": "ISC" }, - "node_modules/set-function-length": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", - "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", - "license": "MIT", - "dependencies": { - "define-data-property": "^1.1.4", - "es-errors": "^1.3.0", - "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.4", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -12499,6 +12417,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.1.2.tgz", + "integrity": "sha512-l63NF9y/cLROq/yqKXSLtcMeeyOfnSQlfMSlzFt/K73oIaD8DGaQWd7Z34X9GPiKqP5rbSh84Hl4bOlLcjiSrQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/superagent": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/superagent/-/superagent-10.3.0.tgz", @@ -12879,26 +12809,6 @@ "node": ">=14.14" } }, - "node_modules/to-buffer": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.2.2.tgz", - "integrity": "sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw==", - "license": "MIT", - "dependencies": { - "isarray": "^2.0.5", - "safe-buffer": "^5.2.1", - "typed-array-buffer": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/to-buffer/node_modules/isarray": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", - "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", - "license": "MIT" - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -13056,20 +12966,6 @@ "node": ">= 0.6" } }, - "node_modules/typed-array-buffer": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz", - "integrity": "sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.3", - "es-errors": "^1.3.0", - "is-typed-array": "^1.1.14" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/typed-error": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/typed-error/-/typed-error-3.2.2.tgz", @@ -13131,12 +13027,6 @@ "dev": true, "license": "MIT" }, - "node_modules/underscore": { - "version": "1.13.7", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.7.tgz", - "integrity": "sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g==", - "license": "MIT" - }, "node_modules/undici": { "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici/-/undici-7.18.2.tgz", @@ -13246,15 +13136,6 @@ "uuid": "bin/uuid" } }, - "node_modules/validator": { - "version": "13.15.26", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.15.26.tgz", - "integrity": "sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/value-or-promise": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/value-or-promise/-/value-or-promise-1.0.11.tgz", @@ -13348,27 +13229,6 @@ "dev": true, "license": "ISC" }, - "node_modules/which-typed-array": { - "version": "1.1.20", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.20.tgz", - "integrity": "sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg==", - "license": "MIT", - "dependencies": { - "available-typed-arrays": "^1.0.7", - "call-bind": "^1.0.8", - "call-bound": "^1.0.4", - "for-each": "^0.3.5", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-tostringtag": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/winston": { "version": "3.19.0", "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", diff --git a/package.json b/package.json index 735889168..ae065b0e4 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "url": "git+https://github.com/clearlydefined/service.git" }, "dependencies": { + "@azure/storage-blob": "12.30.0", + "@azure/storage-queue": "12.29.0", "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.10", "@gitbeaker/rest": "^43.8.0", "@octokit/rest": "^22.0.0", @@ -33,7 +35,6 @@ "ajv-formats": "3.0.1", "applicationinsights": "^3.13.0", "axios": "^1.10.0", - "azure-storage": "^2.10.2", "base-64": "^1.0.0", "body-parser": "^2.2.0", "bottleneck": "^2.15.3", diff --git a/providers/queueing/azureStorageQueue.js b/providers/queueing/azureStorageQueue.js index 7f1523f76..282d2888b 100644 --- a/providers/queueing/azureStorageQueue.js +++ b/providers/queueing/azureStorageQueue.js @@ -1,9 +1,8 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const azure = require('azure-storage') +const { QueueServiceClient } = require('@azure/storage-queue') const logger = require('../logging/logger') -const { promisify } = require('util') class AzureStorageQueue { constructor(options) { @@ -12,10 +11,9 @@ class AzureStorageQueue { } async initialize() { - this.queueService = azure - .createQueueService(this.options.connectionString) - .withFilter(new azure.LinearRetryPolicyFilter()) - await promisify(this.queueService.createQueueIfNotExists).bind(this.queueService)(this.options.queueName) + this.queueServiceClient = QueueServiceClient.fromConnectionString(this.options.connectionString) + this.queueClient = this.queueServiceClient.getQueueClient(this.options.queueName) + await this.queueClient.createIfNotExists() } /** @@ -25,7 +23,7 @@ class AzureStorageQueue { * @param {string} message */ async queue(message) { - await promisify(this.queueService.createMessage).bind(this.queueService)(this.options.queueName, message) + await this.queueClient.sendMessage(message) } /** @@ -34,35 +32,45 @@ class AzureStorageQueue { * Returns null if the queue is empty * If DQ count exceeds 5 the message will be deleted and the next message will be returned * - * @returns {object} - { original: message, data: "JSON parsed, base64 decoded message" } + * @returns {object} - { original: message, data: "JSON parsed message" } */ async dequeue() { - const message = await promisify(this.queueService.getMessage).bind(this.queueService)(this.options.queueName) - if (!message) return null - if (message.dequeueCount <= 5) - return { original: message, data: JSON.parse(Buffer.from(message.messageText, 'base64').toString('utf8')) } + const response = await this.queueClient.receiveMessages({ numberOfMessages: 1 }) + if (!response.receivedMessageItems || response.receivedMessageItems.length === 0) return null + + const message = response.receivedMessageItems[0] + if (message.dequeueCount <= 5) { + return { + original: message, + data: JSON.parse(message.messageText) + } + } await this.delete({ original: message }) return this.dequeue() } /** Similar to dequeue() but returns multiple messages to improve performance */ async dequeueMultiple() { - const messages = await promisify(this.queueService.getMessages).bind(this.queueService)( - this.options.queueName, - this.options.dequeueOptions - ) - if (!messages || messages.length === 0) return [] - for (const i in messages) { - if (messages[i].dequeueCount <= 5) { - messages[i] = { - original: messages[i], - data: JSON.parse(Buffer.from(messages[i].messageText, 'base64').toString('utf8')) - } + const options = this.options.dequeueOptions || {} + const response = await this.queueClient.receiveMessages({ + numberOfMessages: options.numOfMessages || 32, + visibilityTimeout: options.visibilityTimeout + }) + + if (!response.receivedMessageItems || response.receivedMessageItems.length === 0) return [] + + const results = [] + for (const message of response.receivedMessageItems) { + if (message.dequeueCount <= 5) { + results.push({ + original: message, + data: JSON.parse(message.messageText) + }) } else { - await this.delete({ original: messages[i] }) + await this.delete({ original: message }) } } - return messages + return results } /** @@ -72,11 +80,7 @@ class AzureStorageQueue { * @param {object} message */ async delete(message) { - await promisify(this.queueService.deleteMessage).bind(this.queueService)( - this.options.queueName, - message.original.messageId, - message.original.popReceipt - ) + await this.queueClient.deleteMessage(message.original.messageId, message.original.popReceipt) } } diff --git a/providers/stores/abstractAzblobStore.js b/providers/stores/abstractAzblobStore.js index 81d61fb48..0ef6b9fa1 100644 --- a/providers/stores/abstractAzblobStore.js +++ b/providers/stores/abstractAzblobStore.js @@ -1,12 +1,10 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const azure = require('azure-storage') +const { BlobServiceClient } = require('@azure/storage-blob') const AbstractFileStore = require('./abstractFileStore') const logger = require('../logging/logger') -const { promisify } = require('util') - /** * @typedef {import('./abstractAzblobStore').AzBlobStoreOptions} AzBlobStoreOptions * @typedef {import('./abstractAzblobStore').BlobEntry} BlobEntry @@ -40,10 +38,9 @@ class AbstractAzBlobStore { * @returns {Promise} Promise that resolves when initialization is complete */ async initialize() { - this.blobService = azure - .createBlobService(this.options.connectionString) - .withFilter(new azure.LinearRetryPolicyFilter()) - return promisify(this.blobService.createContainerIfNotExists).bind(this.blobService)(this.containerName) + this.blobServiceClient = BlobServiceClient.fromConnectionString(this.options.connectionString) + this.containerClient = this.blobServiceClient.getContainerClient(this.containerName) + await this.containerClient.createIfNotExists() } /** @@ -57,27 +54,20 @@ class AbstractAzBlobStore { async list(coordinates, visitor) { /** @type {any[]} */ const list = [] - let continuation = null - do { - const name = AbstractFileStore.toStoragePathFromCoordinates(coordinates) - // @ts-ignore - azure-storage promisify signature differs from standard promisify - const result = await promisify(this.blobService.listBlobsSegmentedWithPrefix).bind(this.blobService)( - this.containerName, - name, - continuation, - // @ts-ignore - azure-storage expects 4 args for this operation - { - include: azure.BlobUtilities.BlobListingDetails.METADATA - } - ) - continuation = result.continuationToken - result.entries.forEach( - /** @param {BlobEntry} entry */ entry => { - const visitResult = visitor(entry) - if (visitResult !== null) list.push(visitResult) - } - ) - } while (continuation) + const name = AbstractFileStore.toStoragePathFromCoordinates(coordinates) + const listOptions = { + prefix: name, + includeMetadata: true + } + + for await (const blob of this.containerClient.listBlobsFlat(listOptions)) { + const entry = { + name: blob.name, + metadata: blob.metadata || {} + } + const visitResult = visitor(entry) + if (visitResult !== null) list.push(visitResult) + } return list } @@ -91,8 +81,10 @@ class AbstractAzBlobStore { let name = AbstractFileStore.toStoragePathFromCoordinates(coordinates) if (!name.endsWith('.json')) name += '.json' try { - const result = await promisify(this.blobService.getBlobToText).bind(this.blobService)(this.containerName, name) - return JSON.parse(result) + const blobClient = this.containerClient.getBlobClient(name) + const downloadResponse = await blobClient.download() + const content = await this._streamToString(downloadResponse.readableStreamBody) + return JSON.parse(content) } catch (error) { const azureError = /** @type {{statusCode?: number}} */ (error) if (azureError.statusCode === 404) return null @@ -130,6 +122,22 @@ class AbstractAzBlobStore { _toResultCoordinatesFromStoragePath(path) { return AbstractFileStore.toResultCoordinatesFromStoragePath(path) } + + /** + * Helper to convert a readable stream to a string + * + * @protected + * @param {NodeJS.ReadableStream} readableStream - The stream to convert + * @returns {Promise} The string content + */ + async _streamToString(readableStream) { + /** @type {Buffer[]} */ + const chunks = [] + for await (const chunk of readableStream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString('utf8') + } } module.exports = AbstractAzBlobStore diff --git a/providers/stores/azblobAttachmentStore.js b/providers/stores/azblobAttachmentStore.js index 9a947c241..1a4e542a5 100644 --- a/providers/stores/azblobAttachmentStore.js +++ b/providers/stores/azblobAttachmentStore.js @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const azure = require('azure-storage') -const { promisify } = require('util') +const { BlobServiceClient } = require('@azure/storage-blob') const Bottleneck = require('bottleneck').default const limiter = new Bottleneck({ maxConcurrent: 1000 }) const logger = require('../logging/logger') @@ -15,10 +14,9 @@ class AzBlobAttachmentStore { } async initialize() { - this.blobService = azure - .createBlobService(this.options.connectionString) - .withFilter(new azure.LinearRetryPolicyFilter()) - return promisify(this.blobService.createContainerIfNotExists).bind(this.blobService)(this.containerName) + this.blobServiceClient = BlobServiceClient.fromConnectionString(this.options.connectionString) + this.containerClient = this.blobServiceClient.getContainerClient(this.containerName) + await this.containerClient.createIfNotExists() } /** @@ -32,15 +30,33 @@ class AzBlobAttachmentStore { try { const name = 'attachment/' + key + '.json' this.logger.info('2:1:1:notice_generate:get_single_file:start', { ts: new Date().toISOString(), file: key }) - const result = await promisify(this.blobService.getBlobToText).bind(this.blobService)(this.containerName, name) + const blobClient = this.containerClient.getBlobClient(name) + const downloadResponse = await blobClient.download() + const content = await this._streamToString(downloadResponse.readableStreamBody) this.logger.info('2:1:1:notice_generate:get_single_file:end', { ts: new Date().toISOString(), file: key }) - return JSON.parse(result).attachment + return JSON.parse(content).attachment } catch (error) { if (error.statusCode === 404) return null throw error } })() } + + /** + * Helper to convert a readable stream to a string + * + * @private + * @param {NodeJS.ReadableStream} readableStream - The stream to convert + * @returns {Promise} The string content + */ + async _streamToString(readableStream) { + /** @type {Buffer[]} */ + const chunks = [] + for await (const chunk of readableStream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)) + } + return Buffer.concat(chunks).toString('utf8') + } } module.exports = options => new AzBlobAttachmentStore(options) diff --git a/providers/stores/azblobDefinitionStore.js b/providers/stores/azblobDefinitionStore.js index 389a8d775..18efd7502 100644 --- a/providers/stores/azblobDefinitionStore.js +++ b/providers/stores/azblobDefinitionStore.js @@ -5,7 +5,6 @@ const AbstractAzBlobStore = require('./abstractAzblobStore') const AbstractFileStore = require('./abstractFileStore') const EntityCoordinates = require('../../lib/entityCoordinates') const { sortedUniq } = require('lodash') -const { promisify } = require('util') class AzBlobDefinitionStore extends AbstractAzBlobStore { /** @@ -24,26 +23,23 @@ class AzBlobDefinitionStore extends AbstractAzBlobStore { return sortedUniq(list.filter(x => x)) } - store(definition) { + async store(definition) { const blobName = this._toStoragePathFromCoordinates(definition.coordinates) + '.json' - return promisify(this.blobService.createBlockBlobFromText).bind(this.blobService)( - this.containerName, - blobName, - JSON.stringify(definition), - { - blockIdPrefix: 'block', - contentSettings: { contentType: 'application/json' }, - metadata: { id: definition.coordinates.toString() } - } - ) + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName) + const content = JSON.stringify(definition) + await blockBlobClient.upload(content, Buffer.byteLength(content), { + blobHTTPHeaders: { blobContentType: 'application/json' }, + metadata: { id: definition.coordinates.toString() } + }) } async delete(coordinates) { const blobName = this._toStoragePathFromCoordinates(coordinates) + '.json' + const blobClient = this.containerClient.getBlobClient(blobName) try { - await promisify(this.blobService.deleteBlob).bind(this.blobService)(this.containerName, blobName) + await blobClient.delete() } catch (error) { - if (error.code !== 'BlobNotFound') throw error + if (error.statusCode !== 404) throw error } } } diff --git a/providers/stores/azblobHarvestStore.js b/providers/stores/azblobHarvestStore.js index ae110e57f..f98b5c868 100644 --- a/providers/stores/azblobHarvestStore.js +++ b/providers/stores/azblobHarvestStore.js @@ -6,9 +6,6 @@ const AbstractFileStore = require('./abstractFileStore') const ResultCoordinates = require('../../lib/resultCoordinates') const { sortedUniq } = require('lodash') -const resultOrError = (resolve, reject) => (error, result) => (error ? reject(error) : resolve(result)) -const responseOrError = (resolve, reject) => (error, result, response) => (error ? reject(error) : resolve(response)) - class AzHarvestBlobStore extends AbstractAzBlobStore { /** * List all of the results for the given coordinates. @@ -32,12 +29,16 @@ class AzHarvestBlobStore extends AbstractAzBlobStore { * @param {ResultCoordinates} coordinates - The coordinates of the content to access * @param {WriteStream} [stream] - The stream onto which the output is written */ - stream(coordinates, stream) { + async stream(coordinates, stream) { let name = this._toStoragePathFromCoordinates(coordinates) if (!name.endsWith('.json')) name += '.json' - return new Promise((resolve, reject) => - this.blobService.getBlobToStream(this.containerName, name, stream, responseOrError(resolve, reject)) - ) + const blobClient = this.containerClient.getBlobClient(name) + const downloadResponse = await blobClient.download() + return new Promise((resolve, reject) => { + downloadResponse.readableStreamBody.pipe(stream) + downloadResponse.readableStreamBody.on('end', () => resolve(downloadResponse)) + downloadResponse.readableStreamBody.on('error', reject) + }) } /** @@ -54,42 +55,42 @@ class AzHarvestBlobStore extends AbstractAzBlobStore { return await this._getContent(allFilesList) } - _getListOfAllFiles(coordinates) { + async _getListOfAllFiles(coordinates) { const name = this._toStoragePathFromCoordinates(coordinates) - return new Promise((resolve, reject) => - this.blobService.listBlobsSegmentedWithPrefix(this.containerName, name, null, resultOrError(resolve, reject)) - ).then(files => - files.entries.filter(file => { - return ( - file.name.length === name.length || // either an exact match, or - (file.name.length > name.length && // a longer string - (file.name[name.length] === '/' || // where the next character starts extra tool indications - file.name.substr(name.length) === '.json')) - ) - }) - ) + const files = [] + + for await (const blob of this.containerClient.listBlobsFlat({ prefix: name })) { + if ( + blob.name.length === name.length || // either an exact match, or + (blob.name.length > name.length && // a longer string + (blob.name[name.length] === '/' || // where the next character starts extra tool indications + blob.name.substring(name.length) === '.json')) + ) { + files.push({ name: blob.name }) + } + } + return files } - _getContent(files) { - const contents = Promise.all( - files.map(file => { - return new Promise((resolve, reject) => - this.blobService.getBlobToText(this.containerName, file.name, resultOrError(resolve, reject)) - ).then(result => { - return { name: file.name, content: JSON.parse(result) } - }) + async _getContent(files) { + const entries = await Promise.all( + files.map(async file => { + const blobClient = this.containerClient.getBlobClient(file.name) + const downloadResponse = await blobClient.download() + const content = await this._streamToString(downloadResponse.readableStreamBody) + return { name: file.name, content: JSON.parse(content) } }) ) - return contents.then(entries => { - return entries.reduce((result, entry) => { - const { tool, toolVersion } = this._toResultCoordinatesFromStoragePath(entry.name) - // TODO: LOG HERE THERE IF THERE ARE SOME BOGUS FILES HANGING AROUND - if (!tool || !toolVersion) return result - const current = (result[tool] = result[tool] || {}) - current[toolVersion] = entry.content - return result - }, {}) - }) + + return entries.reduce((result, entry) => { + const { tool, toolVersion } = this._toResultCoordinatesFromStoragePath(entry.name) + // TODO: LOG HERE THERE IF THERE ARE SOME BOGUS FILES HANGING AROUND + if (!tool || !toolVersion) return result + const current = result[tool] || {} + result[tool] = current + current[toolVersion] = entry.content + return result + }, {}) } /** diff --git a/test/providers/store/azblobAttachment.js b/test/providers/store/azblobAttachment.js index d0f29f8dc..c932ff6c0 100644 --- a/test/providers/store/azblobAttachment.js +++ b/test/providers/store/azblobAttachment.js @@ -5,6 +5,14 @@ const sinon = require('sinon') const { expect } = require('chai') const Store = require('../../../providers/stores/azblobAttachmentStore') +// Initialize logger for tests +const loggerFactory = require('../../../providers/logging/logger') +try { + loggerFactory({ info: () => {}, error: () => {}, warn: () => {}, debug: () => {} }) +} catch { + // Logger already initialized +} + describe('AzureAttachmentStore list definitions', () => { it('throws original error when not ENOENT', async () => { const store = createStore() @@ -36,16 +44,27 @@ const data = { } function createStore() { - const blobServiceStub = { - getBlobToText: sinon.stub().callsFake((container, path, cb) => { - if (path.includes('error')) return cb(new Error('test error')) - if (data[path]) return cb(null, data[path]) - const error = new Error('not found') - error.statusCode = 404 - cb(error) - }) - } const store = Store({}) - store.blobService = blobServiceStub + + // Mock containerClient + store.containerClient = { + getBlobClient: sinon.stub().callsFake(path => ({ + download: sinon.stub().callsFake(async () => { + if (path.includes('error')) throw new Error('test error') + if (data[path]) { + return { readableStreamBody: createReadableStream(data[path]) } + } + const error = new Error('not found') + error.statusCode = 404 + throw error + }) + })) + } + return store } + +function createReadableStream(content) { + const { Readable } = require('stream') + return Readable.from([Buffer.from(content)]) +} diff --git a/test/providers/store/azblobDefinition.js b/test/providers/store/azblobDefinition.js index bb60afb27..91fca02ae 100644 --- a/test/providers/store/azblobDefinition.js +++ b/test/providers/store/azblobDefinition.js @@ -6,6 +6,14 @@ const sinon = require('sinon') const { expect } = require('chai') const EntityCoordinates = require('../../../lib/entityCoordinates') +// Initialize logger for tests +const loggerFactory = require('../../../providers/logging/logger') +try { + loggerFactory({ info: () => {}, error: () => {}, warn: () => {}, debug: () => {} }) +} catch { + // Logger already initialized +} + describe('azblob Definition store', () => { it('throws original error', async () => { const data = { @@ -65,22 +73,22 @@ describe('azblob Definition store', () => { const definition = createDefinition('npm/npmjs/-/foo/1.0') const store = createStore() await store.store(definition) - expect(store.blobService.createBlockBlobFromText.callCount).to.eq(1) - expect(store.blobService.createBlockBlobFromText.args[0][1]).to.eq('npm/npmjs/-/foo/revision/1.0.json') + expect(store._uploadStub.callCount).to.eq(1) + expect(store._uploadBlobName).to.eq('npm/npmjs/-/foo/revision/1.0.json') }) it('deletes a definition', async () => { const store = createStore() await store.delete(EntityCoordinates.fromString('npm/npmjs/-/foo/1.0')) - expect(store.blobService.deleteBlob.callCount).to.eq(1) - expect(store.blobService.deleteBlob.args[0][1]).to.eq('npm/npmjs/-/foo/revision/1.0.json') + expect(store._deleteStub.callCount).to.eq(1) + expect(store._deleteBlobName).to.eq('npm/npmjs/-/foo/revision/1.0.json') }) it('does not throw deleting missing definition', async () => { const store = createStore() await store.delete(EntityCoordinates.fromString('npm/npmjs/-/missing/1.0')) - expect(store.blobService.deleteBlob.callCount).to.eq(1) - expect(store.blobService.deleteBlob.args[0][1]).to.eq('npm/npmjs/-/missing/revision/1.0.json') + expect(store._deleteStub.callCount).to.eq(1) + expect(store._deleteBlobName).to.eq('npm/npmjs/-/missing/revision/1.0.json') }) it('gets a definition', async () => { @@ -103,37 +111,86 @@ function createDefinitionJson(coordinates) { return JSON.stringify(createDefinition(coordinates)) } -function createStore(data) { - const blobServiceStub = { - listBlobsSegmentedWithPrefix: sinon.stub().callsFake(async (container, name, continuation, metadata, callback) => { - name = name.toLowerCase() - if (name.includes('error')) return callback(new Error('test error')) - callback(null, { - continuationToken: null, - entries: Object.keys(data) - .map(key => (key.startsWith(name) ? data[key] : null)) - .filter(e => e) - }) - }), - createBlockBlobFromText: sinon.stub().callsFake(async (container, name, content, metadata, callback) => { - if (name.includes('error')) return callback(new Error('test error')) - callback() - }), - deleteBlob: sinon.stub().callsFake(async (container, name, callback) => { - if (name.includes('error')) return callback(new Error('test error')) - if (name.includes('missing')) return callback({ code: 'BlobNotFound' }) - callback() - }), - getBlobToText: sinon.stub().callsFake(async (container, name, callback) => { - if (name.includes('error')) return callback(new Error('test error')) - name = name.toLowerCase() - if (data[name]) return callback(null, data[name]) +function createStore(data = {}) { + const store = Store({}) + + // Track blob names for assertions + store._uploadBlobName = null + store._deleteBlobName = null + + // Create stubs + store._uploadStub = sinon.stub().resolves() + store._deleteStub = sinon.stub().callsFake(async () => { + const blobName = store._deleteBlobName + if (blobName && blobName.includes('error')) throw new Error('test error') + if (blobName && blobName.includes('missing')) { const error = new Error('not found') - error.code = 'BlobNotFound' - callback(error) + error.statusCode = 404 + throw error + } + }) + + // Create async iterator for listBlobsFlat + const createAsyncIterator = prefix => { + const prefixLower = prefix.toLowerCase() + const matchingBlobs = Object.keys(data) + .filter(key => key.toLowerCase().startsWith(prefixLower)) + .map(key => ({ name: key, metadata: data[key].metadata })) + + if (prefixLower.includes('error')) { + return { + [Symbol.asyncIterator]() { + return { + next() { + return Promise.reject(new Error('test error')) + } + } + } + } + } + + return { + async *[Symbol.asyncIterator]() { + for (const blob of matchingBlobs) { + yield blob + } + } + } + } + + // Mock containerClient + store.containerClient = { + listBlobsFlat: sinon.stub().callsFake(options => createAsyncIterator(options.prefix)), + getBlockBlobClient: sinon.stub().callsFake(blobName => { + store._uploadBlobName = blobName + return { + upload: store._uploadStub + } + }), + getBlobClient: sinon.stub().callsFake(blobName => { + store._deleteBlobName = blobName + const blobNameLower = blobName.toLowerCase() + return { + delete: store._deleteStub, + download: sinon.stub().callsFake(async () => { + if (blobName.includes('error')) throw new Error('test error') + if (data[blobNameLower]) { + const content = + typeof data[blobNameLower] === 'string' ? data[blobNameLower] : JSON.stringify(data[blobNameLower]) + return { readableStreamBody: createReadableStream(content) } + } + const error = new Error('not found') + error.statusCode = 404 + throw error + }) + } }) } - const store = Store({}) - store.blobService = blobServiceStub + return store } + +function createReadableStream(content) { + const { Readable } = require('stream') + return Readable.from([Buffer.from(content)]) +} diff --git a/test/providers/store/azblobHarvest.js b/test/providers/store/azblobHarvest.js index dcb5a97f0..9a74a66b9 100644 --- a/test/providers/store/azblobHarvest.js +++ b/test/providers/store/azblobHarvest.js @@ -8,6 +8,14 @@ const chai = require('chai') chai.use(deepEqualInAnyOrder) const expect = chai.expect +// Initialize logger for tests +const loggerFactory = require('../../../providers/logging/logger') +try { + loggerFactory({ info: () => {}, error: () => {}, warn: () => {}, debug: () => {} }) +} catch { + // Logger already initialized +} + describe('azblob Harvest store', () => { describe('list', () => { it('should list results', async () => { @@ -21,7 +29,7 @@ describe('azblob Harvest store', () => { metadata: { urn: 'urn:npm:npmjs:-:co:revision:4.6.0:tool:scancode:2.2.1' } } ] - const store = createAzBlobStore(data, true) + const store = createAzBlobStore(data) const result = await store.list({ type: 'npm', @@ -45,7 +53,7 @@ describe('azblob Harvest store', () => { metadata: { urn: 'urn:npm:npmjs:-:JSONStream:revision:1.3.4:tool:scancode:2.2.1' } } ] - const store = createAzBlobStore(data, true) + const store = createAzBlobStore(data) const result = await store.list({ type: 'npm', @@ -86,7 +94,7 @@ describe('azblob Harvest store', () => { metadata: { urn: 'urn:npm:npmjs:-:co:revision:3.6.0:tool:scancode:1.2.1' } } ] - const store = createAzBlobStore(data, true) + const store = createAzBlobStore(data) const result = await store.list({ type: 'npm', @@ -270,14 +278,35 @@ function createEntries(names) { return names.map(name => ({ name })) } -function createAzBlobStore(entries, withMetadata) { - const blobServiceStub = { - listBlobsSegmentedWithPrefix: sinon.stub().callsArgWith(withMetadata ? 4 : 3, null, { entries }), - getBlobToText: sinon.stub().callsArgWith(2, null, '{}'), - createContainerIfNotExists: sinon.stub().callsArgWith(1, null) +function createAzBlobStore(entries) { + // Create async iterator for listBlobsFlat + const createAsyncIterator = (allEntries, prefix) => { + const matchingBlobs = allEntries.filter(e => e.name.startsWith(prefix)) + return { + async *[Symbol.asyncIterator]() { + for (const blob of matchingBlobs) { + yield blob + } + } + } } - blobServiceStub.withFilter = sinon.stub().returns(blobServiceStub) + const store = Store({}) - store.blobService = blobServiceStub + + // Mock containerClient + store.containerClient = { + listBlobsFlat: sinon.stub().callsFake(options => createAsyncIterator(entries, options.prefix || '')), + getBlobClient: sinon.stub().callsFake(() => ({ + download: sinon.stub().resolves({ + readableStreamBody: createReadableStream('{}') + }) + })) + } + return store } + +function createReadableStream(content) { + const { Readable } = require('stream') + return Readable.from([Buffer.from(content)]) +}