diff --git a/package-lock.json b/package-lock.json index f77e53b9..0ca04ba0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1281,64 +1281,88 @@ } }, "node_modules/@google-cloud/paginator": { - "version": "3.0.7", - "dev": true, + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", + "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", "license": "Apache-2.0", "dependencies": { "arrify": "^2.0.0", "extend": "^3.0.2" }, "engines": { - "node": ">=10" + "node": ">=14.0.0" } }, "node_modules/@google-cloud/projectify": { - "version": "3.0.0", - "dev": true, + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", "license": "Apache-2.0", "engines": { - "node": ">=12.0.0" + "node": ">=14.0.0" } }, "node_modules/@google-cloud/promisify": { - "version": "3.0.1", - "dev": true, + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", "license": "Apache-2.0", "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/@google-cloud/storage": { - "version": "6.12.0", - "dev": true, + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.12.0.tgz", + "integrity": "sha512-122Ui67bhnf8MkRnxQAC5lf7wPGkPP5hL3+J5s9HHDw2J9RpaMmnV8iahn+RUn9BH70W6uRe6nMZLXiRaJM/3g==", "license": "Apache-2.0", "dependencies": { - "@google-cloud/paginator": "^3.0.7", - "@google-cloud/projectify": "^3.0.0", - "@google-cloud/promisify": "^3.0.0", + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", "abort-controller": "^3.0.0", "async-retry": "^1.3.3", - "compressible": "^2.0.12", - "duplexify": "^4.0.0", - "ent": "^2.2.0", - "extend": "^3.0.2", - "fast-xml-parser": "^4.2.2", - "gaxios": "^5.0.0", - "google-auth-library": "^8.0.1", + "duplexify": "^4.1.3", + "fast-xml-parser": "^4.3.0", + "gaxios": "^6.0.2", + "google-auth-library": "^9.6.3", + "html-entities": "^2.5.2", "mime": "^3.0.0", - "mime-types": "^2.0.8", "p-limit": "^3.0.1", - "retry-request": "^5.0.0", - "teeny-request": "^8.0.0", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0", "uuid": "^8.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" + } + }, + "node_modules/@google-cloud/storage/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" } }, "node_modules/@google-cloud/storage/node_modules/mime": { "version": "3.0.0", - "dev": true, + "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", + "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", "license": "MIT", "bin": { "mime": "cli.js" @@ -1349,7 +1373,8 @@ }, "node_modules/@google-cloud/storage/node_modules/p-limit": { "version": "3.1.0", - "dev": true, + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", "license": "MIT", "dependencies": { "yocto-queue": "^0.1.0" @@ -2313,8 +2338,8 @@ }, "node_modules/@tootallnate/once": { "version": "2.0.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", "engines": { "node": ">= 10" } @@ -2373,6 +2398,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==", + "license": "MIT" + }, "node_modules/@types/connect": { "version": "3.4.38", "dev": true, @@ -2519,7 +2550,6 @@ }, "node_modules/@types/node": { "version": "20.11.5", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~5.26.4" @@ -2545,6 +2575,32 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "license": "MIT", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/rimraf": { "version": "3.0.2", "dev": true, @@ -2647,6 +2703,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "license": "MIT", @@ -2961,7 +3023,6 @@ }, "node_modules/abort-controller": { "version": "3.0.0", - "dev": true, "license": "MIT", "dependencies": { "event-target-shim": "^5.0.0" @@ -3008,14 +3069,15 @@ } }, "node_modules/agent-base": { - "version": "6.0.2", - "dev": true, + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", "license": "MIT", "dependencies": { - "debug": "4" + "debug": "^4.3.4" }, "engines": { - "node": ">= 6.0.0" + "node": ">= 14" } }, "node_modules/ajv": { @@ -3157,8 +3219,8 @@ }, "node_modules/arrify": { "version": "2.0.1", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", "engines": { "node": ">=8" } @@ -3170,7 +3232,6 @@ }, "node_modules/async-retry": { "version": "1.3.3", - "dev": true, "license": "MIT", "dependencies": { "retry": "0.13.1" @@ -3178,7 +3239,6 @@ }, "node_modules/async-retry/node_modules/retry": { "version": "0.13.1", - "dev": true, "license": "MIT", "engines": { "node": ">= 4" @@ -3186,7 +3246,6 @@ }, "node_modules/asynckit": { "version": "0.4.0", - "dev": true, "license": "MIT" }, "node_modules/available-typed-arrays": { @@ -3206,7 +3265,6 @@ }, "node_modules/base64-js": { "version": "1.5.1", - "dev": true, "funding": [ { "type": "github", @@ -3235,9 +3293,9 @@ } }, "node_modules/bignumber.js": { - "version": "9.1.1", - "dev": true, - "license": "MIT", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", "engines": { "node": "*" } @@ -3309,8 +3367,8 @@ }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", - "dev": true, - "license": "BSD-3-Clause" + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" }, "node_modules/buffer-from": { "version": "1.1.2", @@ -3517,7 +3575,6 @@ }, "node_modules/combined-stream": { "version": "1.0.8", - "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -3531,17 +3588,6 @@ "dev": true, "license": "MIT" }, - "node_modules/compressible": { - "version": "2.0.18", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": ">= 1.43.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/concat-map": { "version": "0.0.1", "license": "MIT" @@ -3748,7 +3794,6 @@ }, "node_modules/delayed-stream": { "version": "1.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -3811,20 +3856,20 @@ } }, "node_modules/duplexify": { - "version": "4.1.2", - "dev": true, - "license": "MIT", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", "dependencies": { "end-of-stream": "^1.4.1", "inherits": "^2.0.3", "readable-stream": "^3.1.1", - "stream-shift": "^1.0.0" + "stream-shift": "^1.0.2" } }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", - "dev": true, - "license": "Apache-2.0", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", "dependencies": { "safe-buffer": "^5.0.1" } @@ -3836,7 +3881,6 @@ }, "node_modules/encoding": { "version": "0.1.13", - "dev": true, "license": "MIT", "optional": true, "peer": true, @@ -3846,7 +3890,6 @@ }, "node_modules/encoding/node_modules/iconv-lite": { "version": "0.6.3", - "dev": true, "license": "MIT", "optional": true, "peer": true, @@ -3859,8 +3902,8 @@ }, "node_modules/end-of-stream": { "version": "1.4.4", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", "dependencies": { "once": "^1.4.0" } @@ -3877,11 +3920,6 @@ "node": ">=8.6" } }, - "node_modules/ent": { - "version": "2.2.0", - "dev": true, - "license": "MIT" - }, "node_modules/error-ex": { "version": "1.3.2", "dev": true, @@ -4310,7 +4348,6 @@ }, "node_modules/event-target-shim": { "version": "5.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -4326,8 +4363,8 @@ }, "node_modules/extend": { "version": "3.0.2", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "node_modules/extendable-error": { "version": "0.1.7", @@ -4382,11 +4419,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fast-text-encoding": { - "version": "1.0.6", - "dev": true, - "license": "Apache-2.0" - }, "node_modules/fast-xml-parser": { "version": "4.2.5", "funding": [ @@ -4584,29 +4616,45 @@ } }, "node_modules/gaxios": { - "version": "5.1.3", - "dev": true, + "version": "6.7.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.0.tgz", + "integrity": "sha512-DSrkyMTfAnAm4ks9Go20QGOcXEyW/NmZhvTYBU2rb4afBB393WIMQPWPEDMl/k8xqiNN9HYq2zao3oWXsdl2Tg==", "license": "Apache-2.0", "dependencies": { "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", + "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", - "node-fetch": "^2.6.9" + "node-fetch": "^2.6.9", + "uuid": "^10.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" } }, "node_modules/gcp-metadata": { - "version": "5.3.0", - "dev": true, + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", "license": "Apache-2.0", "dependencies": { - "gaxios": "^5.0.0", + "gaxios": "^6.0.0", "json-bigint": "^1.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/generic-pool": { @@ -4738,36 +4786,20 @@ } }, "node_modules/google-auth-library": { - "version": "8.9.0", - "dev": true, + "version": "9.13.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.13.0.tgz", + "integrity": "sha512-p9Y03Uzp/Igcs36zAaB0XTSwZ8Y0/tpYiz5KIde5By+H9DCVUSYtDWZu6aFXsWTqENMb8BD/pDT3hR8NVrPkfA==", "license": "Apache-2.0", "dependencies": { - "arrify": "^2.0.0", "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", - "fast-text-encoding": "^1.0.0", - "gaxios": "^5.0.0", - "gcp-metadata": "^5.3.0", - "gtoken": "^6.1.0", - "jws": "^4.0.0", - "lru-cache": "^6.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/google-p12-pem": { - "version": "4.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "node-forge": "^1.3.1" - }, - "bin": { - "gp12-pem": "build/src/bin/gp12-pem.js" + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=14" } }, "node_modules/gopd": { @@ -4795,16 +4827,16 @@ "license": "MIT" }, "node_modules/gtoken": { - "version": "6.1.2", - "dev": true, + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", "license": "MIT", "dependencies": { - "gaxios": "^5.0.1", - "google-p12-pem": "^4.0.0", + "gaxios": "^6.0.0", "jws": "^4.0.0" }, "engines": { - "node": ">=12.0.0" + "node": ">=14.0.0" } }, "node_modules/hard-rejection": { @@ -4925,10 +4957,26 @@ "dev": true, "license": "ISC" }, + "node_modules/html-entities": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", + "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, "node_modules/http-proxy-agent": { "version": "5.0.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", "dependencies": { "@tootallnate/once": "2", "agent-base": "6", @@ -4938,16 +4986,28 @@ "node": ">= 6" } }, + "node_modules/http-proxy-agent/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/https-proxy-agent": { - "version": "5.0.1", - "dev": true, + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", "license": "MIT", "dependencies": { - "agent-base": "6", + "agent-base": "^7.0.2", "debug": "4" }, "engines": { - "node": ">= 6" + "node": ">= 14" } }, "node_modules/human-id": { @@ -5367,8 +5427,8 @@ }, "node_modules/json-bigint": { "version": "1.0.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", "dependencies": { "bignumber.js": "^9.0.0" } @@ -5402,8 +5462,8 @@ }, "node_modules/jwa": { "version": "2.0.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", "dependencies": { "buffer-equal-constant-time": "1.0.1", "ecdsa-sig-formatter": "1.0.11", @@ -5412,8 +5472,8 @@ }, "node_modules/jws": { "version": "4.0.0", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", "dependencies": { "jwa": "^2.0.0", "safe-buffer": "^5.0.1" @@ -5691,7 +5751,6 @@ }, "node_modules/mime-db": { "version": "1.52.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -5699,7 +5758,6 @@ }, "node_modules/mime-types": { "version": "2.1.35", - "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -6010,7 +6068,6 @@ }, "node_modules/node-fetch": { "version": "2.6.12", - "dev": true, "license": "MIT", "dependencies": { "whatwg-url": "^5.0.0" @@ -6027,14 +6084,6 @@ } } }, - "node_modules/node-forge": { - "version": "1.3.1", - "dev": true, - "license": "(BSD-3-Clause OR GPL-2.0)", - "engines": { - "node": ">= 6.13.0" - } - }, "node_modules/node-mocks-http": { "version": "1.14.1", "dev": true, @@ -6664,15 +6713,17 @@ } }, "node_modules/retry-request": { - "version": "5.0.2", - "dev": true, + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", "license": "MIT", "dependencies": { - "debug": "^4.1.1", - "extend": "^3.0.2" + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" } }, "node_modules/reusify": { @@ -6770,7 +6821,7 @@ }, "node_modules/safer-buffer": { "version": "2.1.2", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/semver": { @@ -7085,8 +7136,8 @@ }, "node_modules/stream-events": { "version": "1.0.5", - "dev": true, - "license": "MIT", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", "dependencies": { "stubs": "^3.0.0" } @@ -7113,9 +7164,9 @@ "license": "MIT" }, "node_modules/stream-shift": { - "version": "1.0.1", - "dev": true, - "license": "MIT" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" }, "node_modules/stream-transform": { "version": "2.1.3", @@ -7232,8 +7283,8 @@ }, "node_modules/stubs": { "version": "3.0.0", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" }, "node_modules/superagent": { "version": "8.1.2", @@ -7304,23 +7355,54 @@ } }, "node_modules/teeny-request": { - "version": "8.0.3", - "dev": true, + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", "license": "Apache-2.0", "dependencies": { "http-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.1", + "node-fetch": "^2.6.9", "stream-events": "^1.0.5", "uuid": "^9.0.0" }, "engines": { - "node": ">=12" + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/teeny-request/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" } }, "node_modules/teeny-request/node_modules/uuid": { - "version": "9.0.0", - "dev": true, + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "license": "MIT", "bin": { "uuid": "dist/bin/uuid" @@ -7394,7 +7476,6 @@ }, "node_modules/tr46": { "version": "0.0.3", - "dev": true, "license": "MIT" }, "node_modules/trim-newlines": { @@ -7806,7 +7887,6 @@ }, "node_modules/undici-types": { "version": "5.26.5", - "dev": true, "license": "MIT" }, "node_modules/universalify": { @@ -7867,12 +7947,10 @@ }, "node_modules/webidl-conversions": { "version": "3.0.1", - "dev": true, "license": "BSD-2-Clause" }, "node_modules/whatwg-url": { "version": "5.0.0", - "dev": true, "license": "MIT", "dependencies": { "tr46": "~0.0.3", @@ -8129,7 +8207,7 @@ "debug": "^4.3.4" }, "devDependencies": { - "@google-cloud/storage": "^6.12.0", + "@google-cloud/storage": "^7.12.0", "@tus/server": "^1.7.0", "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", @@ -8143,7 +8221,7 @@ "node": ">=16" }, "peerDependencies": { - "@google-cloud/storage": "*" + "@google-cloud/storage": "^7.12.0" } }, "packages/s3-store": { @@ -8223,6 +8301,7 @@ }, "test": { "dependencies": { + "@google-cloud/storage": "^7.12.0", "@tus/file-store": "^1.4.0", "@tus/gcs-store": "^1.3.0", "@tus/s3-store": "^1.5.0", diff --git a/packages/gcs-store/package.json b/packages/gcs-store/package.json index db2b1ef7..465d972e 100644 --- a/packages/gcs-store/package.json +++ b/packages/gcs-store/package.json @@ -24,7 +24,7 @@ "debug": "^4.3.4" }, "devDependencies": { - "@google-cloud/storage": "^6.12.0", + "@google-cloud/storage": "^7.12.0", "@tus/server": "^1.7.0", "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", @@ -35,7 +35,7 @@ "should": "^13.2.3" }, "peerDependencies": { - "@google-cloud/storage": "*" + "@google-cloud/storage": "^7.12.0" }, "engines": { "node": ">=16" diff --git a/packages/gcs-store/src/index.ts b/packages/gcs-store/src/index.ts index 9ad9814f..f82781e6 100644 --- a/packages/gcs-store/src/index.ts +++ b/packages/gcs-store/src/index.ts @@ -9,6 +9,8 @@ const log = debug('tus-node-server:stores:gcsstore') type Options = {bucket: Bucket} +export {GCSLocker} from './locker/GCSLocker' + export class GCSStore extends DataStore { bucket: Bucket @@ -39,11 +41,7 @@ export class GCSStore extends DataStore { metadata: { metadata: { tus_version: TUS_RESUMABLE, - size: file.size, - sizeIsDeferred: `${file.sizeIsDeferred}`, - offset: file.offset, - metadata: JSON.stringify(file.metadata), - storage: JSON.stringify(file.storage), + ...this.#stringifyUploadKeys(file), }, }, } @@ -77,15 +75,14 @@ export class GCSStore extends DataStore { return new Promise((resolve, reject) => { const file = this.bucket.file(id) const destination = upload.offset === 0 ? file : this.bucket.file(`${id}_patch`) + + upload.offset = offset + const options = { metadata: { metadata: { tus_version: TUS_RESUMABLE, - size: upload.size, - sizeIsDeferred: `${upload.sizeIsDeferred}`, - offset, - metadata: JSON.stringify(upload.metadata), - storage: JSON.stringify(upload.storage), + ...this.#stringifyUploadKeys(upload), }, }, } @@ -151,7 +148,7 @@ export class GCSStore extends DataStore { return resolve( new Upload({ id, - size: size ? Number.parseInt(size, 10) : size, + size: size ? Number.parseInt(size, 10) : undefined, offset: Number.parseInt(metadata.size, 10), // `size` is set by GCS metadata: meta ? JSON.parse(meta) : undefined, storage: {type: 'gcs', path: id, bucket: this.bucket.name}, @@ -166,6 +163,19 @@ export class GCSStore extends DataStore { upload.size = upload_length - await this.bucket.file(id).setMetadata({metadata: upload}) + await this.bucket.file(id).setMetadata({metadata: this.#stringifyUploadKeys(upload)}) + } + + /** + * Convert the Upload object to a format that can be stored in GCS metadata. + */ + #stringifyUploadKeys(upload: Upload) { + return { + size: upload.size ?? null, + sizeIsDeferred: `${upload.sizeIsDeferred}`, + offset: upload.offset, + metadata: JSON.stringify(upload.metadata), + storage: JSON.stringify(upload.storage), + } } } diff --git a/packages/gcs-store/src/locker/GCSLock.ts b/packages/gcs-store/src/locker/GCSLock.ts new file mode 100644 index 00000000..43c91297 --- /dev/null +++ b/packages/gcs-store/src/locker/GCSLock.ts @@ -0,0 +1,118 @@ +import {RequestRelease} from '@tus/utils' +import {Bucket} from '@google-cloud/storage' +import GCSLockFile, {GCSLockFileMetadata} from './GCSLockFile' + +/** + * Handles interaction with a lock. + */ +export default class GCSLock { + protected resourceId: string + protected file: GCSLockFile + protected ttl: number + protected watchInterval: number + protected watcher: NodeJS.Timeout | undefined + + constructor( + resourceId: string, + lockBucket: Bucket, + ttl: number, + watchInterval: number + ) { + this.resourceId = resourceId + this.file = new GCSLockFile(lockBucket, `${resourceId}.lock`) + this.ttl = ttl + this.watchInterval = watchInterval + } + + /** + * Try to create the lockfile and start the watcher. If lock is already taken, requests for release and returns FALSE. + */ + public async take(cancelHandler: RequestRelease): Promise { + try { + //Try to create lock file + const exp = Date.now() + this.ttl + await this.file.create(exp) + + //Lock acquired, start watcher + this.startWatcher(cancelHandler) + + return true + } catch (err) { + //Probably lock is already taken + const isHealthy = await this.insureHealth() + + if (!isHealthy) { + //Lock is not healthy, restart the process + return await this.take(cancelHandler) + } else { + //Lock is still healthy, request release + await this.file.requestRelease() + + return false + } + } + } + + /** + * Release the lock - clear watcher and delete the file. + */ + public async release() { + //Clear watcher + clearInterval(this.watcher) + + //Delete the lock file + this.file.deleteOwn() + } + + /** + * Check if the lock is healthy, delete if not. + * Returns TRUE if the lock is healthy. + */ + protected async insureHealth() { + try { + const meta = await this.file.getMeta() + + if (this.hasExpired(meta)) { + //TTL expired, delete unhealthy lock + await this.file.deleteUnhealthy(meta.metageneration as number) + + return false + } + } catch (err) { + //Probably lock does not exist (anymore) + return false + } + + return true + } + + /** + * Start watching the lock file - keep it healthy and handle release requests. + */ + protected startWatcher(cancelHandler: RequestRelease) { + this.watcher = setInterval(() => { + const handleError = () => { + //Probably the watched lock is freed, terminate watcher + clearInterval(this.watcher) + } + + this.file.checkOwnReleaseRequest().then((shouldRelease) => { + if (shouldRelease) { + cancelHandler() + } + + //Update TTL to keep the lock healthy + const exp = Date.now() + this.ttl + this.file.refreshOwn(exp).catch(handleError) + }, handleError) + }, this.watchInterval) + } + + /** + * Compare lock expiration timestamp with the current time. + */ + protected hasExpired(meta: GCSLockFileMetadata) { + const expDate = Date.parse(meta.exp + '') + return !expDate || expDate < Date.now() + } +} diff --git a/packages/gcs-store/src/locker/GCSLockFile.ts b/packages/gcs-store/src/locker/GCSLockFile.ts new file mode 100644 index 00000000..e5a46d45 --- /dev/null +++ b/packages/gcs-store/src/locker/GCSLockFile.ts @@ -0,0 +1,128 @@ +import {Bucket, File} from '@google-cloud/storage' +import type {FileMetadata} from '@google-cloud/storage' + +export type GCSLockFileMetadata = FileMetadata & { + /** + * The lock file expires at this time (in ms) if its not refreshed. + */ + exp: number +} + +type MetaGeneration = string | number | undefined + +/** + * Handles communication with GCS. + */ +export default class GCSLockFile { + /** + * Name of the file in the bucket. + */ + protected name: string + /** + * GCS File instance for the lock. + */ + protected lockFile: File + /** + * GCS File instance for release request. + */ + protected releaseFile: File + /** + * The last known metageneration of the file. If it does not match the GCS metageneration, this lockfile has been deleted and another instance has already created a new one. + */ + protected currentMetaGeneration: MetaGeneration + + constructor(bucket: Bucket, name: string) { + this.name = name + this.lockFile = bucket.file(name) + this.releaseFile = bucket.file(name + '.release') + } + /** + * Create the lockfile with the specified exp time. Throws if the file already exists + */ + public async create(exp: number) { + const metadata: GCSLockFileMetadata = { + exp, + cacheControl: 'no-store', + } + + await this.lockFile.save('', { + preconditionOpts: {ifGenerationMatch: 0}, + metadata, + }) + this.currentMetaGeneration = 0 + } + + /** + * Fetch metadata of the lock file. + */ + public async getMeta() { + return (await this.lockFile.getMetadata())[0] as GCSLockFileMetadata + } + + /** + * Refresh our own lockfile. Throws if it does not exist or the file is modified by another instance. + */ + public async refreshOwn(exp: number) { + const metadata: GCSLockFileMetadata = { + exp, + } + const res = await this.lockFile.setMetadata(metadata, { + ifMetaGenerationMatch: this.currentMetaGeneration, + }) + this.currentMetaGeneration = res[0].metageneration + } + /** + * Check if a release request has been submitted to our own lockfile. Throws if it does not exist or the file is modified by another instance. + */ + public async checkOwnReleaseRequest() { + const meta = await this.getMeta() + if (meta.metageneration !== this.currentMetaGeneration) { + throw new Error('This lockfile has been already taken by another instance.') + } + + const releaseRequestExists = (await this.releaseFile.exists())[0] + return releaseRequestExists + } + + /** + * Delete our own lockfile if it still exists. + */ + public async deleteOwn() { + try { + await this.deleteReleaseRequest() + await this.lockFile.delete({ifGenerationMatch: this.currentMetaGeneration}) + } catch (err) { + //Probably already deleted, no need to report + } + } + + /** + * Request releasing the lock from another instance. As metadata edits are only prohibited for the owner (so it can keep track of metageneration), we write to a separate file. + */ + public async requestRelease() { + try { + await this.releaseFile.save('', { + preconditionOpts: {ifGenerationMatch: 0}, + }) + } catch (err) { + //Release file already created, no need to report + } + } + + /** + * Delete the unhealthy file of a previous lock. + */ + public async deleteUnhealthy(metaGeneration: number) { + await this.deleteReleaseRequest() + await this.lockFile.delete({ifMetagenerationMatch: metaGeneration}) + } + + /** + * Delete release request file of the lock if exists. + */ + protected async deleteReleaseRequest() { + try { + await this.releaseFile.delete() + } catch (err) {} + } +} diff --git a/packages/gcs-store/src/locker/GCSLocker.ts b/packages/gcs-store/src/locker/GCSLocker.ts new file mode 100644 index 00000000..cdf57470 --- /dev/null +++ b/packages/gcs-store/src/locker/GCSLocker.ts @@ -0,0 +1,146 @@ +import EventEmitter from 'node:events' +import {setTimeout, setImmediate} from 'node:timers/promises' + +import {ERRORS, Lock, Locker, RequestRelease} from '@tus/utils' +import {Bucket} from '@google-cloud/storage' +import GCSLock from './GCSLock' + +/** + * Google Cloud Storage implementation of the Locker mechanism with support for distribution. + * For general information regarding Locker, see MemoryLocker. + * + * Locking is based on separate .lock files created in a GCS bucket (presumably the same as the upload destination, but not necessarily). Concurrency control is ensured by metageneration preconditions. Release mechanism is based on separate .release files. After a lock file is created, we regularly check if another process requested releasing the lock (by creating the release file). To avoid resources being locked forever, each lock's metadata is regularly updated, this way we can make sure the locker process haven't crashed. + * + * Lock file health - possible states of a lock file: + * - non-existing (not locked) + * - healthy (locked) + * - requested to be released (locked, but should be released soon) + * - expired (not locked) + * + * Acquiring a lock: + * - If the lock file does not exist yet, create one with an expiration time and start watching it (see below) + * - If the lock file already exists + * -- If it has expired, delete it and restart the process + * -- If it is active, request releasing the resource by creatin the .release file, then retry locking with an exponential backoff + * + * Releasing a lock: + * Stop the watcher and delete the lock/release files. + * + * Watching a lock (performed in every `watchInterval` ms): + * - If the lock file does not exist anymore, or its created by a different process, stop the watcher + * - If the lock file still exists + * -- Update its expiration time + * -- If a release file exists, call the cancel handler + * + * The implementation is based on 'A robust distributed locking algorithm based on Google Cloud Storage' by Hongli Lai (https://www.joyfulbikeshedding.com/blog/2021-05-19-robust-distributed-locking-algorithm-based-on-google-cloud-storage.html). + */ + +export interface GCSLockerOptions { + /** + * The bucket where the lock file will be created. No need to match the upload destination bucket. + */ + bucket: Bucket + /** + * Maximum time (in milliseconds) to wait for an already existing lock to be released, else deny acquiring the lock. + */ + acquireLockTimeout?: number + /** + * Maximum amount of time (in milliseconds) a lock is considered healthy without being refreshed. When refreshed, expiration will be current time + TTL. If a process unexpectedly ends, lock expiration won't be updated every `watchInterval`, and it will become unhealthy. Must be set according to `watchInterval`, and must be more than it (else would expire before being refreshed). Larger value results more waiting time before releasing an unhealthy lock. + */ + lockTTL?: number + /** + * The amount of time (in milliseconds) to wait between lock file health checks. Must be set according to `lockTTL`, and must be less than `acquireLockTimeout`. Larger value results less queries to GCS. + */ + watchInterval?: number +} + +export class GCSLocker implements Locker { + events: EventEmitter + bucket: Bucket + lockTimeout: number + lockTTL: number + watchInterval: number + + constructor(options: GCSLockerOptions) { + this.events = new EventEmitter() + this.bucket = options.bucket + this.lockTimeout = options.acquireLockTimeout ?? 1000 * 30 + this.lockTTL = options.lockTTL ?? 1000 * 12 + this.watchInterval = options.watchInterval ?? 1000 * 10 + + if (this.watchInterval > this.lockTimeout) { + throw new Error('watchInterval must be less than acquireLockTimeout') + } + } + + newLock(id: string) { + return new GCSLockHandler(id, this) + } +} + +class GCSLockHandler implements Lock { + private gcsLock: GCSLock + + constructor(private id: string, private locker: GCSLocker) { + this.gcsLock = new GCSLock( + this.id, + this.locker.bucket, + this.locker.lockTTL, + this.locker.watchInterval + ) + } + + async lock(requestRelease: RequestRelease): Promise { + const abortController = new AbortController() + + const lock = await Promise.race([ + this.waitForLockTimeoutOrAbort(abortController.signal), + this.acquireLock(requestRelease, abortController.signal), + ]) + + abortController.abort() + + if (!lock) { + throw ERRORS.ERR_LOCK_TIMEOUT + } + } + + async unlock(): Promise { + await this.gcsLock.release() + } + + protected async acquireLock( + cancelHandler: RequestRelease, + signal: AbortSignal, + attempt = 0 + ): Promise { + if (signal.aborted) { + return false + } + + const acquired = await this.gcsLock.take(cancelHandler) + + if (!acquired) { + if (attempt > 0) { + await setTimeout((attempt * this.locker.watchInterval) / 3) + } else { + await setImmediate() + } + return await this.acquireLock(cancelHandler, signal, attempt + 1) + } + + return true + } + + protected async waitForLockTimeoutOrAbort(signal: AbortSignal) { + try { + await setTimeout(this.locker.lockTimeout, undefined, {signal}) + return false + } catch (err) { + if (err.name === 'AbortError') { + return false + } + throw err + } + } +} diff --git a/packages/utils/src/models/Upload.ts b/packages/utils/src/models/Upload.ts index 49a6447c..d5a7c389 100644 --- a/packages/utils/src/models/Upload.ts +++ b/packages/utils/src/models/Upload.ts @@ -34,6 +34,6 @@ export class Upload { } get sizeIsDeferred(): boolean { - return this.size === undefined + return this.size === undefined || this.size === null } } diff --git a/test/package.json b/test/package.json index d0b31b60..bff25578 100644 --- a/test/package.json +++ b/test/package.json @@ -10,6 +10,7 @@ "./stores.test": "./dist/stores.test.js" }, "dependencies": { + "@google-cloud/storage": "^7.12.0", "@tus/file-store": "^1.4.0", "@tus/gcs-store": "^1.3.0", "@tus/s3-store": "^1.5.0",