From 41472ba17bdb3628bbdd598b68657d0cd9952040 Mon Sep 17 00:00:00 2001 From: Marcel Arndt Date: Wed, 20 Aug 2025 10:36:20 +0200 Subject: [PATCH] feat: add schichtplanung sync --- .../data-connector/docker-compose.yaml | 7 + .../data-hub/data-connector/package-lock.json | 318 +++++++++++++++++- service/data-hub/data-connector/package.json | 2 + .../data-connector/src/app.controller.ts | 7 +- .../data-hub/data-connector/src/app.module.ts | 6 + .../data-connector/src/app.service.ts | 8 +- .../src/core/config/config.service.ts | 6 + .../data-access-extractors.module.ts | 9 +- .../dispo-live-api-extractors.service.ts | 92 +++++ .../dyflexis-extractors.service.ts | 2 +- .../feat-importers/feature-importer.module.ts | 14 +- .../feat-importers/importer.service.ts | 24 +- .../processors/schichtplanung.consumer.ts | 51 +++ .../feat-importers/producer/schichtplanung.ts | 174 ++++++++++ .../modules/feat-etl/feat-importers/tokens.ts | 1 + .../workflows/tools/delete-schichtplanung.ts | 67 ++++ service/data-hub/workflows/tools/deno.json | 8 + service/data-hub/workflows/tools/deno.lock | 29 ++ service/docker-compose.deploy.yml | 5 + 19 files changed, 799 insertions(+), 31 deletions(-) create mode 100644 service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dispo-live-api-extractors.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-etl/feat-importers/processors/schichtplanung.consumer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-etl/feat-importers/producer/schichtplanung.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-etl/feat-importers/tokens.ts create mode 100644 service/data-hub/workflows/tools/delete-schichtplanung.ts create mode 100644 service/data-hub/workflows/tools/deno.json create mode 100644 service/data-hub/workflows/tools/deno.lock diff --git a/service/data-hub/data-connector/docker-compose.yaml b/service/data-hub/data-connector/docker-compose.yaml index 296c4fc..7b5c1b6 100644 --- a/service/data-hub/data-connector/docker-compose.yaml +++ b/service/data-hub/data-connector/docker-compose.yaml @@ -41,6 +41,13 @@ services: networks: - avic + avicenna-cache: + image: valkey/valkey:8.1.2-alpine + ports: + - 6379:6379 + networks: + - avic + volumes: pgdata: diff --git a/service/data-hub/data-connector/package-lock.json b/service/data-hub/data-connector/package-lock.json index 7a392d4..0ade1f0 100644 --- a/service/data-hub/data-connector/package-lock.json +++ b/service/data-hub/data-connector/package-lock.json @@ -13,6 +13,7 @@ "@faker-js/faker": "^9.6.0", "@nestjs/apollo": "^13.0.4", "@nestjs/axios": "^4.0.0", + "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.0.1", "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.15", @@ -25,6 +26,7 @@ "@opentelemetry/exporter-trace-otlp-http": "^0.202.0", "@opentelemetry/sdk-node": "^0.202.0", "@prisma/client": "^6.6.0", + "bullmq": "^5.56.0", "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", "graphql": "^16.10.0", @@ -311,6 +313,25 @@ "@types/serve-static": "*" } }, + "node_modules/@apollo/server/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@apollo/usage-reporting-protobuf": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@apollo/usage-reporting-protobuf/-/usage-reporting-protobuf-4.1.1.tgz", @@ -2061,6 +2082,11 @@ } } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -2688,6 +2714,78 @@ "node": ">=16" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nestjs/apollo": { "version": "13.0.4", "resolved": "https://registry.npmjs.org/@nestjs/apollo/-/apollo-13.0.4.tgz", @@ -2730,6 +2828,32 @@ "rxjs": "^7.0.0" } }, + "node_modules/@nestjs/bull-shared": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-11.0.2.tgz", + "integrity": "sha512-dFlttJvBqIFD6M8JVFbkrR4Feb39OTAJPJpFVILU50NOJCM4qziRw3dSNG84Q3v+7/M6xUGMFdZRRGvBBKxoSA==", + "dependencies": { + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0" + } + }, + "node_modules/@nestjs/bullmq": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-11.0.2.tgz", + "integrity": "sha512-Lq6lGpKkETsm0RDcUktlzsthFoE3A5QTMp2FwPi1eztKqKD6/90KS1TcnC9CJFzjpUaYnQzIMrlNs55e+/wsHA==", + "dependencies": { + "@nestjs/bull-shared": "^11.0.2", + "tslib": "2.8.1" + }, + "peerDependencies": { + "@nestjs/common": "^10.0.0 || ^11.0.0", + "@nestjs/core": "^10.0.0 || ^11.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "11.0.6", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-11.0.6.tgz", @@ -5085,6 +5209,25 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/@prisma/internals/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/@prisma/internals/node_modules/resolve": { "version": "1.22.4", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz", @@ -6716,6 +6859,20 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/bullmq": { + "version": "5.56.0", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.56.0.tgz", + "integrity": "sha512-j5ct2tdc9M8PKcjhJw+euO24BsO1wXBAkNGXYI1R1qvh7FvRldZ5wtLixLWqQ4/crafj0Vrwi+y1kXFXMwBJFA==", + "dependencies": { + "cron-parser": "^4.9.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.11.2", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -7069,6 +7226,14 @@ "node": ">=0.8" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -7364,6 +7529,17 @@ "luxon": "~3.5.0" } }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-inspect": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cross-inspect/-/cross-inspect-1.0.1.tgz", @@ -7504,6 +7680,14 @@ "node": ">=0.4.0" } }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -7521,6 +7705,15 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "optional": true, + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -8722,6 +8915,25 @@ "node": ">=14" } }, + "node_modules/gaxios/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/gcp-metadata": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.1.tgz", @@ -9282,6 +9494,29 @@ "node": ">=10" } }, + "node_modules/ioredis": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz", + "integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -10472,6 +10707,11 @@ "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isplainobject": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", @@ -10771,6 +11011,35 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/msgpackr": { + "version": "1.11.4", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.4.tgz", + "integrity": "sha512-uaff7RG9VIC4jacFW9xzL3jc0iM32DNHe4jYVycBcjUePT/Klnfj7pqtWJt9khvDFizmjN2TlYniYmSS2LIaZg==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.5-lts.2", "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.2.tgz", @@ -10839,23 +11108,18 @@ "lodash": "^4.17.21" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "optional": true, "dependencies": { - "whatwg-url": "^5.0.0" + "detect-libc": "^2.0.1" }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" } }, "node_modules/node-int64": { @@ -11956,6 +12220,25 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -12594,6 +12877,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", diff --git a/service/data-hub/data-connector/package.json b/service/data-hub/data-connector/package.json index 3ab7001..7c08e30 100644 --- a/service/data-hub/data-connector/package.json +++ b/service/data-hub/data-connector/package.json @@ -27,6 +27,7 @@ "@faker-js/faker": "^9.6.0", "@nestjs/apollo": "^13.0.4", "@nestjs/axios": "^4.0.0", + "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.0.1", "@nestjs/config": "^4.0.2", "@nestjs/core": "^11.0.15", @@ -39,6 +40,7 @@ "@opentelemetry/exporter-trace-otlp-http": "^0.202.0", "@opentelemetry/sdk-node": "^0.202.0", "@prisma/client": "^6.6.0", + "bullmq": "^5.56.0", "date-fns": "^4.1.0", "date-fns-tz": "^3.2.0", "graphql": "^16.10.0", diff --git a/service/data-hub/data-connector/src/app.controller.ts b/service/data-hub/data-connector/src/app.controller.ts index 9c31464..698148a 100644 --- a/service/data-hub/data-connector/src/app.controller.ts +++ b/service/data-hub/data-connector/src/app.controller.ts @@ -30,7 +30,7 @@ export class AppController { start: new Date(from), end: new Date(to), }; - return this.importer.runImport(interval); + return this.importer.runImportForRange(interval); } @Get('tickets') @@ -63,6 +63,11 @@ export class AppController { .then(() => this.anomaliesService.validateAnomalies()); } + @Get('schichtplanung') + async syncSchichtplanung(@Query('day') day: string) { + return await this.importer.runSchichtplanungSync(new Date(day)); + } + //! Used to create check 2 report @Get('tour-calculation') async tourCalculation(@Query('from') from: string, @Query('to') to: string) { diff --git a/service/data-hub/data-connector/src/app.module.ts b/service/data-hub/data-connector/src/app.module.ts index 20ae048..d6d7527 100644 --- a/service/data-hub/data-connector/src/app.module.ts +++ b/service/data-hub/data-connector/src/app.module.ts @@ -56,6 +56,12 @@ import { AppHomeModule } from './modules/app-home/app-home.module'; }), }), }), + BullModule.forRoot({ + connection: { + host: 'localhost', + port: 6379, + }, + }), // DatabaseModule.forRoot({ // dialect: new PostgresDialect({ // pool: new Pool({ diff --git a/service/data-hub/data-connector/src/app.service.ts b/service/data-hub/data-connector/src/app.service.ts index 5f24079..b7ae7f0 100644 --- a/service/data-hub/data-connector/src/app.service.ts +++ b/service/data-hub/data-connector/src/app.service.ts @@ -34,7 +34,7 @@ export class AppService { end: endOfDay(addDays(now, 7)), }; this.importer - .runImport(fetchInterval) + .runImportForRange(fetchInterval) .pipe( take(1), tap(() => @@ -84,7 +84,7 @@ export class AppService { this.logger.log(`Timeout "${timeoutName}"`); from([interval]) .pipe( - concatMap((interval) => this.importer.runImport(interval)), + concatMap((interval) => this.importer.runImportForRange(interval)), switchMap(() => from(this.ticketService.createTickets(interval))), ) .subscribe(); @@ -107,7 +107,7 @@ export class AppService { from(intervals).pipe( concatMap((interval, index) => this.importer - .runImport(interval) + .runImportForRange(interval) .pipe( tap(() => this.logger.log( @@ -141,7 +141,7 @@ export class AppService { .pipe( concatMap((interval, index) => this.importer - .runImport(interval) + .runImportForRange(interval) .pipe( tap(() => this.logger.log( diff --git a/service/data-hub/data-connector/src/core/config/config.service.ts b/service/data-hub/data-connector/src/core/config/config.service.ts index cf0c87a..ca5df0c 100644 --- a/service/data-hub/data-connector/src/core/config/config.service.ts +++ b/service/data-hub/data-connector/src/core/config/config.service.ts @@ -6,6 +6,12 @@ export class ConfigService extends NestConfigService { dispoliveBaseUrl(): string { return this.get('DISPO_LIVE_BASE_URL'); } + dispoliveApiKey(): string { + return this.get('DISPO_LIVE_API_KEY'); + } + dispoliveApiVersion(): string { + return this.get('DISPO_LIVE_API_VERSION') ?? 'v1.0.1'; + } dispoliveUsername(): string { return this.get('DISPO_LIVE_USERNAME'); } diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/data-access-extractors.module.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/data-access-extractors.module.ts index 7119790..7548d1d 100644 --- a/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/data-access-extractors.module.ts +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/data-access-extractors.module.ts @@ -5,15 +5,22 @@ import { DispoLiveExtractorsService } from './dispo-live-extractors.service'; import { DyflexisExtractorsService } from './dyflexis-extractors.service'; import { ExtractorsService } from './extractors.service'; import { SourceRepository } from './source.repository'; +import { DispoLiveApiExtractorsService } from './dispo-live-api-extractors.service'; @Module({ imports: [PrismaModule, HttpModule], providers: [ ExtractorsService, DispoLiveExtractorsService, + DispoLiveApiExtractorsService, DyflexisExtractorsService, SourceRepository, ], - exports: [ExtractorsService], + exports: [ + ExtractorsService, + DispoLiveApiExtractorsService, + DispoLiveExtractorsService, + DyflexisExtractorsService + ], }) export class DataAccessExtractorsModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dispo-live-api-extractors.service.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dispo-live-api-extractors.service.ts new file mode 100644 index 0000000..f95c0fd --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dispo-live-api-extractors.service.ts @@ -0,0 +1,92 @@ +import { HttpService } from '@nestjs/axios'; +import { Injectable } from '@nestjs/common'; +import { format, Interval } from 'date-fns'; +import { ConfigService } from 'src/core/config/config.service'; +import { + FahrzeugItemDto, + FahrzeugplanungItemDto, + TourSummaryDto, +} from './source.interface'; + +@Injectable() +export class DispoLiveApiExtractorsService { + constructor( + private readonly http: HttpService, + private readonly config: ConfigService, + ) {} + + toursFindByRange(interval: Interval) { + const endpoint = 'custom/open-api/fahrberichte/findByRange'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}/${format(new Date(interval.start), 'yyyy-MM-dd')}_${format(new Date(interval.end), 'yyyy-MM-dd')}`; + + return this.http.get(url, { + headers: { + Authorization: `Bearer ${this.config.dispoliveApiKey()}`, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + }, + }); + } + + fahrzeuge() { + const endpoint = 'custom/open-api/fahrzeuge/all'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}`; + + return this.http.get(url, { + headers: { + Authorization: `Bearer ${this.config.dispoliveApiKey()}`, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + }, + }); + } + + fahrzeugplanungFindByRange(interval: Interval) { + const endpoint = 'custom/open-api/fahrzeugplanung/findByRange'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}/${format(new Date(interval.start), 'yyyy-MM-dd')}_${format(new Date(interval.end), 'yyyy-MM-dd')}`; + + return this.http.get(url, { + headers: { + Authorization: `Bearer ${this.config.dispoliveApiKey()}`, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + }, + }); + } + + fahrzeugplanungAdd(item: FahrzeugplanungItemDto) { + const endpoint = 'custom/open-api/fahrzeugplanung/add'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}`; + const authToken = `Bearer ${this.config.dispoliveApiKey()}`; + + return this.http.post(url, item, { + headers: { + Authorization: authToken, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + 'Content-Type': 'application/json', + }, + }); + } + + fahrzeugplanungUpdate(item: FahrzeugplanungItemDto) { + const endpoint = 'custom/open-api/fahrzeugplanung/update'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}`; + + return this.http.post(url, item, { + headers: { + Authorization: `Bearer ${this.config.dispoliveApiKey()}`, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + 'Content-Type': 'application/json', + }, + }); + } + + fahrzeugplanungDelete(id: string) { + const endpoint = 'custom/open-api/fahrzeugplanung/delete'; + const url = `${this.config.dispoliveBaseUrl()}/${endpoint}/${id}`; + + return this.http.get(url, { + headers: { + Authorization: `Bearer ${this.config.dispoliveApiKey()}`, + 'dispolive-api-version': this.config.dispoliveApiVersion(), + }, + }); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dyflexis-extractors.service.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dyflexis-extractors.service.ts index 1edab87..278e817 100644 --- a/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dyflexis-extractors.service.ts +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-extractors/dyflexis-extractors.service.ts @@ -271,7 +271,7 @@ export class DyflexisExtractorsService { ); } - private fetchPlannedTimeForMonth( + fetchPlannedTimeForMonth( month: number, year: number, ): Observable { diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/feature-importer.module.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/feature-importer.module.ts index 2461c42..addd6b2 100644 --- a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/feature-importer.module.ts +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/feature-importer.module.ts @@ -1,17 +1,27 @@ +import { BullModule } from '@nestjs/bullmq'; import { Module } from '@nestjs/common'; - +import { FeatBusinessObjectsModule } from 'src/modules/feat-business-objects/feat-business-objects.module'; import { DataAccessExtractorsModule } from '../feat-extractors/data-access-extractors.module'; import { DataAccessLoadersModule } from '../feat-loaders/data-access-loaders.module'; import { DataAccessTransformersModule } from '../feat-transformers/data-access-transformers.module'; import { ImporterService } from './importer.service'; +import { SchichtplanungConsumer } from './processors/schichtplanung.consumer'; +import { SchichtplanungSynchronizer } from './producer/schichtplanung'; +import { SCHICHTPLANUNG_QUEUE } from './tokens'; @Module({ imports: [ + FeatBusinessObjectsModule, DataAccessExtractorsModule, DataAccessTransformersModule, DataAccessLoadersModule, + BullModule.registerQueue({ name: SCHICHTPLANUNG_QUEUE }), + ], + providers: [ + ImporterService, + SchichtplanungSynchronizer, + SchichtplanungConsumer, ], - providers: [ImporterService], exports: [ImporterService], }) export class FeatureImporterModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/importer.service.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/importer.service.ts index c01b7e7..9b69ab3 100644 --- a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/importer.service.ts +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/importer.service.ts @@ -11,6 +11,7 @@ import { ETLEventTypes } from './events/event-types.enum'; import { ExtractionDoneEvent } from './events/extraction-done.event'; import { LoadingDoneEvent } from './events/loading-done.event'; import { TransformationDoneEvent } from './events/transformation-done.event'; +import { SchichtplanungSynchronizer } from './producer/schichtplanung'; @Injectable() export class ImporterService { @@ -20,13 +21,19 @@ export class ImporterService { private readonly transformersService: TransformersService, private readonly loadersService: LoadersService, private readonly eventEmitter: EventEmitter2, + private readonly schichtplanungSynchronizer: SchichtplanungSynchronizer, ) {} - runImport(interval: Interval) { + + runSchichtplanungSync(day: Date) { + return this.schichtplanungSynchronizer.syncSchichtplanung(day); + } + + runImportForRange(interval: Interval) { const startTS = new Date().getTime(); const jobId = randomUUID(); return from(this.extractorsService.clearSource()).pipe( - switchMap(() => this.runExtractors(interval, jobId)), + switchMap(() => this.runExtractorsForRange(interval, jobId)), tap(() => this.logger.log( `Extraction of ${jobId} took ${new Date().getTime() - startTS}ms`, @@ -34,7 +41,7 @@ export class ImporterService { ), switchMap(() => from(this.transformersService.clearStaging()).pipe( - switchMap(() => this.runTransformers(jobId)), + switchMap(() => this.runTransformersForRange(jobId)), ), ), tap(() => @@ -42,7 +49,7 @@ export class ImporterService { `Transformation of ${jobId} took ${new Date().getTime() - startTS}ms`, ), ), - switchMap(() => this.runLoaders(jobId)), + switchMap(() => this.runLoadersForRange(jobId)), tap(() => this.logger.log( `Loading of ${jobId} took ${new Date().getTime() - startTS}ms`, @@ -51,7 +58,10 @@ export class ImporterService { ); } - private runExtractors(interval: Interval, jobId: string): Observable { + private runExtractorsForRange( + interval: Interval, + jobId: string, + ): Observable { return combineLatest([ this.extractorsService.extractAttendanceRegistrations(interval, jobId), this.extractorsService.extractEmployees(jobId), @@ -72,7 +82,7 @@ export class ImporterService { ); } - private runTransformers(jobId: string): Observable { + private runTransformersForRange(jobId: string): Observable { return combineLatest([ this.transformersService.transformAttendanceRegistrations(jobId), this.transformersService.transformDyflexisEmployees(jobId), @@ -94,7 +104,7 @@ export class ImporterService { ); } - private runLoaders(jobId: string): Observable { + private runLoadersForRange(jobId: string): Observable { return this.loadersService.loadTours().pipe( switchMap(() => this.loadersService.loadPatients()), switchMap(() => this.loadersService.loadDyflexisEmployees()), diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/processors/schichtplanung.consumer.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/processors/schichtplanung.consumer.ts new file mode 100644 index 0000000..6bfe539 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/processors/schichtplanung.consumer.ts @@ -0,0 +1,51 @@ +import { Processor, WorkerHost } from '@nestjs/bullmq'; +import { Logger } from '@nestjs/common'; +import { Job } from 'bullmq'; +import { catchError, lastValueFrom } from 'rxjs'; +import { DispoLiveApiExtractorsService } from '../../feat-extractors/dispo-live-api-extractors.service'; +import { SCHICHTPLANUNG_QUEUE } from '../tokens'; + +@Processor(SCHICHTPLANUNG_QUEUE, { + limiter: { + duration: 300, + max: 1, + }, +}) +export class SchichtplanungConsumer extends WorkerHost { + private readonly logger = new Logger(SchichtplanungConsumer.name); + + constructor(private readonly dlApiService: DispoLiveApiExtractorsService) { + super(); + } + + async process(job: Job, token?: string): Promise { + try { + switch (job.name) { + case 'Create': + return await lastValueFrom( + this.dlApiService.fahrzeugplanungAdd(job.data), + ); + case 'Update': + return await lastValueFrom( + this.dlApiService.fahrzeugplanungUpdate(job.data).pipe( + catchError((err) => { + throw err?.response?.data ?? err; + }), + ), + ); + case 'Delete': + return await lastValueFrom( + this.dlApiService.fahrzeugplanungDelete(job.data), + ); + default: + throw new Error( + `[${SchichtplanungConsumer.name}]: job name not implemented ${job.name}`, + ); + } + } catch (error) { + this.logger.error( + `job processing failed, ${typeof error === 'object' ? JSON.stringify(error) : error}, ${JSON.stringify(job.data)}`, + ); + } + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/producer/schichtplanung.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/producer/schichtplanung.ts new file mode 100644 index 0000000..3754b28 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/producer/schichtplanung.ts @@ -0,0 +1,174 @@ +import { InjectQueue } from '@nestjs/bullmq'; +import { Injectable } from '@nestjs/common'; +import { Queue } from 'bullmq'; +import { addMinutes, endOfDay, getMonth, getYear, isSameDay, startOfDay } from 'date-fns'; +import { + lastValueFrom +} from 'rxjs'; +import { EmployeeService } from 'src/modules/feat-business-objects/entities/employee/employee.service'; +import { DispoLiveApiExtractorsService } from '../../feat-extractors/dispo-live-api-extractors.service'; +import { DyflexisExtractorsService } from '../../feat-extractors/dyflexis-extractors.service'; +import { + parseCETDateTime, + pick, + transformProperties, +} from '../../feat-transformers/transformer.functions'; +import { SCHICHTPLANUNG_QUEUE } from '../tokens'; + +@Injectable() +export class SchichtplanungSynchronizer { + constructor( + private readonly dlApiService: DispoLiveApiExtractorsService, + private readonly dyflexisService: DyflexisExtractorsService, + private readonly employeeService: EmployeeService, + @InjectQueue(SCHICHTPLANUNG_QUEUE) + private readonly schichtplanungQueue: Queue, + ) {} + + // TODO: Do we really want to load everything fresh? Anwer: For now it's fine, by this we circumvent problems arising from our application code + async syncSchichtplanung(day: Date) { + // Fetch and prepare data + const plannedTimes = await lastValueFrom( + this.dyflexisService.fetchPlannedTimeForMonth( + getMonth(day) + 1, + getYear(day), + ), + ); + const plannedTimeForDay = plannedTimes + .filter(({ mark }) => mark !== 'sick') + .filter(({ deleted }) => deleted != '1') + .filter(({ user_id }) => user_id != '0') + .filter( + ({ department_name }) => + department_name !== 'Büro' && department_name !== 'Leitstelle', + ) + .filter(({ start_date }) => isSameDay(new Date(start_date), day)); + + const cars = await lastValueFrom(this.dlApiService.fahrzeuge()); + + // Create Schichtplanung + const schichtplanung = []; + for (const plannedTime of plannedTimeForDay) { + const { start_date, end_date } = transformProperties( + parseCETDateTime('yyyy-MM-dd HH:mm:ss'), + )( + pick(['start_date', 'end_date'])( + plannedTime as Record<'start_date' | 'end_date', string>, + ), + ); + + const employee = await this.employeeService.findByPersonnelNumber( + plannedTime.user_id, + ); + if (!employee) { + console.error(`couldn\'t find employee with id ${plannedTime.user_id}`); + continue; + } + + const car = cars.data.find(({ kfzNr }) => + this.isCarNameForDepartment(plannedTime.department_name, kfzNr), + ); + if (!car) { + console.error( + `couldn\'t find car with department ${plannedTime.department_name}`, + ); + continue; + } + + let planning = { + className: 'greenItem', + start: start_date.toISOString(), + end: end_date.toISOString(), + group: car._id, + mitarbeiterId: employee.id, + } + + if(car.kfzNr.indexOf("KTW") > -1) { + planning = { + ...planning, + start: addMinutes(new Date(planning.start), 15).toISOString(), + } + } + + schichtplanung.push(planning); + } + + // Determine update set + const currentPlanung = await lastValueFrom( + this.dlApiService.fahrzeugplanungFindByRange({ + start: startOfDay(day), + end: endOfDay(day), + }), + ).then(({ data }) => + data.filter(({ className }) => className === 'greenItem'), + ); + + const toCreate = []; + const toUpdate = []; + const toDelete = currentPlanung + .filter( + ({ mitarbeiterId }) => + schichtplanung.findIndex( + ({ mitarbeiterId: id }) => id === mitarbeiterId, + ) === -1, + ) + .map(({ _id }) => ({ _id })); + + for (const plan of schichtplanung) { + const plannedTime = currentPlanung.find( + ({ mitarbeiterId }) => mitarbeiterId === plan.mitarbeiterId, + ); + if (!plannedTime) { + toCreate.push(plan); + continue; + } + if ( + !!plannedTime && + (plannedTime?.start !== plan.start || plannedTime?.end !== plan.end || plannedTime?.group !== plan.group) + ) { + toUpdate.push({ ...plan, _id: plannedTime._id }); + } + } + + await Promise.all([ + this.schichtplanungQueue.addBulk( + toCreate.map((item) => ({ + name: 'Create', + data: item, + })), + ), + this.schichtplanungQueue.addBulk( + toUpdate.map((item) => ({ + name: 'Update', + data: item, + })), + ), + this.schichtplanungQueue.addBulk( + toDelete.map((item) => ({ + name: 'Delete', + data: item._id, + })), + ), + ]); + + return { + create: toCreate, + update: toUpdate, + delete: toDelete, + }; + } + + private isCarNameForDepartment(department: string, carName: string): boolean { + if (!department || !carName) return false; + const [ordinanceType, carNumber] = department + .replace('1/', '') + .replace('E', '') + .split('-') + .map((part) => part.trim()); + return ( + (carName.indexOf(ordinanceType) > -1 && + carName.indexOf(carNumber) > -1) || + carName === department + ); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/tokens.ts b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/tokens.ts new file mode 100644 index 0000000..16cad9b --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-etl/feat-importers/tokens.ts @@ -0,0 +1 @@ +export const SCHICHTPLANUNG_QUEUE = 'schichtplanung' \ No newline at end of file diff --git a/service/data-hub/workflows/tools/delete-schichtplanung.ts b/service/data-hub/workflows/tools/delete-schichtplanung.ts new file mode 100644 index 0000000..7ed3c31 --- /dev/null +++ b/service/data-hub/workflows/tools/delete-schichtplanung.ts @@ -0,0 +1,67 @@ +import { + format, +} from 'npm:date-fns@4.1.0'; + +export interface FahrzeugplanungItemDto { + _id?: string; + className: string; // 'greenItem' OR 'blocked' + group: string; // Fahrzeug ID + mitarbeiterId: string; // Mitarbeiter ID + start: string; // ISO 8601 TS (UTC) + end: string; // ISO 8601 TS (UTC) +} + +function chunk(input: T[], size: number): T[][] { + return input.reduce( + (array, item, index) => + index % size === 0 + ? [...array, [item]] + : [...array.slice(0, -1), [...array.slice(-1)[0], item]], + [] as T[][] + ); +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)) +} + +async function deleteSchichtplanungItems(ids: string[]) { + const endpoint = 'custom/open-api/fahrzeugplanung/delete'; + const url = `https://avicenna.dispolive.de/${endpoint}`; + + console.log(`Deleting ids: ${ids.join(', ')}`) + return await Promise.all( + ids.map(id => fetch(`${url}/${id}`, { + method: "GET", + headers: { + Authorization: `Bearer ba397e8a-29f2-4c99-9141-75a71fd55830`, + 'dispolive-api-version': 'v1.0.1', + }, + })) + ); +} + +async function getSchichtplanungForDay(day: Date) { + const endpoint = 'custom/open-api/fahrzeugplanung/findByRange'; + const url = `https://avicenna.dispolive.de/${endpoint}/${format(new Date(day), 'yyyy-MM-dd')}_${format(new Date(day), 'yyyy-MM-dd')}`; + + return await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ba397e8a-29f2-4c99-9141-75a71fd55830`, + 'dispolive-api-version': 'v1.0.1', + }, + }); +} + +async function main() { + const schichtplanung = await getSchichtplanungForDay(new Date('2025-08-11')).then(response => response.json()); + const chunks = chunk(schichtplanung.map(({id}) => id), 1) + for(const chunk of chunks) { + const res = await deleteSchichtplanungItems(chunk as string[]) + console.log(res) + await sleep(200) + } +} + +await main() \ No newline at end of file diff --git a/service/data-hub/workflows/tools/deno.json b/service/data-hub/workflows/tools/deno.json new file mode 100644 index 0000000..5b320c2 --- /dev/null +++ b/service/data-hub/workflows/tools/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "dev": "deno run --watch main.ts" + }, + "imports": { + "@std/assert": "jsr:@std/assert@1" + } +} diff --git a/service/data-hub/workflows/tools/deno.lock b/service/data-hub/workflows/tools/deno.lock new file mode 100644 index 0000000..a018d33 --- /dev/null +++ b/service/data-hub/workflows/tools/deno.lock @@ -0,0 +1,29 @@ +{ + "version": "5", + "specifiers": { + "jsr:@std/assert@1": "1.0.9", + "jsr:@std/internal@^1.0.5": "1.0.5", + "npm:date-fns@4.1.0": "4.1.0" + }, + "jsr": { + "@std/assert@1.0.9": { + "integrity": "a9f0c611a869cc791b26f523eec54c7e187aab7932c2c8e8bea0622d13680dcd", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/internal@1.0.5": { + "integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba" + } + }, + "npm": { + "date-fns@4.1.0": { + "integrity": "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==" + } + }, + "workspace": { + "dependencies": [ + "jsr:@std/assert@1" + ] + } +} diff --git a/service/docker-compose.deploy.yml b/service/docker-compose.deploy.yml index 869a0fe..d1235ca 100644 --- a/service/docker-compose.deploy.yml +++ b/service/docker-compose.deploy.yml @@ -77,3 +77,8 @@ services: - pgdata:/var/lib/postgresql/data networks: - data-hub_net + + cache: + image: valkey/valkey:8.1.2-alpine + networks: + - data-hub_net \ No newline at end of file