From 8be07e1f9b13832d501f380aec7f190480c3d98f Mon Sep 17 00:00:00 2001 From: Marcel Arndt Date: Tue, 9 Jul 2024 17:23:08 +0200 Subject: [PATCH] (data-hub): add etl feature --- service/data-hub/data-connector/.gitignore | 2 + .../data-connector/docker-compose.yaml | 19 + .../data-hub/data-connector/package-lock.json | 432 +++++++++++++++- service/data-hub/data-connector/package.json | 8 + .../20240709133540_init/migration.sql | 473 +++++++++++++++++ .../prisma/migrations/migration_lock.toml | 3 + .../data-connector/prisma/schema.prisma | 480 +++++++++++++++++ .../data-connector/src/app.controller.ts | 27 +- .../data-hub/data-connector/src/app.module.ts | 13 +- .../data-connector/src/app.service.ts | 167 +++++- .../src/core/config/config.module.ts | 11 + .../src/core/config/config.service.ts | 27 + .../src/core/database/constants.ts | 1 + .../src/core/database/pagination.interface.ts | 9 + .../core/database/prisma-module.options.ts | 51 ++ .../src/core/database/prisma.module.ts | 100 ++++ .../src/core/database/prisma.service.spec.ts | 18 + .../src/core/database/prisma.service.ts | 62 +++ .../data-access-extractors.module.ts | 19 + .../dispo-live-extractors.service.ts | 481 ++++++++++++++++++ .../dyflexis-extractors.service.ts | 343 +++++++++++++ .../feat-extractors/extractors.service.ts | 141 +++++ .../feat-extractors/source.interface.ts | 142 ++++++ .../feat-extractors/source.repository.ts | 163 ++++++ .../feat-importers/events/event-types.enum.ts | 5 + .../events/extraction-done.event.ts | 3 + .../events/loading-done.event.ts | 3 + .../events/transformation-done.event.ts | 3 + .../feat-importers/feature-importer.module.ts | 17 + .../feat-importers/importer.service.ts | 113 ++++ .../data-access-loaders.module.ts | 13 + .../modules/feat-loaders/final.repository.ts | 278 ++++++++++ .../modules/feat-loaders/loaders.service.ts | 204 ++++++++ .../loaders/attendance-registration.loader.ts | 82 +++ .../feat-loaders/loaders/patient.loader.ts | 77 +++ .../loaders/planned-time.loader.ts | 75 +++ .../feat-loaders/loaders/tour.loader.ts | 149 ++++++ .../feat-loaders/loaders/work-time.loader.ts | 70 +++ .../adapter/tours-calculation.adapter.ts | 54 ++ .../data-access-transformers.module.ts | 13 + .../feat-transformers/staging.interface.ts | 117 +++++ .../feat-transformers/staging.repository.ts | 118 +++++ .../transformer.functions.ts | 54 ++ .../feat-transformers/transformers.service.ts | 149 ++++++ .../attendance-registration.transformer.ts | 36 ++ .../transformers/employee.transformer.ts | 67 +++ .../transformers/patient.transformer.ts | 50 ++ .../transformers/planned-time.transformer.ts | 52 ++ .../transformers/tour.transformer.ts | 142 ++++++ .../transformers/work-time.transformer.ts | 42 ++ .../data-connector/src/utils/chunk.spec.ts | 19 + .../data-connector/src/utils/chunk.ts | 9 + .../data-connector/src/utils/index.ts | 3 + .../src/utils/is-valid-date.spec.ts | 12 + .../data-connector/src/utils/is-valid-date.ts | 3 + .../data-connector/src/utils/range.spec.ts | 9 + .../data-connector/src/utils/range.ts | 3 + 57 files changed, 5211 insertions(+), 25 deletions(-) create mode 100644 service/data-hub/data-connector/docker-compose.yaml create mode 100644 service/data-hub/data-connector/prisma/migrations/20240709133540_init/migration.sql create mode 100644 service/data-hub/data-connector/prisma/migrations/migration_lock.toml create mode 100644 service/data-hub/data-connector/prisma/schema.prisma create mode 100644 service/data-hub/data-connector/src/core/config/config.module.ts create mode 100644 service/data-hub/data-connector/src/core/config/config.service.ts create mode 100644 service/data-hub/data-connector/src/core/database/constants.ts create mode 100644 service/data-hub/data-connector/src/core/database/pagination.interface.ts create mode 100644 service/data-hub/data-connector/src/core/database/prisma-module.options.ts create mode 100644 service/data-hub/data-connector/src/core/database/prisma.module.ts create mode 100644 service/data-hub/data-connector/src/core/database/prisma.service.spec.ts create mode 100644 service/data-hub/data-connector/src/core/database/prisma.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/data-access-extractors.module.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/dispo-live-extractors.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/dyflexis-extractors.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/extractors.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/source.interface.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-extractors/source.repository.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/events/event-types.enum.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/events/extraction-done.event.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/events/loading-done.event.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/events/transformation-done.event.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/feature-importer.module.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-importers/importer.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/data-access-loaders.module.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/final.repository.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders/attendance-registration.loader.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders/patient.loader.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders/planned-time.loader.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders/tour.loader.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-loaders/loaders/work-time.loader.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/adapter/tours-calculation.adapter.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/data-access-transformers.module.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/staging.interface.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/staging.repository.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformer.functions.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers.service.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/attendance-registration.transformer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/employee.transformer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/patient.transformer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/planned-time.transformer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/tour.transformer.ts create mode 100644 service/data-hub/data-connector/src/modules/feat-transformers/transformers/work-time.transformer.ts create mode 100644 service/data-hub/data-connector/src/utils/chunk.spec.ts create mode 100644 service/data-hub/data-connector/src/utils/chunk.ts create mode 100644 service/data-hub/data-connector/src/utils/index.ts create mode 100644 service/data-hub/data-connector/src/utils/is-valid-date.spec.ts create mode 100644 service/data-hub/data-connector/src/utils/is-valid-date.ts create mode 100644 service/data-hub/data-connector/src/utils/range.spec.ts create mode 100644 service/data-hub/data-connector/src/utils/range.ts diff --git a/service/data-hub/data-connector/.gitignore b/service/data-hub/data-connector/.gitignore index 4b56acf..07f9376 100644 --- a/service/data-hub/data-connector/.gitignore +++ b/service/data-hub/data-connector/.gitignore @@ -1,3 +1,5 @@ +.local/ + # compiled output /dist /node_modules diff --git a/service/data-hub/data-connector/docker-compose.yaml b/service/data-hub/data-connector/docker-compose.yaml new file mode 100644 index 0000000..d9b61a9 --- /dev/null +++ b/service/data-hub/data-connector/docker-compose.yaml @@ -0,0 +1,19 @@ +version: '3' + +services: + avicenna-db: + image: postgres:16.3 + ports: + - ${DATABASE_PORT}:${DATABASE_PORT} + env_file: .env + environment: + POSTGRES_HOST: ${DATABASE_HOST} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} + POSTGRES_USER: ${DATABASE_USERNAME} + POSTGRES_DB: ${DATABASE_DATABASE} + PG_DATA: /var/lib/postgresql/data + volumes: + - ./.local/pgdata:/var/lib/postgresql/data + +volumes: + pgdata: diff --git a/service/data-hub/data-connector/package-lock.json b/service/data-hub/data-connector/package-lock.json index 8b2f0b2..47d8453 100644 --- a/service/data-hub/data-connector/package-lock.json +++ b/service/data-hub/data-connector/package-lock.json @@ -9,9 +9,16 @@ "version": "0.0.1", "license": "UNLICENSED", "dependencies": { + "@nestjs/axios": "^3.0.2", "@nestjs/common": "^10.0.0", + "@nestjs/config": "^3.2.3", "@nestjs/core": "^10.0.0", + "@nestjs/event-emitter": "^2.0.4", "@nestjs/platform-express": "^10.0.0", + "@nestjs/schedule": "^4.1.0", + "@prisma/client": "^5.16.2", + "date-fns": "^3.6.0", + "date-fns-tz": "^3.1.3", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1" }, @@ -30,6 +37,7 @@ "eslint-plugin-prettier": "^5.0.0", "jest": "^29.5.0", "prettier": "^3.0.0", + "prisma": "^5.16.2", "source-map-support": "^0.5.21", "supertest": "^7.0.0", "ts-jest": "^29.1.0", @@ -1559,6 +1567,16 @@ "node": ">=8" } }, + "node_modules/@nestjs/axios": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.0.2.tgz", + "integrity": "sha512-Z6GuOUdNQjP7FX+OuV2Ybyamse+/e0BFdTWBX5JxpBDKA+YkdLynDgG6HTF04zy6e9zPa19UX0WA2VDoehwhXQ==", + "peerDependencies": { + "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0", + "axios": "^1.3.1", + "rxjs": "^6.0.0 || ^7.0.0" + } + }, "node_modules/@nestjs/cli": { "version": "10.4.2", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.2.tgz", @@ -1645,6 +1663,20 @@ } } }, + "node_modules/@nestjs/config": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-3.2.3.tgz", + "integrity": "sha512-p6yv/CvoBewJ72mBq4NXgOAi2rSQNWx3a+IMJLVKS2uiwFCOQQuiIatGwq6MRjXV3Jr+B41iUO8FIf4xBrZ4/w==", + "dependencies": { + "dotenv": "16.4.5", + "dotenv-expand": "10.0.0", + "lodash": "4.17.21" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "rxjs": "^7.1.0" + } + }, "node_modules/@nestjs/core": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.10.tgz", @@ -1682,6 +1714,18 @@ } } }, + "node_modules/@nestjs/event-emitter": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-2.0.4.tgz", + "integrity": "sha512-quMiw8yOwoSul0pp3mOonGz8EyXWHSBTqBy8B0TbYYgpnG1Ix2wGUnuTksLWaaBiiOTDhciaZ41Y5fJZsSJE1Q==", + "dependencies": { + "eventemitter2": "6.4.9" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, "node_modules/@nestjs/platform-express": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.10.tgz", @@ -1702,6 +1746,19 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.1.0.tgz", + "integrity": "sha512-WEc96WTXZW+VI/Ng+uBpiBUwm6TWtAbQ4RKWkfbmzKvmbRGzA/9k/UyAWDS9k0pp+ZcbC+MaZQtt7TjQHrwX6g==", + "dependencies": { + "cron": "3.1.7", + "uuid": "10.0.0" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.1.2.tgz", @@ -1825,6 +1882,68 @@ "url": "https://opencollective.com/unts" } }, + "node_modules/@prisma/client": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.16.2.tgz", + "integrity": "sha512-+1lmkhR9gHWcTC5oghm2ZKpWljyWdzfazCVlLKUWXVmwHSf52g81aZ8qb6Km5Bs025yBi7puLp3qSLEvktoUtw==", + "hasInstallScript": true, + "engines": { + "node": ">=16.13" + }, + "peerDependencies": { + "prisma": "*" + }, + "peerDependenciesMeta": { + "prisma": { + "optional": true + } + } + }, + "node_modules/@prisma/debug": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.16.2.tgz", + "integrity": "sha512-ItzB4nR4O8eLzuJiuP3WwUJfoIvewMHqpGCad+64gvThcKEVOtaUza9AEJo2DPqAOa/AWkFyK54oM4WwHeew+A==", + "devOptional": true + }, + "node_modules/@prisma/engines": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.16.2.tgz", + "integrity": "sha512-qUxwMtrwoG3byd4PbX6T7EjHJ8AUhzTuwniOGkh/hIznBfcE2QQnGakyEq4VnwNuttMqvh/GgPFapHQ3lCuRHg==", + "devOptional": true, + "hasInstallScript": true, + "dependencies": { + "@prisma/debug": "5.16.2", + "@prisma/engines-version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "@prisma/fetch-engine": "5.16.2", + "@prisma/get-platform": "5.16.2" + } + }, + "node_modules/@prisma/engines-version": { + "version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303.tgz", + "integrity": "sha512-HkT2WbfmFZ9WUPyuJHhkiADxazHg8Y4gByrTSVeb3OikP6tjQ7txtSUGu9OBOBH0C13dPKN2qqH12xKtHu/Hiw==", + "devOptional": true + }, + "node_modules/@prisma/fetch-engine": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.16.2.tgz", + "integrity": "sha512-sq51lfHKfH2jjYSjBtMjP+AznFqOJzXpqmq6B9auWrlTJrMgZ7lPyhWUW7VU7LsQU48/TJ+DZeIz8s9bMYvcHg==", + "devOptional": true, + "dependencies": { + "@prisma/debug": "5.16.2", + "@prisma/engines-version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "@prisma/get-platform": "5.16.2" + } + }, + "node_modules/@prisma/get-platform": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.16.2.tgz", + "integrity": "sha512-cXiHPgNLNyj22vLouPVNegklpRL/iX2jxTeap5GRO3DmCoVyIHmJAV1CgUMUJhHlcol9yYy7EHvsnXTDJ/PKEA==", + "devOptional": true, + "dependencies": { + "@prisma/debug": "5.16.2" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -2044,6 +2163,11 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==" + }, "node_modules/@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -2706,8 +2830,18 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "peer": true, + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } }, "node_modules/babel-jest": { "version": "29.7.0", @@ -3310,7 +3444,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -3491,6 +3624,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.7.tgz", + "integrity": "sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==", + "dependencies": { + "@types/luxon": "~3.4.0", + "luxon": "~3.4.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -3505,6 +3647,23 @@ "node": ">= 8" } }, + "node_modules/date-fns": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", + "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/kossnocorp" + } + }, + "node_modules/date-fns-tz": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-3.1.3.tgz", + "integrity": "sha512-ZfbMu+nbzW0mEzC8VZrLiSWvUIaI3aRHeq33mTe7Y38UctKukgqPR4nTDwcwS4d64Gf8GghnVsroBuMY3eiTeA==", + "peerDependencies": { + "date-fns": "^3.0.0" + } + }, "node_modules/debug": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", @@ -3583,7 +3742,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -3666,6 +3824,25 @@ "node": ">=6.0.0" } }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==", + "engines": { + "node": ">=12" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -4043,6 +4220,11 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter2": { + "version": "6.4.9", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", + "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -4356,6 +4538,26 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "peer": true, + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz", @@ -4426,7 +4628,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -5968,8 +6169,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.memoize": { "version": "4.1.2", @@ -6008,6 +6208,14 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.8", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz", @@ -6741,6 +6949,22 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/prisma": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.16.2.tgz", + "integrity": "sha512-rFV/xoBR2hBGGlu4LPLQd4U8WVA+tSAmYyFWGPRVfj+xg7N4kiZV4lSk38htSpF+/IuHKzlrbh4SFk8Z18cI8A==", + "devOptional": true, + "hasInstallScript": true, + "dependencies": { + "@prisma/engines": "5.16.2" + }, + "bin": { + "prisma": "build/index.js" + }, + "engines": { + "node": ">=16.13" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -6771,6 +6995,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "peer": true + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -8174,6 +8404,18 @@ "node": ">= 0.4.0" } }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -9643,6 +9885,12 @@ "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==" }, + "@nestjs/axios": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@nestjs/axios/-/axios-3.0.2.tgz", + "integrity": "sha512-Z6GuOUdNQjP7FX+OuV2Ybyamse+/e0BFdTWBX5JxpBDKA+YkdLynDgG6HTF04zy6e9zPa19UX0WA2VDoehwhXQ==", + "requires": {} + }, "@nestjs/cli": { "version": "10.4.2", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.2.tgz", @@ -9688,6 +9936,16 @@ "uid": "2.0.2" } }, + "@nestjs/config": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/@nestjs/config/-/config-3.2.3.tgz", + "integrity": "sha512-p6yv/CvoBewJ72mBq4NXgOAi2rSQNWx3a+IMJLVKS2uiwFCOQQuiIatGwq6MRjXV3Jr+B41iUO8FIf4xBrZ4/w==", + "requires": { + "dotenv": "16.4.5", + "dotenv-expand": "10.0.0", + "lodash": "4.17.21" + } + }, "@nestjs/core": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/@nestjs/core/-/core-10.3.10.tgz", @@ -9701,6 +9959,14 @@ "uid": "2.0.2" } }, + "@nestjs/event-emitter": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-2.0.4.tgz", + "integrity": "sha512-quMiw8yOwoSul0pp3mOonGz8EyXWHSBTqBy8B0TbYYgpnG1Ix2wGUnuTksLWaaBiiOTDhciaZ41Y5fJZsSJE1Q==", + "requires": { + "eventemitter2": "6.4.9" + } + }, "@nestjs/platform-express": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.3.10.tgz", @@ -9713,6 +9979,15 @@ "tslib": "2.6.3" } }, + "@nestjs/schedule": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.1.0.tgz", + "integrity": "sha512-WEc96WTXZW+VI/Ng+uBpiBUwm6TWtAbQ4RKWkfbmzKvmbRGzA/9k/UyAWDS9k0pp+ZcbC+MaZQtt7TjQHrwX6g==", + "requires": { + "cron": "3.1.7", + "uuid": "10.0.0" + } + }, "@nestjs/schematics": { "version": "10.1.2", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.1.2.tgz", @@ -9792,6 +10067,56 @@ "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", "dev": true }, + "@prisma/client": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.16.2.tgz", + "integrity": "sha512-+1lmkhR9gHWcTC5oghm2ZKpWljyWdzfazCVlLKUWXVmwHSf52g81aZ8qb6Km5Bs025yBi7puLp3qSLEvktoUtw==", + "requires": {} + }, + "@prisma/debug": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.16.2.tgz", + "integrity": "sha512-ItzB4nR4O8eLzuJiuP3WwUJfoIvewMHqpGCad+64gvThcKEVOtaUza9AEJo2DPqAOa/AWkFyK54oM4WwHeew+A==", + "devOptional": true + }, + "@prisma/engines": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.16.2.tgz", + "integrity": "sha512-qUxwMtrwoG3byd4PbX6T7EjHJ8AUhzTuwniOGkh/hIznBfcE2QQnGakyEq4VnwNuttMqvh/GgPFapHQ3lCuRHg==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.16.2", + "@prisma/engines-version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "@prisma/fetch-engine": "5.16.2", + "@prisma/get-platform": "5.16.2" + } + }, + "@prisma/engines-version": { + "version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303.tgz", + "integrity": "sha512-HkT2WbfmFZ9WUPyuJHhkiADxazHg8Y4gByrTSVeb3OikP6tjQ7txtSUGu9OBOBH0C13dPKN2qqH12xKtHu/Hiw==", + "devOptional": true + }, + "@prisma/fetch-engine": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.16.2.tgz", + "integrity": "sha512-sq51lfHKfH2jjYSjBtMjP+AznFqOJzXpqmq6B9auWrlTJrMgZ7lPyhWUW7VU7LsQU48/TJ+DZeIz8s9bMYvcHg==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.16.2", + "@prisma/engines-version": "5.16.0-24.34ace0eb2704183d2c05b60b52fba5c43c13f303", + "@prisma/get-platform": "5.16.2" + } + }, + "@prisma/get-platform": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.16.2.tgz", + "integrity": "sha512-cXiHPgNLNyj22vLouPVNegklpRL/iX2jxTeap5GRO3DmCoVyIHmJAV1CgUMUJhHlcol9yYy7EHvsnXTDJ/PKEA==", + "devOptional": true, + "requires": { + "@prisma/debug": "5.16.2" + } + }, "@sinclair/typebox": { "version": "0.27.8", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", @@ -10011,6 +10336,11 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/luxon": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==" + }, "@types/methods": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", @@ -10524,8 +10854,18 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "peer": true, + "requires": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } }, "babel-jest": { "version": "29.7.0", @@ -10945,7 +11285,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -11079,6 +11418,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cron": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.1.7.tgz", + "integrity": "sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==", + "requires": { + "@types/luxon": "~3.4.0", + "luxon": "~3.4.0" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -11090,6 +11438,17 @@ "which": "^2.0.1" } }, + "date-fns": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz", + "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==" + }, + "date-fns-tz": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/date-fns-tz/-/date-fns-tz-3.1.3.tgz", + "integrity": "sha512-ZfbMu+nbzW0mEzC8VZrLiSWvUIaI3aRHeq33mTe7Y38UctKukgqPR4nTDwcwS4d64Gf8GghnVsroBuMY3eiTeA==", + "requires": {} + }, "debug": { "version": "4.3.5", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", @@ -11140,8 +11499,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "depd": { "version": "2.0.0", @@ -11199,6 +11557,16 @@ "esutils": "^2.0.2" } }, + "dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==" + }, + "dotenv-expand": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-10.0.0.tgz", + "integrity": "sha512-GopVGCpVS1UKH75VKHGuQFqS1Gusej0z4FyQkPdwjil2gNIv+LNsqBlboOzpJFZKVT95GkCyWJbBSdFEFUWI2A==" + }, "eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -11467,6 +11835,11 @@ "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, + "eventemitter2": { + "version": "6.4.9", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", + "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" + }, "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -11737,6 +12110,12 @@ "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==", "dev": true }, + "follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "peer": true + }, "foreground-child": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz", @@ -11792,7 +12171,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -12929,8 +13307,7 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.memoize": { "version": "4.1.2", @@ -12963,6 +13340,11 @@ "yallist": "^3.0.2" } }, + "luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==" + }, "magic-string": { "version": "0.30.8", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz", @@ -13489,6 +13871,15 @@ } } }, + "prisma": { + "version": "5.16.2", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.16.2.tgz", + "integrity": "sha512-rFV/xoBR2hBGGlu4LPLQd4U8WVA+tSAmYyFWGPRVfj+xg7N4kiZV4lSk38htSpF+/IuHKzlrbh4SFk8Z18cI8A==", + "devOptional": true, + "requires": { + "@prisma/engines": "5.16.2" + } + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -13513,6 +13904,12 @@ "ipaddr.js": "1.9.1" } }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "peer": true + }, "punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -14497,6 +14894,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==" }, + "uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/service/data-hub/data-connector/package.json b/service/data-hub/data-connector/package.json index ea15552..a0b4857 100644 --- a/service/data-hub/data-connector/package.json +++ b/service/data-hub/data-connector/package.json @@ -20,9 +20,16 @@ "test:e2e": "jest --config ./test/jest-e2e.json" }, "dependencies": { + "@nestjs/axios": "^3.0.2", "@nestjs/common": "^10.0.0", + "@nestjs/config": "^3.2.3", "@nestjs/core": "^10.0.0", + "@nestjs/event-emitter": "^2.0.4", "@nestjs/platform-express": "^10.0.0", + "@nestjs/schedule": "^4.1.0", + "@prisma/client": "^5.16.2", + "date-fns": "^3.6.0", + "date-fns-tz": "^3.1.3", "reflect-metadata": "^0.2.0", "rxjs": "^7.8.1" }, @@ -41,6 +48,7 @@ "eslint-plugin-prettier": "^5.0.0", "jest": "^29.5.0", "prettier": "^3.0.0", + "prisma": "^5.16.2", "source-map-support": "^0.5.21", "supertest": "^7.0.0", "ts-jest": "^29.1.0", diff --git a/service/data-hub/data-connector/prisma/migrations/20240709133540_init/migration.sql b/service/data-hub/data-connector/prisma/migrations/20240709133540_init/migration.sql new file mode 100644 index 0000000..91cbf11 --- /dev/null +++ b/service/data-hub/data-connector/prisma/migrations/20240709133540_init/migration.sql @@ -0,0 +1,473 @@ +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- CreateEnum +CREATE TYPE "TicketState" AS ENUM ('CREATED', 'APPROVAL_REQUIRED', 'READY', 'DONE', 'BILLABLE', 'ARCHIVED'); + +-- CreateEnum +CREATE TYPE "TicketValidationState" AS ENUM ('CREATED', 'USER_INTERACTION_REQUIRED', 'TD_UNCERTAIN', 'DOCUMENTS_MISSING', 'ARCHIVED'); + +-- CreateEnum +CREATE TYPE "ApprovalState" AS ENUM ('UNKNOWN', 'FREE', 'REQUIRED', 'REQUESTED', 'APPROVED'); + +-- CreateTable +CREATE TABLE "TourSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "check" TEXT, + "createTime" TEXT, + "empfangenVonId" TEXT, + "endgen" TEXT, + "serienId" TEXT, + "serienKategorName" TEXT, + "serienKategorie" TEXT, + "zadStatus" TEXT, + "abrDate" TEXT, + "abrNum" TEXT, + "abrTime" TEXT, + "ausrueckZeit" TEXT, + "startTime" TEXT, + "startDate" TEXT, + "date" TEXT, + "date_iso" TEXT, + "dauergenehmigung" TEXT, + "direction" TEXT, + "distanceValue" TEXT, + "besetztKm" TEXT, + "gefahren" TEXT, + "gesamtPreis" TEXT, + "infektion" TEXT, + "itDesc" TEXT, + "itKrz" TEXT, + "itName" TEXT, + "kkId" TEXT, + "krankenkasse" TEXT, + "patId" TEXT, + "patName" TEXT, + "patSurname" TEXT, + "patStreet" TEXT, + "patZip" TEXT, + "patCity" TEXT, + "startInstitution" TEXT, + "startStreet" TEXT, + "startZip" TEXT, + "startCity" TEXT, + "targetInstitution" TEXT, + "targetStreet" TEXT, + "targetZip" TEXT, + "targetCity" TEXT, + "transportKrz" TEXT, + "transportName" TEXT, + "transportart" TEXT, + "verordnungsId" TEXT, + "verordnungsName" TEXT, + "type" TEXT, + "startBegin" TEXT, + "startEnd" TEXT, + "frei" TEXT, + "ziel" TEXT, + "zielBegin" TEXT, + "endTime" TEXT, + "endDate" TEXT, + "einsatzNummer" TEXT, + "carName" TEXT, + "carNo" TEXT, + "beifahrerName" TEXT, + "beifahrer" TEXT, + "fahrerName" TEXT, + "fahrer" TEXT, + "color" TEXT, + + CONSTRAINT "TourSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "EmployeeDispoLiveSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "pScheinAblaufDatum" TEXT, + "name" TEXT, + "surname" TEXT, + "persId" TEXT, + "importPersId" TEXT, + "qualifikation" TEXT, + + CONSTRAINT "EmployeeDispoLiveSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "AttendanceRegistrationSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "attendanceRegistrationId" TEXT, + "employeeId" TEXT, + "personnelNumber" TEXT, + "dateTime" TEXT, + "event" TEXT, + + CONSTRAINT "AttendanceRegistrationSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "PlannedTimeSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "user_id" TEXT, + "firstname" TEXT, + "surname" TEXT, + "department_id" TEXT, + "department_name" TEXT, + "start_date" TEXT, + "end_date" TEXT, + "pauze" TEXT, + "duration" TEXT, + "deleted" TEXT, + "mark" TEXT, + + CONSTRAINT "PlannedTimeSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "WorkTimeSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "user_id" TEXT, + "firstname" TEXT, + "surname" TEXT, + "department_id" TEXT, + "department_name" TEXT, + "start_date" TEXT, + "end_date" TEXT, + "pauze" TEXT, + "duration" TEXT, + + CONSTRAINT "WorkTimeSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "EmployeeDyflexisSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "firstname" TEXT, + "surname" TEXT, + "contract_id" TEXT, + "contract_start" TEXT, + "contract_end" TEXT, + "contract_hours_week" TEXT, + "contract_salary_hour" TEXT, + + CONSTRAINT "EmployeeDyflexisSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "PatientSource" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "insertedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "jobId" TEXT NOT NULL, + "id" TEXT, + "surname" TEXT, + "name" TEXT, + "street" TEXT, + "zip" TEXT, + "city" TEXT, + "birthday" TEXT, + "kkId" TEXT, + "krankenkasse" TEXT, + "pflegestufe" TEXT, + "behinderung1" TEXT, + "behinderung2" TEXT, + + CONSTRAINT "PatientSource_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "TourStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" TEXT, + "done" BOOLEAN, + "direction" TEXT, + "carName" TEXT, + "operationId" TEXT, + "driverId" TEXT, + "driverName" TEXT, + "codriverId" TEXT, + "codriverName" TEXT, + "patientId" TEXT, + "patientName" TEXT, + "patientSurname" TEXT, + "patientStreet" TEXT, + "patientZip" TEXT, + "patientCity" TEXT, + "healthInsurance" TEXT, + "healthInsuranceNumber" TEXT, + "type" TEXT, + "category" TEXT, + "transportType" TEXT, + "ordinanceType" TEXT, + "rangeEndDate" TIMESTAMP(3), + "startInstitution" TEXT, + "startStreet" TEXT, + "startZip" TEXT, + "startCity" TEXT, + "targetInstitution" TEXT, + "targetStreet" TEXT, + "targetZip" TEXT, + "targetCity" TEXT, + "startDate" TIMESTAMP(3), + "check" INTEGER NOT NULL DEFAULT 0, + "occupiedKm" DOUBLE PRECISION, + "totalKm" DOUBLE PRECISION, + "startBegin" TEXT, + "startEnd" TEXT, + "target" TEXT, + "targetBegin" TEXT, + "empty" TEXT, + "hasInfection" BOOLEAN NOT NULL DEFAULT false, + "infectionName" TEXT, + "revenueDispoLive" DOUBLE PRECISION, + "billDate" TIMESTAMP(3), + "billNumber" TEXT, + "revenue" DOUBLE PRECISION NOT NULL DEFAULT 0, + "revenueDeviation" DOUBLE PRECISION NOT NULL DEFAULT 0, + "consumptionCosts" DOUBLE PRECISION NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP(3), + + CONSTRAINT "TourStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "PatientStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" TEXT, + "surname" TEXT, + "name" TEXT, + "street" TEXT, + "zip" TEXT, + "city" TEXT, + "birthday" TIMESTAMP(3), + "healthinsuranceId" TEXT, + "healthinsurance" TEXT, + "careDegree" TEXT, + "disabilityMark1" TEXT, + "disabilityMark2" TEXT, + + CONSTRAINT "PatientStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "AttendanceRegistrationStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "attendanceRegistrationId" INTEGER NOT NULL, + "employeeId" INTEGER NOT NULL, + "dateTime" TIMESTAMP(3) NOT NULL, + "event" TEXT NOT NULL, + + CONSTRAINT "AttendanceRegistrationStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "PlannedTimeStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" INTEGER NOT NULL, + "employeeId" INTEGER NOT NULL, + "firstname" TEXT, + "surname" TEXT, + "departmentName" TEXT, + "startDate" TIMESTAMP(3) NOT NULL, + "endDate" TIMESTAMP(3) NOT NULL, + "deleted" BOOLEAN NOT NULL, + "mark" TEXT, + + CONSTRAINT "PlannedTimeStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "WorkTimeStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" INTEGER NOT NULL, + "employeeId" INTEGER NOT NULL, + "firstname" TEXT NOT NULL, + "surname" TEXT NOT NULL, + "departmentName" TEXT, + "startDate" TIMESTAMP(3) NOT NULL, + "endDate" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "WorkTimeStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "EmployeeStaging" ( + "internalId" UUID NOT NULL DEFAULT gen_random_uuid(), + "id" INTEGER NOT NULL, + "firstname" TEXT NOT NULL, + "surname" TEXT NOT NULL, + "contractStart" TIMESTAMP(3) NOT NULL, + "contractEnd" TIMESTAMP(3) NOT NULL, + "contractHoursWeek" DOUBLE PRECISION NOT NULL, + "contractSalaryHour" DOUBLE PRECISION NOT NULL, + "personTransportCertificate" TIMESTAMP(3), + "qualification" TEXT, + + CONSTRAINT "EmployeeStaging_pkey" PRIMARY KEY ("internalId") +); + +-- CreateTable +CREATE TABLE "Tour" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "done" BOOLEAN NOT NULL, + "direction" TEXT NOT NULL, + "carName" TEXT, + "operationId" TEXT NOT NULL, + "driverId" TEXT, + "driverName" TEXT, + "codriverId" TEXT, + "codriverName" TEXT, + "patientId" TEXT, + "patientName" TEXT, + "patientSurname" TEXT, + "patientStreet" TEXT, + "patientZip" TEXT, + "patientCity" TEXT, + "healthInsurance" TEXT, + "healthInsuranceNumber" TEXT, + "type" TEXT, + "category" TEXT, + "transportType" TEXT, + "ordinanceType" TEXT, + "rangeEndDate" TIMESTAMP(3), + "startInstitution" TEXT, + "startStreet" TEXT, + "startZip" TEXT, + "startCity" TEXT, + "targetInstitution" TEXT, + "targetStreet" TEXT, + "targetZip" TEXT, + "targetCity" TEXT, + "startDate" TIMESTAMP(3) NOT NULL, + "check" INTEGER NOT NULL DEFAULT 0, + "occupiedKm" DOUBLE PRECISION, + "totalKm" DOUBLE PRECISION, + "startBegin" TEXT, + "startEnd" TEXT, + "target" TEXT, + "targetBegin" TEXT, + "empty" TEXT, + "hasInfection" BOOLEAN NOT NULL DEFAULT false, + "infectionName" TEXT, + "revenueDispoLive" DOUBLE PRECISION, + "billDate" TIMESTAMP(3), + "billNumber" TEXT, + "revenue" DOUBLE PRECISION NOT NULL DEFAULT 0, + "consumptionCosts" DOUBLE PRECISION NOT NULL DEFAULT 0, + "revenueDeviation" DOUBLE PRECISION NOT NULL DEFAULT 0, + "createdAt" TIMESTAMP(3), + "deletedAt" TIMESTAMP(3), + "ticketId" UUID, + + CONSTRAINT "Tour_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Patient" ( + "id" TEXT NOT NULL, + "surname" TEXT, + "name" TEXT, + "street" TEXT, + "zip" TEXT, + "city" TEXT, + "birthday" TIMESTAMP(3), + "healthinsuranceId" TEXT, + "healthinsurance" TEXT, + "careDegree" TEXT, + "disabilityMark1" TEXT, + "disabilityMark2" TEXT, + + CONSTRAINT "Patient_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Ticket" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "currentState" "TicketValidationState" NOT NULL, + "notes" TEXT[], + "errors" TEXT[], + "isUrgent" BOOLEAN NOT NULL DEFAULT false, + "urgency" TIMESTAMP(3) NOT NULL DEFAULT '2099-12-31 23:59:59.999 +00:00', + "approvalState" "ApprovalState", + "documentInfo" JSONB DEFAULT '{"approval": {"isNeeded": null, "whoRequested": "","answeredFrom": ""},"tdLocation": ""}', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "Ticket_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "TicketStateHistory" ( + "id" UUID NOT NULL DEFAULT gen_random_uuid(), + "state" "TicketValidationState" NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "ticketId" UUID NOT NULL, + + CONSTRAINT "TicketStateHistory_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "AttendanceRegistration" ( + "attendanceRegistrationId" INTEGER NOT NULL, + "employeeId" INTEGER NOT NULL, + "personnelNumber" TEXT NOT NULL, + "dateTime" TIMESTAMP(3) NOT NULL, + "event" TEXT NOT NULL, + + CONSTRAINT "AttendanceRegistration_pkey" PRIMARY KEY ("attendanceRegistrationId") +); + +-- CreateTable +CREATE TABLE "WorkTime" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "firstname" TEXT NOT NULL, + "surname" TEXT NOT NULL, + "startDate" TIMESTAMP(3), + "endDate" TIMESTAMP(3), + "department" TEXT, + "pause" INTEGER NOT NULL, + "duration" INTEGER NOT NULL, + + CONSTRAINT "WorkTime_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "PlannedTime" ( + "id" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "firstname" TEXT, + "surname" TEXT, + "startDate" TIMESTAMP(3), + "endDate" TIMESTAMP(3), + "department" TEXT, + "pause" INTEGER NOT NULL, + "duration" INTEGER NOT NULL, + "note" TEXT, + + CONSTRAINT "PlannedTime_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Tour_operationId_key" ON "Tour"("operationId"); + +-- AddForeignKey +ALTER TABLE "Tour" ADD CONSTRAINT "Tour_ticketId_fkey" FOREIGN KEY ("ticketId") REFERENCES "Ticket"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "TicketStateHistory" ADD CONSTRAINT "TicketStateHistory_ticketId_fkey" FOREIGN KEY ("ticketId") REFERENCES "Ticket"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/service/data-hub/data-connector/prisma/migrations/migration_lock.toml b/service/data-hub/data-connector/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..fbffa92 --- /dev/null +++ b/service/data-hub/data-connector/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file diff --git a/service/data-hub/data-connector/prisma/schema.prisma b/service/data-hub/data-connector/prisma/schema.prisma new file mode 100644 index 0000000..a150860 --- /dev/null +++ b/service/data-hub/data-connector/prisma/schema.prisma @@ -0,0 +1,480 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? +// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init + +generator client { + provider = "prisma-client-js" + previewFeatures = ["postgresqlExtensions", "fieldReference"] +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") + extensions = [pgcrypto] +} + +model TourSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + check String? + createTime String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + empfangenVonId String? + endgen String? // yyyy-MM-dd + serienId String? + serienKategorName String? + serienKategorie String? + zadStatus String? + abrDate String? // yyyy-MM-dd + abrNum String? + abrTime String? // HH:mm + + ausrueckZeit String? // HH:mm 'Europe/Berlin' + startTime String? // HH:mm 'Europe/Berlin' (is after ausrueckZeit) + startDate String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + date String? // yyyy-MM-dd + date_iso String? // ISO 8601 String + dauergenehmigung String? + direction String? + distanceValue String? // meters (probably int) + besetztKm String? // kilometers (float) + gefahren String? // Boolean as number + gesamtPreis String? // float + infektion String? // Boolean as number + itDesc String? + itKrz String? + itName String? + kkId String? + krankenkasse String? + patId String? + patName String? + patSurname String? + patStreet String? + patZip String? + patCity String? + startInstitution String? + startStreet String? + startZip String? + startCity String? + targetInstitution String? + targetStreet String? + targetZip String? + targetCity String? + transportKrz String? + transportName String? + transportart String? // ID + verordnungsId String? + verordnungsName String? + type String? + + startBegin String? // HH:mm 'Europe/Berlin' + startEnd String? // HH:mm 'Europe/Berlin' + frei String? // HH:mm 'Europe/Berlin' + ziel String? // HH:mm 'Europe/Berlin' + zielBegin String? // HH:mm 'Europe/Berlin' + endTime String? // HH:mm 'Europe/Berlin' + endDate String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + + einsatzNummer String? + carName String? + carNo String? // ID + beifahrerName String? + beifahrer String? // ID + fahrerName String? + fahrer String? // ID + + color String? // Hex rgb +} + +model EmployeeDispoLiveSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + pScheinAblaufDatum String? // yyyy-MM-dd + name String? + surname String? + persId String? + importPersId String? // Personal ID aus Dyflexis + qualifikation String? // ID +} + +model AttendanceRegistrationSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + attendanceRegistrationId String? + employeeId String? + personnelNumber String? + dateTime String? // ISO 8601 String with timezone + event String? +} + +model PlannedTimeSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + user_id String? + firstname String? + surname String? + department_id String? + department_name String? + start_date String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + end_date String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + pauze String? + duration String? + deleted String? + mark String? +} + +model WorkTimeSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + user_id String? + firstname String? + surname String? + department_id String? + department_name String? + start_date String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + end_date String? // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + pauze String? + duration String? +} + +model EmployeeDyflexisSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + firstname String? + surname String? + contract_id String? + contract_start String? // yyyy-MM-dd + contract_end String? // yyyy-MM-dd + contract_hours_week String? // Float string + contract_salary_hour String? // Float string +} + +model PatientSource { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + insertedAt DateTime @default(now()) + jobId String + + id String? + surname String? + name String? + street String? + zip String? + city String? + + birthday String? + kkId String? + krankenkasse String? + pflegestufe String? + behinderung1 String? + behinderung2 String? +} + +model TourStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + id String? + done Boolean? + direction String? + carName String? + operationId String? + + driverId String? + driverName String? + codriverId String? + codriverName String? + + patientId String? + patientName String? + patientSurname String? + patientStreet String? + patientZip String? + patientCity String? + healthInsurance String? + healthInsuranceNumber String? + + type String? + category String? + transportType String? + ordinanceType String? + rangeEndDate DateTime? + + startInstitution String? + startStreet String? + startZip String? + startCity String? + targetInstitution String? + targetStreet String? + targetZip String? + targetCity String? + + startDate DateTime? + + check Int @default(0) + occupiedKm Float? + totalKm Float? + + startBegin String? + startEnd String? + target String? + targetBegin String? + empty String? + + hasInfection Boolean @default(false) + infectionName String? + + revenueDispoLive Float? + billDate DateTime? + billNumber String? + revenue Float @default(0) + revenueDeviation Float @default(0) + consumptionCosts Float @default(0) + + createdAt DateTime? +} + +model PatientStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + id String? + surname String? + name String? + street String? + zip String? + city String? + + birthday DateTime? + healthinsuranceId String? + healthinsurance String? + careDegree String? + disabilityMark1 String? + disabilityMark2 String? +} + +model AttendanceRegistrationStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + attendanceRegistrationId Int + employeeId Int + dateTime DateTime + event String +} + +model PlannedTimeStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + id Int + employeeId Int + firstname String? + surname String? + departmentName String? + startDate DateTime + endDate DateTime + deleted Boolean + mark String? +} + +model WorkTimeStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + id Int + employeeId Int + firstname String + surname String + departmentName String? + startDate DateTime + endDate DateTime +} + +model EmployeeStaging { + internalId String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + + id Int + firstname String + surname String + contractStart DateTime + contractEnd DateTime + contractHoursWeek Float + contractSalaryHour Float + personTransportCertificate DateTime? + qualification String? +} + +model Tour { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + done Boolean + direction String + carName String? + operationId String @unique + + driverId String? + driverName String? + codriverId String? + codriverName String? + + patientId String? + patientName String? + patientSurname String? + patientStreet String? + patientZip String? + patientCity String? + healthInsurance String? + healthInsuranceNumber String? + + type String? + category String? + transportType String? + ordinanceType String? + rangeEndDate DateTime? + + startInstitution String? + startStreet String? + startZip String? + startCity String? + targetInstitution String? + targetStreet String? + targetZip String? + targetCity String? + + startDate DateTime + + check Int @default(0) + occupiedKm Float? + totalKm Float? + + startBegin String? + startEnd String? + target String? + targetBegin String? + empty String? + + hasInfection Boolean @default(false) + infectionName String? + + revenueDispoLive Float? + billDate DateTime? + billNumber String? + + revenue Float @default(0) + consumptionCosts Float @default(0) + revenueDeviation Float @default(0) + + createdAt DateTime? + deletedAt DateTime? + Ticket Ticket? @relation(fields: [ticketId], references: [id]) + ticketId String? @db.Uuid +} + +model Patient { + id String @id + surname String? + name String? + street String? + zip String? + city String? + + birthday DateTime? + healthinsuranceId String? + healthinsurance String? + careDegree String? + disabilityMark1 String? + disabilityMark2 String? +} + +enum TicketState { + CREATED + APPROVAL_REQUIRED + READY + DONE + BILLABLE + ARCHIVED +} + +enum TicketValidationState { + CREATED + USER_INTERACTION_REQUIRED + TD_UNCERTAIN + DOCUMENTS_MISSING + ARCHIVED +} + +enum ApprovalState { + UNKNOWN + FREE + REQUIRED + REQUESTED + APPROVED +} + +model Ticket { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + tours Tour[] + state TicketStateHistory[] + currentState TicketValidationState + notes String[] + errors String[] + isUrgent Boolean @default(false) + urgency DateTime @default("2099-12-31T23:59:59.999Z") + approvalState ApprovalState? + documentInfo Json? @default("{\"approval\": {\"isNeeded\": null, \"whoRequested\": \"\",\"answeredFrom\": \"\"},\"tdLocation\": \"\"}") + createdAt DateTime @default(now()) +} + +model TicketStateHistory { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + state TicketValidationState + createdAt DateTime @default(now()) + + Ticket Ticket @relation(fields: [ticketId], references: [id]) + ticketId String @db.Uuid +} + +model AttendanceRegistration { + attendanceRegistrationId Int @id + employeeId Int + personnelNumber String + dateTime DateTime + event String +} + +model WorkTime { + id String @id + userId String + firstname String + surname String + startDate DateTime? + endDate DateTime? + department String? + pause Int + duration Int +} + +model PlannedTime { + id String @id + userId String + firstname String? + surname String? + startDate DateTime? + endDate DateTime? + department String? + pause Int + duration Int + note String? +} diff --git a/service/data-hub/data-connector/src/app.controller.ts b/service/data-hub/data-connector/src/app.controller.ts index cce879e..ae544b2 100644 --- a/service/data-hub/data-connector/src/app.controller.ts +++ b/service/data-hub/data-connector/src/app.controller.ts @@ -1,12 +1,27 @@ -import { Controller, Get } from '@nestjs/common'; -import { AppService } from './app.service'; +import { Controller, Get, Query } from '@nestjs/common'; +import { ImporterService } from './modules/feat-importers/importer.service'; @Controller() export class AppController { - constructor(private readonly appService: AppService) {} + constructor(private readonly importer: ImporterService) {} - @Get() - getHello(): string { - return this.appService.getHello(); + @Get('etl') + etl(@Query('from') from: string, @Query('to') to: string) { + if (!from || !to) { + return { + error: 'from and to must be defined', + }; + } + const interval = { + start: new Date(from), + end: new Date(to), + }; + return this.importer.runImport(interval); } + + // Call Extractors + + // Call Transformers + + // Call Loaders } diff --git a/service/data-hub/data-connector/src/app.module.ts b/service/data-hub/data-connector/src/app.module.ts index 8662803..833ecd6 100644 --- a/service/data-hub/data-connector/src/app.module.ts +++ b/service/data-hub/data-connector/src/app.module.ts @@ -1,9 +1,20 @@ import { Module } from '@nestjs/common'; +import { EventEmitterModule } from '@nestjs/event-emitter'; +import { ScheduleModule } from '@nestjs/schedule'; import { AppController } from './app.controller'; import { AppService } from './app.service'; +import { PrismaModule } from './core/database/prisma.module'; +import { FeatureImporterModule } from './modules/feat-importers/feature-importer.module'; +import { ConfigModule } from './core/config/config.module'; @Module({ - imports: [], + imports: [ + ConfigModule, + PrismaModule.forRoot(), + ScheduleModule.forRoot(), + EventEmitterModule.forRoot(), + FeatureImporterModule, + ], controllers: [AppController], providers: [AppService], }) diff --git a/service/data-hub/data-connector/src/app.service.ts b/service/data-hub/data-connector/src/app.service.ts index 927d7cc..3b8018a 100644 --- a/service/data-hub/data-connector/src/app.service.ts +++ b/service/data-hub/data-connector/src/app.service.ts @@ -1,8 +1,169 @@ -import { Injectable } from '@nestjs/common'; +// import { TicketService } from '@dks/avicenna/api-feature-ticket-system'; +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, SchedulerRegistry } from '@nestjs/schedule'; +import { + Interval, + addDays, + eachDayOfInterval, + endOfDay, + format, + set, + startOfDay, +} from 'date-fns'; +import { concatMap, from, lastValueFrom, take, tap } from 'rxjs'; +import { ImporterService } from './modules/feat-importers/importer.service'; +import { chunk } from './utils'; @Injectable() export class AppService { - getHello(): string { - return 'Hello World!'; + private readonly logger = new Logger(AppService.name); + + @Cron('0 */5 2-20 * * *') + async fetchDataContinously() { + const now = new Date(); + const fetchInterval = { + start: startOfDay(addDays(now, -14)), + end: endOfDay(addDays(now, 7)), + }; + this.importer + .runImport(fetchInterval) + .pipe( + take(1), + tap(() => + this.logger.log( + `import took ${(Date.now() - now.getTime()) / 1000}s`, + ), + ), + // switchMap(() => from(this.ticketService.createTickets(fetchInterval))), + ) + .subscribe(); + } + + // @Interval(5 * 60 * 1000) + async fetchDataInIntervals() { + const timeouts = this.schedulerRegistry.getTimeouts(); + this.logger.log(`Timeouts: ${timeouts.join(', ')}`); + if (timeouts.length) return; + + const now = new Date(); + const fetchInterval: Interval = { + start: set(new Date(), { + year: now.getFullYear(), + month: 0, + date: 1, + }), + end: endOfDay(now), + }; + + const intervals = [ + ...this.splitIntervalIntoPeriods(fetchInterval, 14), + ].reverse(); + + intervals.forEach((interval, index) => { + const readableInterval = `${format( + new Date(interval.start), + 'yyyy-MM-dd', + )} - ${format(new Date(interval.end), 'yyyy-MM-dd')}`; + const timeoutName = `Year: ${readableInterval}`; + const callback = () => { + this.logger.log(`Timeout "${timeoutName}"`); + from([interval]) + .pipe( + concatMap((interval) => this.importer.runImport(interval)), + // switchMap(() => from(this.ticketService.createTickets(interval))), + ) + .subscribe(); + }; + this.addTimeout(timeoutName, 60 * 1000 * index + 60 * 1000, callback); + }); + } + + // @Cron('0 2,7,12,17,22,27,32,37,42,47,52,57 2-20 * * *') + async fetchDataSixWeeks() { + const now = new Date(); + const fetchInterval: Interval = { + start: startOfDay(addDays(now, -90)), + end: endOfDay(now), + }; + + const intervals = this.splitIntervalIntoPeriods(fetchInterval, 14); + + await lastValueFrom( + from(intervals).pipe( + concatMap((interval, index) => + this.importer + .runImport(interval) + .pipe( + tap(() => + this.logger.log( + `${index + 1}. import three months took ${ + (Date.now() - now.getTime()) / 1000 + }s`, + ), + ), + ), + ), + ), + ); + // await this.ticketService.createTickets(fetchInterval); + } + + // @Cron('* * 0 * * *') + async fetchDataYearly() { + const now = new Date(); + const fetchInterval: Interval = { + start: set(new Date(), { + year: now.getFullYear(), + month: 0, + date: 1, + }), + end: endOfDay(now), + }; + + const intervals = this.splitIntervalIntoPeriods(fetchInterval); + + from(intervals) + .pipe( + concatMap((interval, index) => + this.importer + .runImport(interval) + .pipe( + tap(() => + this.logger.log( + `${index + 1}. import year took ${ + (Date.now() - now.getTime()) / 1000 + }s`, + ), + ), + ), + ), + // switchMap(() => from(this.ticketService.createTickets(fetchInterval))), + ) + .subscribe(); + } + + constructor( + private readonly importer: ImporterService, + // private readonly ticketService: TicketService, + private readonly schedulerRegistry: SchedulerRegistry, + ) {} + + private splitIntervalIntoPeriods( + interval: Interval, + chunkSize = 30, + ): Interval[] { + return chunk(eachDayOfInterval(interval), chunkSize).map((dates) => ({ + start: startOfDay(dates.at(0) as Date), + end: endOfDay(dates.at(-1) as Date), + })); + } + + private addTimeout( + name: string, + milliseconds: number, + callback: () => void, + ): void { + const timeout = setTimeout(callback, milliseconds); + this.schedulerRegistry.addTimeout(name, timeout); } } diff --git a/service/data-hub/data-connector/src/core/config/config.module.ts b/service/data-hub/data-connector/src/core/config/config.module.ts new file mode 100644 index 0000000..7a6b40f --- /dev/null +++ b/service/data-hub/data-connector/src/core/config/config.module.ts @@ -0,0 +1,11 @@ +import { Global, Module } from '@nestjs/common'; +import { ConfigModule as NestConfigModule } from '@nestjs/config'; +import { ConfigService } from './config.service'; + +@Global() +@Module({ + imports: [NestConfigModule.forRoot()], + providers: [ConfigService], + exports: [ConfigService], +}) +export class ConfigModule {} diff --git a/service/data-hub/data-connector/src/core/config/config.service.ts b/service/data-hub/data-connector/src/core/config/config.service.ts new file mode 100644 index 0000000..0b96c37 --- /dev/null +++ b/service/data-hub/data-connector/src/core/config/config.service.ts @@ -0,0 +1,27 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService as NestConfigService } from '@nestjs/config'; + +@Injectable() +export class ConfigService extends NestConfigService { + dispoliveBaseUrl(): string { + return this.get('DISPO_LIVE_BASE_URL'); + } + dispoliveUsername(): string { + return this.get('DISPO_LIVE_USERNAME'); + } + dispolivePassword(): string { + return this.get('DISPO_LIVE_PASSWORD'); + } + dyflexisV0BaseUrl(): string { + return this.get('DYFLEXIS_V0_BASE_URL'); + } + dyflexisV0ApiKey(): string { + return this.get('DYFLEXIS_V0_API_KEY'); + } + dyflexisV2BaseUrl(): string { + return this.get('DYFLEXIS_V2_BASE_URL'); + } + dyflexisV2ApiKey(): string { + return this.get('DYFLEXIS_V2_API_KEY'); + } +} diff --git a/service/data-hub/data-connector/src/core/database/constants.ts b/service/data-hub/data-connector/src/core/database/constants.ts new file mode 100644 index 0000000..0ef59f5 --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/constants.ts @@ -0,0 +1 @@ +export const PRISMA_SERVICE_OPTIONS = 'PRISMA_SERVICE_OPTIONS'; \ No newline at end of file diff --git a/service/data-hub/data-connector/src/core/database/pagination.interface.ts b/service/data-hub/data-connector/src/core/database/pagination.interface.ts new file mode 100644 index 0000000..9d579db --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/pagination.interface.ts @@ -0,0 +1,9 @@ +export interface PaginationInterface { + items: T[]; + meta: { + totalItems: number; + items: number; + totalPages: number; + page: number; + }; +} diff --git a/service/data-hub/data-connector/src/core/database/prisma-module.options.ts b/service/data-hub/data-connector/src/core/database/prisma-module.options.ts new file mode 100644 index 0000000..0085a10 --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/prisma-module.options.ts @@ -0,0 +1,51 @@ +import { ModuleMetadata, Type } from "@nestjs/common"; +import { Prisma } from "@prisma/client"; + +export interface PrismaModuleOptions { + /** + * If true, registers `PrismaModule` as a global module. + * See: https://docs.nestjs.com/modules#global-modules + */ + isGlobal?: boolean; + prismaServiceOptions?: PrismaServiceOptions; +} + +export interface PrismaServiceOptions { + /** + * Pass options directly to the `PrismaClient`. + * See: https://www.prisma.io/docs/reference/api-reference/prisma-client-reference/#prismaclient + */ + prismaOptions?: Prisma.PrismaClientOptions; + + /** + * If true, `PrismaClient` explicitly creates a connection pool and your first query will respond instantly. + * + * For most use cases the lazy connect behavior of `PrismaClient` will do. The first query of `PrismaClient` creates the connection pool. + * See: https://www.prisma.io/docs/concepts/components/prisma-client/working-with-prismaclient/connection-management + */ + explicitConnect?: boolean; + + /** + * Apply Prisma middlewares to perform actions before or after db queries. + * + * See: https://www.prisma.io/docs/concepts/components/prisma-client/middleware + */ + middlewares?: Array; +} + +export interface PrismaOptionsFactory { + createPrismaOptions(): Promise | PrismaServiceOptions; +} + +export interface PrismaModuleAsyncOptions + extends Pick { + isGlobal?: boolean; + useExisting?: Type; + useClass?: Type; + useFactory?: ( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...args: any[] + ) => Promise | PrismaServiceOptions; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inject?: any[]; +} diff --git a/service/data-hub/data-connector/src/core/database/prisma.module.ts b/service/data-hub/data-connector/src/core/database/prisma.module.ts new file mode 100644 index 0000000..086b550 --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/prisma.module.ts @@ -0,0 +1,100 @@ +import { DynamicModule, Module, Provider } from '@nestjs/common'; +import { PRISMA_SERVICE_OPTIONS } from './constants'; +import { + PrismaModuleAsyncOptions, + PrismaModuleOptions, + PrismaOptionsFactory, +} from './prisma-module.options'; +import { PrismaService } from './prisma.service'; + +@Module({ + providers: [PrismaService], + exports: [PrismaService], +}) +export class PrismaModule { + static forRoot(options: PrismaModuleOptions = {}): DynamicModule { + return { + global: options.isGlobal, + module: PrismaModule, + providers: [ + { + provide: PRISMA_SERVICE_OPTIONS, + useValue: options.prismaServiceOptions, + }, + ], + }; + } + + static forRootAsync(options: PrismaModuleAsyncOptions): DynamicModule { + return { + global: options.isGlobal, + module: PrismaModule, + imports: options.imports || [], + providers: this.createAsyncProviders(options), + }; + } + + private static createAsyncProviders( + options: PrismaModuleAsyncOptions, + ): Provider[] { + if (options.useExisting || options.useFactory) { + return this.createAsyncOptionsProvider(options); + } + + if (options.useClass) { + return [ + ...this.createAsyncOptionsProvider(options), + { + provide: options.useClass, + useClass: options.useClass, + }, + ]; + } + return [...this.createAsyncOptionsProvider(options)]; + } + + private static createAsyncOptionsProvider( + options: PrismaModuleAsyncOptions, + ): Provider[] { + if (options.useFactory) { + return [ + { + provide: PRISMA_SERVICE_OPTIONS, + useFactory: options.useFactory, + inject: options.inject || [], + }, + ]; + } + + if (options.useExisting) { + return [ + { + provide: PRISMA_SERVICE_OPTIONS, + useFactory: async (optionsFactory: PrismaOptionsFactory) => + await optionsFactory.createPrismaOptions(), + inject: [options.useExisting], + }, + ]; + } + + if (options.useClass) { + return [ + { + provide: PRISMA_SERVICE_OPTIONS, + useFactory: async (optionsFactory: PrismaOptionsFactory) => + await optionsFactory.createPrismaOptions(), + inject: [options.useClass], + }, + ]; + } + + return [ + { + provide: PRISMA_SERVICE_OPTIONS, + useFactory: async (optionsFactory: PrismaOptionsFactory) => + await optionsFactory.createPrismaOptions(), + inject: [], + }, + ]; + } +} diff --git a/service/data-hub/data-connector/src/core/database/prisma.service.spec.ts b/service/data-hub/data-connector/src/core/database/prisma.service.spec.ts new file mode 100644 index 0000000..9e45f1b --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/prisma.service.spec.ts @@ -0,0 +1,18 @@ +import { Test } from '@nestjs/testing'; +import { PrismaService } from './prisma.service'; + +describe('PrismaService', () => { + let service: PrismaService; + + beforeEach(async () => { + const module = await Test.createTestingModule({ + providers: [PrismaService], + }).compile(); + + service = module.get(PrismaService); + }); + + it('should be defined', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/service/data-hub/data-connector/src/core/database/prisma.service.ts b/service/data-hub/data-connector/src/core/database/prisma.service.ts new file mode 100644 index 0000000..f473edc --- /dev/null +++ b/service/data-hub/data-connector/src/core/database/prisma.service.ts @@ -0,0 +1,62 @@ +import { + Inject, + Injectable, + Logger, + OnModuleInit, + Optional, +} from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { PrismaClientOptions } from '@prisma/client/runtime/library'; + +import { PRISMA_SERVICE_OPTIONS } from './constants'; +import { PrismaServiceOptions } from './prisma-module.options'; + +@Injectable() +export class PrismaService extends PrismaClient implements OnModuleInit { + private readonly logger = new Logger('Prisma'); + + constructor( + @Optional() + @Inject(PRISMA_SERVICE_OPTIONS) + private readonly prismaServiceOptions: PrismaServiceOptions = {}, + ) { + super({ + ...prismaServiceOptions.prismaOptions, + // Make sure queries are always emitted as events, so that we can log them. + log: [ + { emit: 'event', level: 'query' }, + ...(prismaServiceOptions.prismaOptions?.log + ? prismaServiceOptions.prismaOptions.log + : (['info', 'warn', 'error'] as PrismaClientOptions['log'])), + ], + }); + + if (this.prismaServiceOptions.middlewares) { + this.prismaServiceOptions.middlewares.forEach((middleware) => + this.$use(middleware), + ); + } + + /* + Sadly typescript does not recognize the Prisma Client configuration passed in the super call. + As Prisma defines the events generically, based on the configuration, the event type, typescript inferred, is wrong. + */ + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + //@ts-ignore + this.$on('query', (e: QueryEvent) => { + const { timestamp, query, params, duration } = e; + const message = `${timestamp.toISOString()} ${query} ${params} ${duration}`; + // this.logger.log(message); + }); + } + + async onModuleInit() { + if (this.prismaServiceOptions.explicitConnect) { + await this.$connect(); + } + } + + // async enableShutdownHooks(app: INestApplication) { + // this.$on('beforeExit', async () => await app.close()); + // } +} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/data-access-extractors.module.ts b/service/data-hub/data-connector/src/modules/feat-extractors/data-access-extractors.module.ts new file mode 100644 index 0000000..7119790 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/data-access-extractors.module.ts @@ -0,0 +1,19 @@ +import { HttpModule } from '@nestjs/axios'; +import { Module } from '@nestjs/common'; +import { PrismaModule } from 'src/core/database/prisma.module'; +import { DispoLiveExtractorsService } from './dispo-live-extractors.service'; +import { DyflexisExtractorsService } from './dyflexis-extractors.service'; +import { ExtractorsService } from './extractors.service'; +import { SourceRepository } from './source.repository'; + +@Module({ + imports: [PrismaModule, HttpModule], + providers: [ + ExtractorsService, + DispoLiveExtractorsService, + DyflexisExtractorsService, + SourceRepository, + ], + exports: [ExtractorsService], +}) +export class DataAccessExtractorsModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/dispo-live-extractors.service.ts b/service/data-hub/data-connector/src/modules/feat-extractors/dispo-live-extractors.service.ts new file mode 100644 index 0000000..56531a1 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/dispo-live-extractors.service.ts @@ -0,0 +1,481 @@ +import { HttpService } from '@nestjs/axios'; +import { Injectable, Logger } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { + Interval, + eachDayOfInterval, + endOfDay, + format, + startOfDay, +} from 'date-fns'; +import { + Observable, + combineLatest, + concatMap, + delay, + forkJoin, + from, + map, + of, + reduce, + switchMap, +} from 'rxjs'; +import { EmployeeDispoLiveRaw, PatientRaw, TourRaw } from './source.interface'; +import { SourceRepository } from './source.repository'; +import { chunk } from 'src/utils'; +import { ConfigService } from 'src/core/config/config.service'; + +@Injectable() +export class DispoLiveExtractorsService { + private readonly logger = new Logger(DispoLiveExtractorsService.name); + + constructor( + private readonly http: HttpService, + private readonly repository: SourceRepository, + private readonly config: ConfigService, + ) {} + + extractTours( + interval: Interval, + jobId: string, + ): Observable { + const intervals = chunk(eachDayOfInterval(interval), 30).map((dates) => ({ + start: startOfDay(dates.at(0)), + end: endOfDay(dates.at(-1)), + })); + + return this.fetchTours(intervals).pipe( + map((rawTours) => { + return rawTours; + }), + switchMap((rawTours) => + from(this.repository.createTours(rawTours, jobId)), + ), + ); + } + + private fetchTours(intervals: Interval[]): Observable { + const archiveEndpoint = 'custom/getArchive/getArchive'; + const carsEndpoint = 'data/stammdatenFahrzeuge'; + + const fetchCars = ( + authCookies: string, + ): Observable<{ id: string; name: string }[]> => + this.http + .post<{ data: { _id: string; name: string }[] }>( + `${this.config.dispoliveBaseUrl()}/${carsEndpoint}`, + {}, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.data.map(({ _id, name }) => ({ id: _id, name })), + ), + ); + + const fetchToursInInterval = (interval: Interval, authCookies: string) => + this.http + .post( + `${this.config.dispoliveBaseUrl()}/${archiveEndpoint}`, + { + dateFrom: format(new Date(interval.start), 'yyyy-MM-dd'), + dateTo: format(new Date(interval.end), 'yyyy-MM-dd'), + }, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.map((tour) => { + const { + _id, + check, + createTime, + empfangenVonId, + endgen, + serienId, + serienKategorName, + serienKategorie, + zadStatus, + abrDate, + abrNum, + abrTime, + ausrueckZeit, + startTime, + startDate, + date, + date_iso, + dauergenehmigung, + direction, + distanceValue, + besetztKm, + gefahren, + gesamtPreis, + infektion, + itDesc, + itKrz, + itName, + kkId, + krankenkasse, + patId, + patName, + patSurname, + patStreet, + patZip, + patCity, + startInstitution, + startStreet, + startZip, + startCity, + targetInstitution, + targetStreet, + targetCity, + targetZip, + transportKrz, + transportName, + transportart, + verordnungsId, + verordnungsName, + type, + startBegin, + startEnd, + frei, + ziel, + zielBegin, + endTime, + endDate, + einsatzNummer, + carName, + carNo, + beifahrer, + beifahrerName, + fahrer, + fahrerName, + color, + } = tour; + return { + id: _id ? `${_id}` : undefined, + check: check ? `${check}` : undefined, + createTime: createTime ? `${createTime}` : undefined, + empfangenVonId: empfangenVonId + ? `${empfangenVonId}` + : undefined, + endgen: endgen ? `${endgen}` : undefined, + serienId: serienId ? `${serienId}` : undefined, + serienKategorName: serienKategorName + ? `${serienKategorName}` + : undefined, + serienKategorie: serienKategorie + ? `${serienKategorie}` + : undefined, + zadStatus: zadStatus ? `${zadStatus}` : undefined, + abrDate: abrDate ? `${abrDate}` : undefined, + abrNum: abrNum ? `${abrNum}` : undefined, + abrTime: abrTime ? `${abrTime}` : undefined, + ausrueckZeit: ausrueckZeit ? `${ausrueckZeit}` : undefined, + startTime: startTime ? `${startTime}` : undefined, + startDate: startDate ? `${startDate}` : undefined, + date: date ? `${date}` : undefined, + date_iso: date_iso ? `${date_iso}` : undefined, + dauergenehmigung: dauergenehmigung + ? `${dauergenehmigung}` + : undefined, + direction: direction ? `${direction}` : undefined, + distanceValue: distanceValue ? `${distanceValue}` : undefined, + besetztKm: besetztKm ? `${besetztKm}` : undefined, + gefahren: gefahren ? `${gefahren}` : undefined, + gesamtPreis: gesamtPreis ? `${gesamtPreis}` : undefined, + infektion: infektion ? `${infektion}` : undefined, + itDesc: itDesc ? `${itDesc}` : undefined, + itKrz: itKrz ? `${itKrz}` : undefined, + itName: itName ? `${itName}` : undefined, + kkId: kkId ? `${kkId}` : undefined, + krankenkasse: krankenkasse ? `${krankenkasse}` : undefined, + patId: patId ? `${patId}` : undefined, + patName: patName ? `${patName}` : undefined, + patSurname: patSurname ? `${patSurname}` : undefined, + patStreet: patStreet ? `${patStreet}` : undefined, + patZip: patZip ? `${patZip}` : undefined, + patCity: patCity ? `${patCity}` : undefined, + startInstitution: startInstitution + ? `${startInstitution}` + : undefined, + startStreet: startStreet ? `${startStreet}` : undefined, + startZip: startZip ? `${startZip}` : undefined, + startCity: startCity ? `${startCity}` : undefined, + targetInstitution: targetInstitution + ? `${targetInstitution}` + : undefined, + targetStreet: targetStreet ? `${targetStreet}` : undefined, + targetCity: targetCity ? `${targetCity}` : undefined, + targetZip: targetZip ? `${targetZip}` : undefined, + transportKrz: transportKrz ? `${transportKrz}` : undefined, + transportName: transportName ? `${transportName}` : undefined, + transportart: transportart ? `${transportart}` : undefined, + verordnungsId: verordnungsId ? `${verordnungsId}` : undefined, + verordnungsName: verordnungsName + ? `${verordnungsName}` + : undefined, + type: type ? `${type}` : undefined, + startBegin: startBegin ? `${startBegin}` : undefined, + startEnd: startEnd ? `${startEnd}` : undefined, + frei: frei ? `${frei}` : undefined, + ziel: ziel ? `${ziel}` : undefined, + zielBegin: zielBegin ? `${zielBegin}` : undefined, + endTime: endTime ? `${endTime}` : undefined, + endDate: endDate ? `${endDate}` : undefined, + einsatzNummer: einsatzNummer ? `${einsatzNummer}` : undefined, + carName: carName ? `${carName}` : undefined, + carNo: carNo ? `${carNo}` : undefined, + beifahrer: beifahrer ? `${beifahrer}` : undefined, + beifahrerName: beifahrerName ? `${beifahrerName}` : undefined, + fahrer: fahrer ? `${fahrer}` : undefined, + fahrerName: fahrerName ? `${fahrerName}` : undefined, + color: color ? `${color}` : undefined, + }; + }), + ), + ); + + return this.dispoliveLogin( + this.config.dispoliveUsername(), + this.config.dispolivePassword(), + ).pipe( + switchMap((authCookies) => + forkJoin([of(authCookies), fetchCars(authCookies)]), + ), + switchMap(([authCookies, cars]) => + from(intervals).pipe( + delay(300), + concatMap((interval) => fetchToursInInterval(interval, authCookies)), + reduce((full, tours) => [...full, ...tours], []), + map((tours) => + tours.map(({ carNo, ...tour }) => { + return { + ...tour, + carNo, + carName: + cars.find(({ id }) => id === carNo)?.name ?? tour.carName, + }; + }), + ), + ), + ), + ); + } + + extractEmployees(jobId: string): Observable { + return this.dispoliveLogin( + this.config.dispoliveUsername(), + this.config.dispolivePassword(), + ).pipe( + switchMap((authCookies) => this.fetchEmployees(authCookies)), + switchMap((rawEmployees) => + from(this.repository.createEmployeesDispoLive(rawEmployees, jobId)), + ), + ); + } + + private fetchEmployees( + authCookies: string, + ): Observable { + const employeeEndpoint = 'data/mitarbeiter'; + + return this.http + .post<{ data: EmployeeDispoLiveRaw[] }>( + `${this.config.dispoliveBaseUrl()}/${employeeEndpoint}`, + { sort: { surname: 1, status_name: -1 } }, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.data.map((employee) => { + const { + _id, + pScheinAblaufDatum, + name, + surname, + persId, + importPersId, + qualifikation, + } = employee; + return { + id: _id, + pScheinAblaufDatum, + name, + surname, + persId, + importPersId, + qualifikation, + }; + }), + ), + ); + } + + extractPatients(jobId: string): Observable { + return this.dispoliveLogin( + this.config.dispoliveUsername(), + this.config.dispolivePassword(), + ).pipe( + switchMap((authCookies) => this.fetchPatients(authCookies)), + switchMap((rawPatients) => + from(this.repository.createPatients(rawPatients, jobId)), + ), + ); + } + + private fetchPatients(authCookies: string): Observable { + const patientEndpoint = 'data/patientenStammdaten'; + const careDegreeEndpoint = 'data/pflegestufe'; + const disabilityMarkEndpoint = 'data/schwerbehindertenausweis'; + + const fetchCareDegrees = ( + authCookies: string, + ): Observable<{ id: string; careDegree: string }[]> => + this.http + .post<{ data: { _id: string; pflegestufe: string }[] }>( + `${this.config.dispoliveBaseUrl()}/${careDegreeEndpoint}`, + {}, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.data.map(({ _id, pflegestufe }) => ({ + id: _id, + careDegree: pflegestufe, + })), + ), + ); + const fetchDisabilityMarks = ( + authCookies: string, + ): Observable<{ id: string; disabilityMark: string }[]> => + this.http + .post<{ data: { _id: string; behinderungShortcut: string }[] }>( + `${this.config.dispoliveBaseUrl()}/${disabilityMarkEndpoint}`, + {}, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.data.map(({ _id, behinderungShortcut }) => ({ + id: _id, + disabilityMark: behinderungShortcut, + })), + ), + ); + + return combineLatest([ + fetchCareDegrees(authCookies), + fetchDisabilityMarks(authCookies), + ]).pipe( + switchMap(([careDegrees, disabilityMarks]) => + this.http + .post<{ data: PatientRaw[] }>( + `${this.config.dispoliveBaseUrl()}/${patientEndpoint}`, + { sort: { surname: 1, status_name: -1 } }, + { + headers: { + 'Content-Type': 'application/json', + Cookie: authCookies, + }, + }, + ) + .pipe( + map((response) => + response.data.data.map((patient) => { + const { + _id, + name, + surname, + street, + zip, + city, + birthday, + kkId, + krankenkasse, + pflegestufe, + behinderung1, + behinderung2, + } = patient; + return { + id: _id, + name, + surname, + street, + zip, + city, + birthday, + kkId, + krankenkasse, + pflegestufe: careDegrees.find(({ id }) => id === pflegestufe) + ?.careDegree, + behinderung1: disabilityMarks.find( + ({ id }) => id === behinderung1, + )?.disabilityMark, + behinderung2: disabilityMarks.find( + ({ id }) => id === behinderung2, + )?.disabilityMark, + }; + }), + ), + ), + ), + ); + } + + private dispoliveLogin( + username: string, + password: string, + ): Observable { + const endpoint = 'login'; + + return this.http + .post( + `${this.config.dispoliveBaseUrl()}/${endpoint}`, + { + username, + password, + }, + { + headers: { + 'Content-Type': 'application/json', + }, + }, + ) + .pipe( + map((response) => + response.headers['set-cookie'] + // To retrieve only the necessary information of the 'set-cookie' header, we combine them with the same separator, which is used inside one 'set-cookie' request (e.g "key=value; path=/; httponly") + .join(';') + .split(';') + .filter((cookie) => cookie.indexOf('koa.sid') > -1) + .join('; '), + ), + ); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/dyflexis-extractors.service.ts b/service/data-hub/data-connector/src/modules/feat-extractors/dyflexis-extractors.service.ts new file mode 100644 index 0000000..1edab87 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/dyflexis-extractors.service.ts @@ -0,0 +1,343 @@ +import { HttpService } from '@nestjs/axios'; +import { Injectable } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { + Interval, + eachDayOfInterval, + eachMonthOfInterval, + format, + getMonth, + getYear, +} from 'date-fns'; +import { + EMPTY, + Observable, + catchError, + concatMap, + expand, + from, + map, + of, + reduce, + switchMap, +} from 'rxjs'; +import { + AttendanceRegistrationRaw, + EmployeeDyflexisRaw, + PlannedTimeRaw, + WorkTimeRaw, +} from './source.interface'; +import { SourceRepository } from './source.repository'; +import { ConfigService } from 'src/core/config/config.service'; + +@Injectable() +export class DyflexisExtractorsService { + constructor( + private readonly http: HttpService, + private readonly repository: SourceRepository, + private readonly config: ConfigService, + ) {} + + extractEmployees(jobId: string): Observable { + return this.fetchEmployees().pipe( + switchMap((rawEmployees) => + from(this.repository.createEmployeesDyflexis(rawEmployees, jobId)), + ), + ); + } + + fetchEmployees(): Observable { + const endpoint = '/employee'; + + const getEmployeesPage = (pageNumber: number) => + this.http + .get<{ response: { data: EmployeeDyflexisRaw[] } }>( + `${this.config.dyflexisV0BaseUrl()}${endpoint}?page=${pageNumber}`, + { + headers: { + 'Content-Type': 'application/json', + 'X-Dyflexis-AuthToken': `${this.config.dyflexisV0ApiKey()}`, + }, + }, + ) + .pipe( + map((response) => + response.data.response.data.map((employee) => { + const { + id, + firstname, + surname, + contract_id, + contract_start, + contract_end, + contract_hours_week, + contract_salary_hour, + } = employee; + return { + id, + firstname, + surname, + contract_id, + contract_start, + contract_end, + contract_hours_week, + contract_salary_hour, + }; + }), + ), + ); + + return new Observable((observer) => + getEmployeesPage(0) + .pipe( + expand((data, i) => (data.length ? getEmployeesPage(i + 1) : EMPTY)), + reduce((acc, data) => [...acc, ...data], []), + catchError((error) => { + observer.error(error); + return of(error); + }), + ) + .subscribe((employees) => { + observer.next(employees); + observer.complete(); + }), + ); + } + + extractAttendanceRegistrations( + interval: Interval, + jobId: string, + ): Observable { + const daysInInterval = eachDayOfInterval(interval); + return from(daysInInterval).pipe( + concatMap((day) => this.fetchAttendanceRegistrationsForDay(day)), + reduce( + (full, attendanceRegistrations) => [ + ...full, + ...attendanceRegistrations, + ], + [], + ), + switchMap((rawAttendanceRegistrations) => + from( + this.repository.createAttendanceRegistrations( + rawAttendanceRegistrations, + jobId, + ), + ), + ), + ); + } + + fetchAttendanceRegistrationsForDay( + date: Date | string, + ): Observable { + const endpoint = '/attendance-registration'; + + return this.http + .get<{ attendanceRegistration: AttendanceRegistrationRaw[] }>( + `${this.config.dyflexisV2BaseUrl()}${endpoint}/${format( + new Date(date), + 'yyyy-MM-dd', + )}`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Token ${this.config.dyflexisV2ApiKey()}`, + }, + }, + ) + .pipe( + map((response) => + response.data.attendanceRegistration.map((attendanceRegistration) => { + const { + attendanceRegistrationId, + employeeId, + personnelNumber, + dateTime, + event, + } = attendanceRegistration; + return { + attendanceRegistrationId: attendanceRegistrationId + ? `${attendanceRegistrationId}` + : undefined, + employeeId: employeeId ? `${employeeId}` : undefined, + personnelNumber: personnelNumber + ? `${personnelNumber}` + : undefined, + dateTime: dateTime ? `${dateTime}` : undefined, + event: event ? `${event}` : undefined, + }; + }), + ), + ); + } + + extractWorkTimes( + interval: Interval, + jobId: string, + ): Observable { + const monthsInInterval = eachMonthOfInterval(interval); + + return from(monthsInInterval).pipe( + concatMap((month) => + this.fetchWorkTimeForMonth(getMonth(month) + 1, getYear(month)), + ), + reduce((full, workTimes) => [...full, ...workTimes], []), + switchMap((rawWorkTimes) => + from(this.repository.createWorkTimes(rawWorkTimes, jobId)), + ), + ); + } + + private fetchWorkTimeForMonth( + month: number, + year: number, + ): Observable { + const endpoint = '/worked'; + + const getWorkTimePage = (pageNumber: number) => + this.http + .get<{ response: { data: WorkTimeRaw[] } }>( + `${this.config.dyflexisV0BaseUrl()}${endpoint}?month=${month}&year=${year}&page=${pageNumber}`, + { + headers: { + 'Content-Type': 'application/json', + 'X-Dyflexis-AuthToken': `${this.config.dyflexisV0ApiKey()}`, + }, + }, + ) + .pipe( + map((response) => + response.data.response.data.map((workTime) => { + const { + id, + user_id, + firstname, + surname, + department_id, + department_name, + start_date, + end_date, + pauze, + duration, + } = workTime; + return { + id, + user_id, + firstname, + surname, + department_id, + department_name, + start_date, + end_date, + pauze, + duration, + }; + }), + ), + ); + return new Observable((observer) => + getWorkTimePage(0) + .pipe( + expand((data, i) => (data.length ? getWorkTimePage(i + 1) : EMPTY)), + reduce((acc, data) => [...acc, ...data], []), + catchError((error) => { + observer.error(error); + return of(error); + }), + ) + .subscribe((workTimes) => { + observer.next(workTimes); + observer.complete(); + }), + ); + } + + extractPlannedTimes( + interval: Interval, + jobId: string, + ): Observable { + const monthsInInterval = eachMonthOfInterval(interval); + + return from(monthsInInterval).pipe( + concatMap((month) => + this.fetchPlannedTimeForMonth(getMonth(month) + 1, getYear(month)), + ), + reduce((full, plannedTimes) => [...full, ...plannedTimes], []), + switchMap((rawPlannedTimes) => + from(this.repository.createPlannedTimes(rawPlannedTimes, jobId)), + ), + ); + } + + private fetchPlannedTimeForMonth( + month: number, + year: number, + ): Observable { + const endpoint = '/planned'; + + const getPlannedTimePage = (pageNumber: number) => + this.http + .get<{ response: { data: PlannedTimeRaw[] } }>( + `${this.config.dyflexisV0BaseUrl()}${endpoint}?month=${month}&year=${year}&page=${pageNumber}`, + { + headers: { + 'Content-Type': 'application/json', + 'X-Dyflexis-AuthToken': `${this.config.dyflexisV0ApiKey()}`, + }, + }, + ) + .pipe( + map((response) => + response.data.response.data.map((plannedTime) => { + const { + id, + user_id, + firstname, + surname, + department_id, + department_name, + start_date, + end_date, + pauze, + duration, + deleted, + mark, + } = plannedTime; + return { + id, + user_id, + firstname, + surname, + department_id, + department_name, + start_date, + end_date, + pauze, + duration, + deleted, + mark, + }; + }), + ), + ); + return new Observable((observer) => + getPlannedTimePage(0) + .pipe( + expand((data, i) => + data.length ? getPlannedTimePage(i + 1) : EMPTY, + ), + reduce((acc, data) => [...acc, ...data], []), + catchError((error) => { + observer.error(error); + return of(error); + }), + ) + .subscribe((plannedTimes) => { + observer.next(plannedTimes); + observer.complete(); + }), + ); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/extractors.service.ts b/service/data-hub/data-connector/src/modules/feat-extractors/extractors.service.ts new file mode 100644 index 0000000..5ea0651 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/extractors.service.ts @@ -0,0 +1,141 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { + AttendanceRegistrationSource, + EmployeeDispoLiveSource, + EmployeeDyflexisSource, + PatientSource, + PlannedTimeSource, + Prisma, + TourSource, + WorkTimeSource, +} from '@prisma/client'; +import { Interval } from 'date-fns'; +import { Observable, catchError, combineLatest, map } from 'rxjs'; +import { DispoLiveExtractorsService } from './dispo-live-extractors.service'; +import { DyflexisExtractorsService } from './dyflexis-extractors.service'; +import { SourceRepository } from './source.repository'; + +@Injectable() +export class ExtractorsService { + private logger = new Logger(ExtractorsService.name); + + constructor( + private readonly dispoLiveExtractors: DispoLiveExtractorsService, + private readonly dyflexisExtractors: DyflexisExtractorsService, + private readonly repository: SourceRepository, + ) {} + + clearSource(): Promise { + return this.repository.clearSource(); + } + + extractTours( + interval: Interval, + jobId: string, + ): Observable { + return this.dispoLiveExtractors.extractTours(interval, jobId).pipe( + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + extractEmployees(jobId: string): Observable { + return combineLatest([ + this.dispoLiveExtractors.extractEmployees(jobId), + this.dyflexisExtractors.extractEmployees(jobId), + ]).pipe( + map(([{ count: countDispoLive }, { count: countDyflexis }]) => { + return { + count: countDispoLive + countDyflexis, + }; + }), + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + extractPatients(jobId: string): Observable { + return this.dispoLiveExtractors.extractPatients(jobId).pipe( + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + extractPlannedTimes( + interval: Interval, + jobId: string, + ): Observable { + return this.dyflexisExtractors.extractPlannedTimes(interval, jobId).pipe( + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + extractWorkTimes( + interval: Interval, + jobId: string, + ): Observable { + return this.dyflexisExtractors.extractWorkTimes(interval, jobId).pipe( + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + extractAttendanceRegistrations( + interval: Interval, + jobId: string, + ): Observable { + return this.dyflexisExtractors + .extractAttendanceRegistrations(interval, jobId) + .pipe( + catchError((error) => { + this.logger.error(error); + throw error; + }), + ); + } + + findToursByJobId(jobId: string): Promise { + return this.repository.findToursByJobId(jobId); + } + + findPatientsByJobId(jobId: string): Promise { + return this.repository.findPatientsByJobId(jobId); + } + + findPlannedTimesByJobId(jobId: string): Promise { + return this.repository.findPlannedTimesByJobId(jobId); + } + + findWorkTimesByJobId(jobId: string): Promise { + return this.repository.findWorkTimesByJobId(jobId); + } + + findAttendanceRegistrationsByJobId( + jobId: string, + ): Promise { + return this.repository.findAttendanceRegistrationsByJobId(jobId); + } + + findEmployeesDispoLiveByJobId( + jobId: string, + ): Promise { + return this.repository.findEmployeesDispoLiveByJobId(jobId); + } + + findEmployeesDyflexisByJobId( + jobId: string, + ): Promise { + return this.repository.findEmployeesDyflexisByJobId(jobId); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/source.interface.ts b/service/data-hub/data-connector/src/modules/feat-extractors/source.interface.ts new file mode 100644 index 0000000..e87a50e --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/source.interface.ts @@ -0,0 +1,142 @@ +export interface TourRaw { + _id?: string; + check?: string; + createTime?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' OR yyyy-MM-dd 'Europe/Berlin' + empfangenVonId?: string; + endgen?: string; // yyyy-MM-dd + serienId?: string; + serienKategorName?: string; + serienKategorie?: string; + zadStatus?: string; + abrDate?: string; // yyyy-MM-dd + abrNum?: string; + abrTime?: string; // HH:mm + + ausrueckZeit?: string; // HH:mm 'Europe/Berlin' + startTime?: string; // HH:mm 'Europe/Berlin' (is after ausrueckZeit) + startDate?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + date?: string; // yyyy-MM-dd + date_iso?: string; // ISO 8601 String + dauergenehmigung?: string; + direction?: string; + distanceValue?: string; // meters (probably int) + besetztKm?: string; // kilometers (float) + gefahren?: string; // Boolean as number + gesamtPreis?: string; // float + infektion?: string; // Boolean as number + itDesc?: string; + itKrz?: string; + itName?: string; + kkId?: string; + krankenkasse?: string; + patId?: string; + patName?: string; + patSurname?: string; + patStreet?: string; + patZip?: string; + patCity?: string; + startInstitution?: string; + startStreet?: string; + startZip?: string; + startCity?: string; + targetInstitution?: string; + targetStreet?: string; + targetZip?: string; + targetCity?: string; + transportKrz?: string; + transportName?: string; + transportart?: string; // ID + verordnungsId?: string; + verordnungsName?: string; + type?: string; + + startBegin?: string; // HH:mm 'Europe/Berlin' + startEnd?: string; // HH:mm 'Europe/Berlin' + frei?: string; // HH:mm 'Europe/Berlin' + ziel?: string; // HH:mm 'Europe/Berlin' + zielBegin?: string; // HH:mm 'Europe/Berlin' + endTime?: string; // HH:mm 'Europe/Berlin' + endDate?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + + einsatzNummer?: string; + carName?: string; + carNo?: string; // ID + beifahrerName?: string; + beifahrer?: string; // ID + fahrerName?: string; + fahrer?: string; // ID + + color?: string; // Hex rgb +} + +export interface EmployeeDispoLiveRaw { + _id?: string; + pScheinAblaufDatum?: string; // yyyy-MM-dd + name?: string; + surname?: string; + persId?: string; + importPersId?: string; // Personal ID aus Dyflexis + qualifikation?: string; // ID +} + +export interface PatientRaw { + _id?: string; + birthday?: string; // yyyy-MM-dd + name?: string; + surname?: string; + street?: string; + zip?: string; + city?: string; + kkId?: string; + krankenkasse?: string; + pflegestufe?: string; + behinderung1?: string; + behinderung2?: string; +} + +export interface AttendanceRegistrationRaw { + attendanceRegistrationId?: string; + employeeId?: string; + personnelNumber?: string; + dateTime?: string; // ISO 8601 String with timezone + event?: string; +} + +export interface PlannedTimeRaw { + id?: string; + user_id?: string; + firstname?: string; + surname?: string; + department_id?: string; + department_name?: string; + start_date?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + end_date?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + pauze?: string; + duration?: string; + deleted?: string; + mark?: string; +} + +export interface WorkTimeRaw { + id?: string; + user_id?: string; + firstname?: string; + surname?: string; + department_id?: string; + department_name?: string; + start_date?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + end_date?: string; // yyyy-MM-dd HH:mm:ss 'Europe/Berlin' + pauze?: string; + duration?: string; +} + +export interface EmployeeDyflexisRaw { + id?: string; + firstname?: string; + surname?: string; + contract_id?: string; + contract_start?: string; // yyyy-MM-dd + contract_end?: string; // yyyy-MM-dd + contract_hours_week?: string; // Float string + contract_salary_hour?: string; // Float string +} diff --git a/service/data-hub/data-connector/src/modules/feat-extractors/source.repository.ts b/service/data-hub/data-connector/src/modules/feat-extractors/source.repository.ts new file mode 100644 index 0000000..5f95997 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-extractors/source.repository.ts @@ -0,0 +1,163 @@ +import { Injectable } from '@nestjs/common'; +import { + AttendanceRegistrationSource, + EmployeeDispoLiveSource, + EmployeeDyflexisSource, + PatientSource, + PlannedTimeSource, + Prisma, + TourSource, + WorkTimeSource, +} from '@prisma/client'; +import { PrismaService } from 'src/core/database/prisma.service'; +import { + AttendanceRegistrationRaw, + EmployeeDispoLiveRaw, + EmployeeDyflexisRaw, + PatientRaw, + PlannedTimeRaw, + TourRaw, + WorkTimeRaw, +} from './source.interface'; + +@Injectable() +export class SourceRepository { + constructor(private readonly prisma: PrismaService) {} + + clearSource(): Promise { + return Promise.all([ + this.prisma.tourSource.deleteMany(), + this.prisma.employeeDispoLiveSource.deleteMany(), + this.prisma.employeeDyflexisSource.deleteMany(), + this.prisma.workTimeSource.deleteMany(), + this.prisma.plannedTimeSource.deleteMany(), + this.prisma.attendanceRegistrationSource.deleteMany(), + this.prisma.patientSource.deleteMany(), + ]); + } + + createTours( + tours: TourRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.tourSource.createMany({ + data: tours.map((data) => ({ jobId, ...data })), + }); + } + + createEmployeesDispoLive( + employees: EmployeeDispoLiveRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.employeeDispoLiveSource.createMany({ + data: employees.map((data) => ({ jobId, ...data })), + }); + } + + createPatients( + patients: PatientRaw[], + jobId: string, + ): Promise { + return this.prisma.patientSource.createMany({ + data: patients.map((data) => ({ jobId, ...data })), + }); + } + + createEmployeesDyflexis( + employees: EmployeeDyflexisRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.employeeDyflexisSource.createMany({ + data: employees.map((data) => ({ jobId, ...data })), + }); + } + + createAttendanceRegistrations( + attendanceRegistrations: AttendanceRegistrationRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.attendanceRegistrationSource.createMany({ + data: attendanceRegistrations.map((data) => ({ jobId, ...data })), + }); + } + + createWorkTimes( + workTimes: WorkTimeRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.workTimeSource.createMany({ + data: workTimes.map((data) => ({ jobId, ...data })), + }); + } + + createPlannedTimes( + plannedTimes: PlannedTimeRaw[], + jobId: string, + ): Prisma.PrismaPromise { + return this.prisma.plannedTimeSource.createMany({ + data: plannedTimes.map((data) => ({ jobId, ...data })), + }); + } + + findToursByJobId(jobId: string): Promise { + return this.prisma.tourSource.findMany({ + where: { + jobId, + }, + }); + } + + findPatientsByJobId(jobId: string): Promise { + return this.prisma.patientSource.findMany({ + where: { + jobId, + }, + }); + } + + findPlannedTimesByJobId(jobId: string): Promise { + return this.prisma.plannedTimeSource.findMany({ + where: { + jobId, + }, + }); + } + + findWorkTimesByJobId(jobId: string): Promise { + return this.prisma.workTimeSource.findMany({ + where: { + jobId, + }, + }); + } + + findAttendanceRegistrationsByJobId( + jobId: string, + ): Promise { + return this.prisma.attendanceRegistrationSource.findMany({ + where: { + jobId, + }, + }); + } + + findEmployeesDyflexisByJobId( + jobId: string, + ): Promise { + return this.prisma.employeeDyflexisSource.findMany({ + where: { + jobId, + }, + }); + } + + findEmployeesDispoLiveByJobId( + jobId: string, + ): Promise { + return this.prisma.employeeDispoLiveSource.findMany({ + where: { + jobId, + }, + }); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/events/event-types.enum.ts b/service/data-hub/data-connector/src/modules/feat-importers/events/event-types.enum.ts new file mode 100644 index 0000000..3c9c3e8 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/events/event-types.enum.ts @@ -0,0 +1,5 @@ +export enum EventTypes { + ExtractionDone = 'etl.extraction', + TransformationDone = 'etl.transformation', + LoadingDone = 'etl.loading', +} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/events/extraction-done.event.ts b/service/data-hub/data-connector/src/modules/feat-importers/events/extraction-done.event.ts new file mode 100644 index 0000000..27e7513 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/events/extraction-done.event.ts @@ -0,0 +1,3 @@ +export class ExtractionDoneEvent { + constructor(public readonly jobId: string) {} +} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/events/loading-done.event.ts b/service/data-hub/data-connector/src/modules/feat-importers/events/loading-done.event.ts new file mode 100644 index 0000000..eb63ce0 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/events/loading-done.event.ts @@ -0,0 +1,3 @@ +export class LoadingDoneEvent { + constructor(public readonly jobId: string) {} +} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/events/transformation-done.event.ts b/service/data-hub/data-connector/src/modules/feat-importers/events/transformation-done.event.ts new file mode 100644 index 0000000..bf9d398 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/events/transformation-done.event.ts @@ -0,0 +1,3 @@ +export class TransformationDoneEvent { + constructor(public readonly jobId: string) {} +} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/feature-importer.module.ts b/service/data-hub/data-connector/src/modules/feat-importers/feature-importer.module.ts new file mode 100644 index 0000000..2461c42 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/feature-importer.module.ts @@ -0,0 +1,17 @@ +import { Module } from '@nestjs/common'; + +import { DataAccessExtractorsModule } from '../feat-extractors/data-access-extractors.module'; +import { DataAccessLoadersModule } from '../feat-loaders/data-access-loaders.module'; +import { DataAccessTransformersModule } from '../feat-transformers/data-access-transformers.module'; +import { ImporterService } from './importer.service'; + +@Module({ + imports: [ + DataAccessExtractorsModule, + DataAccessTransformersModule, + DataAccessLoadersModule, + ], + providers: [ImporterService], + exports: [ImporterService], +}) +export class FeatureImporterModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-importers/importer.service.ts b/service/data-hub/data-connector/src/modules/feat-importers/importer.service.ts new file mode 100644 index 0000000..d4e8d19 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-importers/importer.service.ts @@ -0,0 +1,113 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { randomUUID } from 'crypto'; +import { Interval } from 'date-fns'; +import { Observable, combineLatest, from, map, switchMap, tap } from 'rxjs'; + +import { ExtractorsService } from '../feat-extractors/extractors.service'; +import { LoadersService } from '../feat-loaders/loaders.service'; +import { TransformersService } from '../feat-transformers/transformers.service'; +import { EventTypes } from './events/event-types.enum'; +import { ExtractionDoneEvent } from './events/extraction-done.event'; +import { LoadingDoneEvent } from './events/loading-done.event'; +import { TransformationDoneEvent } from './events/transformation-done.event'; + +@Injectable() +export class ImporterService { + private logger = new Logger(ImporterService.name); + constructor( + private readonly extractorsService: ExtractorsService, + private readonly transformersService: TransformersService, + private readonly loadersService: LoadersService, + private readonly eventEmitter: EventEmitter2, + ) {} + + runImport(interval: Interval) { + const startTS = new Date().getTime(); + const jobId = randomUUID(); + return from(this.extractorsService.clearSource()).pipe( + switchMap(() => this.runExtractors(interval, jobId)), + tap(() => + this.logger.log( + `Extraction of ${jobId} took ${new Date().getTime() - startTS}ms`, + ), + ), + switchMap(() => + from(this.transformersService.clearStaging()).pipe( + switchMap(() => this.runTransformers(jobId)), + ), + ), + tap(() => + this.logger.log( + `Transformation of ${jobId} took ${new Date().getTime() - startTS}ms`, + ), + ), + switchMap(() => this.runLoaders(jobId)), + tap(() => + this.logger.log( + `Loading of ${jobId} took ${new Date().getTime() - startTS}ms`, + ), + ), + ); + } + + private runExtractors(interval: Interval, jobId: string): Observable { + return combineLatest([ + this.extractorsService.extractAttendanceRegistrations(interval, jobId), + this.extractorsService.extractEmployees(jobId), + this.extractorsService.extractPatients(jobId), + this.extractorsService.extractPlannedTimes(interval, jobId), + this.extractorsService.extractTours(interval, jobId), + this.extractorsService.extractWorkTimes(interval, jobId), + ]).pipe( + tap(() => + this.eventEmitter.emit( + EventTypes.ExtractionDone, + new ExtractionDoneEvent(jobId), + ), + ), + map(() => { + return; + }), + ); + } + + private runTransformers(jobId: string): Observable { + return combineLatest([ + this.transformersService.transformAttendanceRegistrations(jobId), + this.transformersService.transformEmployees(jobId), + this.transformersService.transformPatients(jobId), + this.transformersService.transformPlannedTimes(jobId), + this.transformersService.transformWorkTimes(jobId), + this.transformersService.transformTours(jobId), + ]).pipe( + tap(() => + this.eventEmitter.emit( + EventTypes.TransformationDone, + new TransformationDoneEvent(jobId), + ), + ), + map(() => { + return; + }), + ); + } + + private runLoaders(jobId: string): Observable { + return this.loadersService.loadTours().pipe( + switchMap(() => this.loadersService.loadPatients()), + switchMap(() => this.loadersService.loadAttendanceRegistrations()), + switchMap(() => this.loadersService.loadPlannedTimes()), + switchMap(() => this.loadersService.loadWorkTimes()), + tap(() => + this.eventEmitter.emit( + EventTypes.LoadingDone, + new LoadingDoneEvent(jobId), + ), + ), + map(() => { + return; + }), + ); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/data-access-loaders.module.ts b/service/data-hub/data-connector/src/modules/feat-loaders/data-access-loaders.module.ts new file mode 100644 index 0000000..a9694f8 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/data-access-loaders.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; + +import { PrismaModule } from 'src/core/database/prisma.module'; +import { DataAccessTransformersModule } from '../feat-transformers/data-access-transformers.module'; +import { FinalRepository } from './final.repository'; +import { LoadersService } from './loaders.service'; + +@Module({ + imports: [PrismaModule, DataAccessTransformersModule], + providers: [LoadersService, FinalRepository], + exports: [LoadersService], +}) +export class DataAccessLoadersModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/final.repository.ts b/service/data-hub/data-connector/src/modules/feat-loaders/final.repository.ts new file mode 100644 index 0000000..a02427f --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/final.repository.ts @@ -0,0 +1,278 @@ +import { Injectable } from '@nestjs/common'; +import { + AttendanceRegistration, + Patient, + PlannedTime, + Prisma, + Tour, + WorkTime, +} from '@prisma/client'; +import { Interval } from 'date-fns'; + +import { PrismaService } from 'src/core/database/prisma.service'; + +@Injectable() +export class FinalRepository { + constructor(private readonly prisma: PrismaService) {} + + findToursInInterval( + interval: Interval, + operationIds: string[], + ): Promise { + return this.prisma.tour.findMany({ + where: { + OR: [ + { + AND: [ + { + startDate: { + gte: new Date(interval.start), + }, + }, + { + startDate: { + lte: new Date(interval.end), + }, + }, + ], + }, + { + operationId: { + in: operationIds, + }, + }, + ], + }, + }); + } + + createTours( + tourCreateInput: Prisma.TourCreateInput[], + ): Promise { + return this.prisma.tour.createMany({ + data: tourCreateInput, + skipDuplicates: true, + }); + } + + async updateTours( + tourCreateInput: Prisma.TourCreateInput[], + ): Promise { + await this.prisma.tour.deleteMany({ + where: { + operationId: { + in: tourCreateInput.map(({ operationId }) => operationId), + }, + }, + }); + return this.createTours(tourCreateInput); + } + + deleteTours(operationIds: string[]): Promise { + return this.prisma.tour.updateMany({ + where: { + operationId: { + in: operationIds, + }, + }, + data: { + deletedAt: new Date(), + }, + }); + } + + findPatients(): Promise { + return this.prisma.patient.findMany({ + orderBy: { + surname: 'asc', + }, + }); + } + + createPatients( + patientCreateInput: Prisma.PatientCreateInput[], + ): Promise { + return this.prisma.patient.createMany({ + data: patientCreateInput, + skipDuplicates: true, + }); + } + + async updatePatients( + patientCreateInput: Prisma.PatientCreateInput[], + ): Promise { + await this.deletePatients(patientCreateInput.map(({ id }) => id)); + return this.createPatients(patientCreateInput); + } + + deletePatients(ids: string[]): Promise { + return this.prisma.patient.deleteMany({ + where: { + id: { + in: ids, + }, + }, + }); + } + + findAttendanceRegistrationsInInterval( + interval: Interval, + ): Promise { + return this.prisma.attendanceRegistration.findMany({ + where: { + AND: [ + { + dateTime: { + gte: new Date(interval.start), + }, + }, + { + dateTime: { + lte: new Date(interval.end), + }, + }, + ], + }, + }); + } + + createAttendanceRegistrations( + attendanceRegistrationCreateInput: Prisma.AttendanceRegistrationCreateInput[], + ): Promise { + return this.prisma.attendanceRegistration.createMany({ + data: attendanceRegistrationCreateInput, + skipDuplicates: true, + }); + } + + async updateAttendanceRegistrations( + attendanceRegistrationCreateInput: Prisma.AttendanceRegistrationCreateInput[], + ): Promise { + await this.prisma.attendanceRegistration.deleteMany({ + where: { + attendanceRegistrationId: { + in: attendanceRegistrationCreateInput.map( + ({ attendanceRegistrationId }) => attendanceRegistrationId, + ), + }, + }, + }); + return this.createAttendanceRegistrations( + attendanceRegistrationCreateInput, + ); + } + + deleteAttendanceRegistrations( + attendanceRegistrationIds: number[], + ): Promise { + return this.prisma.attendanceRegistration.deleteMany({ + where: { + attendanceRegistrationId: { + in: attendanceRegistrationIds, + }, + }, + }); + } + + findPlannedTimesInInterval(interval: Interval): Promise { + return this.prisma.plannedTime.findMany({ + where: { + AND: [ + { + startDate: { + gte: new Date(interval.start), + }, + }, + { + startDate: { + lte: new Date(interval.end), + }, + }, + ], + }, + }); + } + + createPlannedTimes( + plannedTimeCreateInput: Prisma.PlannedTimeCreateInput[], + ): Promise { + return this.prisma.plannedTime.createMany({ + data: plannedTimeCreateInput, + skipDuplicates: true, + }); + } + + async updatePlannedTimes( + plannedTimeCreateInput: Prisma.PlannedTimeCreateInput[], + ): Promise { + await this.prisma.plannedTime.deleteMany({ + where: { + id: { + in: plannedTimeCreateInput.map(({ id }) => id), + }, + }, + }); + return this.createPlannedTimes(plannedTimeCreateInput); + } + + deletePlannedTimes(plannedTimeIds: string[]): Promise { + return this.prisma.plannedTime.deleteMany({ + where: { + id: { + in: plannedTimeIds, + }, + }, + }); + } + + findWorkTimesInInterval(interval: Interval): Promise { + return this.prisma.workTime.findMany({ + where: { + AND: [ + { + startDate: { + gte: new Date(interval.start), + }, + }, + { + startDate: { + lte: new Date(interval.end), + }, + }, + ], + }, + }); + } + + createWorkTimes( + workTimeCreateInput: Prisma.WorkTimeCreateInput[], + ): Promise { + return this.prisma.workTime.createMany({ + data: workTimeCreateInput, + skipDuplicates: true, + }); + } + + async updateWorkTimes( + workTimeCreateInput: Prisma.WorkTimeCreateInput[], + ): Promise { + await this.prisma.workTime.deleteMany({ + where: { + id: { + in: workTimeCreateInput.map(({ id }) => id), + }, + }, + }); + return this.createWorkTimes(workTimeCreateInput); + } + + deleteWorkTimes(workTimeIds: string[]): Promise { + return this.prisma.workTime.deleteMany({ + where: { + id: { + in: workTimeIds, + }, + }, + }); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders.service.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders.service.ts new file mode 100644 index 0000000..2b560f4 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders.service.ts @@ -0,0 +1,204 @@ +import { Injectable } from '@nestjs/common'; +import { from, map, of, switchMap } from 'rxjs'; + +import { TransformersService } from '../feat-transformers/transformers.service'; +import { FinalRepository } from './final.repository'; +import { AttendanceRegistrationLoader } from './loaders/attendance-registration.loader'; +import { PatientLoader } from './loaders/patient.loader'; +import { PlannedTimeLoader } from './loaders/planned-time.loader'; +import { TourLoader } from './loaders/tour.loader'; +import { WorkTimeLoader } from './loaders/work-time.loader'; + +@Injectable() +export class LoadersService { + constructor( + private readonly repository: FinalRepository, + private readonly transformersService: TransformersService, + ) {} + + loadTours() { + return from(this.transformersService.findTours()).pipe( + switchMap((stagingTours) => { + if (stagingTours.length === 0) { + return of({ + staging: [], + final: [], + }); + } + const { startDate: firstTourDate } = stagingTours.at(0); + const { startDate: lastTourDate } = stagingTours.at(-1); + return from( + this.repository.findToursInInterval( + { + start: firstTourDate, + end: lastTourDate, + }, + stagingTours.map(({ operationId }) => operationId), + ), + ).pipe( + map((finalTours) => ({ + final: finalTours, + staging: stagingTours, + })), + ); + }), + map(({ staging, final }) => + TourLoader.determineLoadOperations(staging, final), + ), + switchMap(({ deleted, updated, created }) => { + return from( + Promise.all([ + this.repository.deleteTours(deleted), + this.repository.updateTours(updated), + this.repository.createTours(created), + ]), + ); + }), + ); + } + + loadPatients() { + return from(this.transformersService.findPatients()).pipe( + switchMap((stagingPatients) => + from(this.repository.findPatients()).pipe( + map((finalPatients) => ({ + final: finalPatients, + staging: stagingPatients, + })), + ), + ), + map(({ staging, final }) => + PatientLoader.determineLoadOperations(staging, final), + ), + switchMap(({ deleted, updated, created }) => + from( + Promise.all([ + this.repository.deletePatients(deleted), + this.repository.updatePatients(updated), + this.repository.createPatients(created), + ]), + ), + ), + ); + } + + loadAttendanceRegistrations() { + return from(this.transformersService.findAttendanceRegistrations()).pipe( + switchMap((stagingAttendanceRegistrations) => { + if (stagingAttendanceRegistrations.length === 0) { + return of({ + staging: [], + final: [], + }); + } + const { dateTime: firstAttendanceRegistration } = [ + ...stagingAttendanceRegistrations, + ].shift(); + const { dateTime: lastAttendanceRegistration } = [ + ...stagingAttendanceRegistrations, + ].pop(); + return from( + this.repository.findAttendanceRegistrationsInInterval({ + start: firstAttendanceRegistration, + end: lastAttendanceRegistration, + }), + ).pipe( + map((finalAttendanceRegistrations) => ({ + final: finalAttendanceRegistrations, + staging: stagingAttendanceRegistrations, + })), + ); + }), + map(({ staging, final }) => + AttendanceRegistrationLoader.determineLoadOperations(staging, final), + ), + switchMap(({ deleted, updated, created }) => { + return from( + Promise.all([ + this.repository.deleteAttendanceRegistrations(deleted), + this.repository.updateAttendanceRegistrations(updated), + this.repository.createAttendanceRegistrations(created), + ]), + ); + }), + ); + } + + loadPlannedTimes() { + return from(this.transformersService.findPlannedTimes()).pipe( + switchMap((stagingPlannedTimes) => { + if (stagingPlannedTimes.length === 0) { + return of({ + staging: [], + final: [], + }); + } + const { startDate: firstPlannedTime } = [ + ...stagingPlannedTimes, + ].shift(); + const { startDate: lastPlannedTime } = [...stagingPlannedTimes].pop(); + return from( + this.repository.findPlannedTimesInInterval({ + start: firstPlannedTime, + end: lastPlannedTime, + }), + ).pipe( + map((finalPlannedTimes) => ({ + final: finalPlannedTimes, + staging: stagingPlannedTimes, + })), + ); + }), + map(({ staging, final }) => + PlannedTimeLoader.determineLoadOperations(staging, final), + ), + switchMap(({ deleted, updated, created }) => { + return from( + Promise.all([ + this.repository.deletePlannedTimes(deleted), + this.repository.updatePlannedTimes(updated), + this.repository.createPlannedTimes(created), + ]), + ); + }), + ); + } + + loadWorkTimes() { + return from(this.transformersService.findWorkTimes()).pipe( + switchMap((stagingWorkTimes) => { + if (stagingWorkTimes.length === 0) { + return of({ + staging: [], + final: [], + }); + } + const { startDate: firstWorkTime } = [...stagingWorkTimes].shift(); + const { startDate: lastWorkTime } = [...stagingWorkTimes].pop(); + return from( + this.repository.findWorkTimesInInterval({ + start: firstWorkTime, + end: lastWorkTime, + }), + ).pipe( + map((finalTours) => ({ + final: finalTours, + staging: stagingWorkTimes, + })), + ); + }), + map(({ staging, final }) => + WorkTimeLoader.determineLoadOperations(staging, final), + ), + switchMap(({ deleted, updated, created }) => { + return from( + Promise.all([ + this.repository.deleteWorkTimes(deleted), + this.repository.updateWorkTimes(updated), + this.repository.createWorkTimes(created), + ]), + ); + }), + ); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders/attendance-registration.loader.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/attendance-registration.loader.ts new file mode 100644 index 0000000..7777e6c --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/attendance-registration.loader.ts @@ -0,0 +1,82 @@ +import { + Prisma, + AttendanceRegistration, + AttendanceRegistrationStaging, +} from '@prisma/client'; + +export class AttendanceRegistrationLoader { + public static determineLoadOperations( + stagingAttendanceRegistrations: AttendanceRegistrationStaging[], + finalAttendanceRegistrations: AttendanceRegistration[], + ): { + deleted: number[]; + updated: Prisma.AttendanceRegistrationCreateInput[]; + created: Prisma.AttendanceRegistrationCreateInput[]; + } { + const deletedAttendanceRegistrations = finalAttendanceRegistrations.filter( + (finalAttendanceRegistration) => + !stagingAttendanceRegistrations.find( + (stagingAttendanceRegistration) => + stagingAttendanceRegistration.attendanceRegistrationId === + finalAttendanceRegistration.attendanceRegistrationId, + ), + ); + const attendanceRegistrationsToInsert = + stagingAttendanceRegistrations.filter( + ({ attendanceRegistrationId }) => + !!attendanceRegistrationId && + !deletedAttendanceRegistrations.find( + (deletedAttendanceRegistration) => + deletedAttendanceRegistration.attendanceRegistrationId === + attendanceRegistrationId, + ), + ); + const newAttendanceRegistrations: Prisma.AttendanceRegistrationCreateInput[] = + []; + const updatedAttendanceRegistrations: Prisma.AttendanceRegistrationCreateInput[] = + []; + for (const stagingAttendanceRegistration of attendanceRegistrationsToInsert) { + const finalAttendanceRegistration = finalAttendanceRegistrations.find( + (finalAttendanceRegistration) => + finalAttendanceRegistration.attendanceRegistrationId === + stagingAttendanceRegistration.attendanceRegistrationId, + ); + if (finalAttendanceRegistration) { + updatedAttendanceRegistrations.push( + this.attendanceRegistrationStagingToFinal( + stagingAttendanceRegistration, + ), + ); + } else { + newAttendanceRegistrations.push( + this.attendanceRegistrationStagingToFinal( + stagingAttendanceRegistration, + ), + ); + } + } + + return { + deleted: deletedAttendanceRegistrations.map( + ({ attendanceRegistrationId }) => attendanceRegistrationId, + ), + updated: updatedAttendanceRegistrations, + created: newAttendanceRegistrations, + }; + } + + private static attendanceRegistrationStagingToFinal( + attendanceregistration: AttendanceRegistrationStaging, + ): Prisma.AttendanceRegistrationCreateInput { + const { attendanceRegistrationId, employeeId, dateTime, event } = + attendanceregistration; + + return { + attendanceRegistrationId, + employeeId, + dateTime, + event, + personnelNumber: '', + }; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders/patient.loader.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/patient.loader.ts new file mode 100644 index 0000000..b93e22d --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/patient.loader.ts @@ -0,0 +1,77 @@ +import { Prisma, Patient, PatientStaging } from '@prisma/client'; + +export class PatientLoader { + public static determineLoadOperations( + stagingPatients: PatientStaging[], + finalPatients: Patient[], + ): { + deleted: string[]; + updated: Prisma.PatientCreateInput[]; + created: Prisma.PatientCreateInput[]; + } { + const deletedPatients = finalPatients.filter( + (finalPatient) => + !stagingPatients.find( + (stagingPatient) => stagingPatient.id === finalPatient.id, + ), + ); + + const patientsToInsert = stagingPatients.filter( + ({ id }) => + !!id && + !deletedPatients.find((deletedPatient) => deletedPatient.id === id), + ); + const newPatients: Prisma.PatientCreateInput[] = []; + const updatedPatients: Prisma.PatientCreateInput[] = []; + + for (const stagingPatient of patientsToInsert) { + const finalPatient = finalPatients.find( + (finalPatient) => finalPatient.id === stagingPatient.id, + ); + if (finalPatient) { + updatedPatients.push(this.patientStagingToFinal(stagingPatient)); + } else { + newPatients.push(this.patientStagingToFinal(stagingPatient)); + } + } + return { + deleted: deletedPatients.map(({ id }) => id), + updated: updatedPatients, + created: newPatients, + }; + } + + private static patientStagingToFinal( + patient: PatientStaging, + ): Prisma.PatientCreateInput { + const { + id, + surname, + name, + street, + zip, + city, + birthday, + healthinsurance, + healthinsuranceId, + careDegree, + disabilityMark1, + disabilityMark2, + } = patient; + + return { + id, + surname, + name, + street, + zip, + city, + birthday, + healthinsurance, + healthinsuranceId, + careDegree, + disabilityMark1, + disabilityMark2, + }; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders/planned-time.loader.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/planned-time.loader.ts new file mode 100644 index 0000000..125e9ca --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/planned-time.loader.ts @@ -0,0 +1,75 @@ +import { Prisma, PlannedTime, PlannedTimeStaging } from '@prisma/client'; + +export class PlannedTimeLoader { + public static determineLoadOperations( + stagingPlannedTimes: PlannedTimeStaging[], + finalPlannedTimes: PlannedTime[], + ): { + deleted: string[]; + updated: Prisma.PlannedTimeCreateInput[]; + created: Prisma.PlannedTimeCreateInput[]; + } { + const deletedPlannedTimes = stagingPlannedTimes.filter( + ({ deleted }) => deleted, + ); + console.log(deletedPlannedTimes.length); + const plannedTimesToInsert = stagingPlannedTimes.filter( + ({ id }) => + !!id && + !deletedPlannedTimes.find( + (deletedPlannedTime) => deletedPlannedTime.id === id, + ), + ); + const newPlannedTimes: Prisma.PlannedTimeCreateInput[] = []; + const updatedPlannedTimes: Prisma.PlannedTimeCreateInput[] = []; + for (const stagingPlannedTime of plannedTimesToInsert) { + const finalPlannedTime = finalPlannedTimes.find( + (finalPlannedTime) => + finalPlannedTime.id === `${stagingPlannedTime.id}`, + ); + if (finalPlannedTime) { + updatedPlannedTimes.push( + this.plannedTimeStagingToFinal(stagingPlannedTime), + ); + } else { + newPlannedTimes.push( + this.plannedTimeStagingToFinal(stagingPlannedTime), + ); + } + } + + return { + deleted: deletedPlannedTimes.map(({ id }) => `${id}`), + updated: updatedPlannedTimes, + created: newPlannedTimes, + }; + } + + private static plannedTimeStagingToFinal( + plannedtime: PlannedTimeStaging, + ): Prisma.PlannedTimeCreateInput { + const { + id, + employeeId, + firstname, + surname, + departmentName, + startDate, + endDate, + mark, + } = plannedtime; + + return { + firstname, + surname, + startDate, + endDate, + id: `${id}`, + userId: `${employeeId}`, + department: departmentName, + pause: 0, + duration: 0, + note: mark, + }; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders/tour.loader.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/tour.loader.ts new file mode 100644 index 0000000..6eb94a5 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/tour.loader.ts @@ -0,0 +1,149 @@ +import { Prisma, Tour, TourStaging } from '@prisma/client'; + +export class TourLoader { + public static determineLoadOperations( + stagingTours: TourStaging[], + finalTours: Tour[], + ): { + deleted: string[]; + updated: Prisma.TourCreateInput[]; + created: Prisma.TourCreateInput[]; + } { + const deletedTours = finalTours.filter( + (finalTour) => + !stagingTours.find( + (stagingTour) => stagingTour.operationId === finalTour.operationId, + ), + ); + const toursToInsert = stagingTours.filter( + ({ id, operationId }) => + !!id && + !deletedTours.find( + (deletedTour) => deletedTour.operationId === operationId, + ), + ); + const newTours: Prisma.TourCreateInput[] = []; + const updatedTours: Prisma.TourCreateInput[] = []; + for (const stagingTour of toursToInsert) { + const finalTour = finalTours.find( + (finalTour) => finalTour.operationId === stagingTour.operationId, + ); + if (finalTour) { + updatedTours.push( + this.tourStagingToFinal(stagingTour, finalTour.ticketId), + ); + } else { + newTours.push(this.tourStagingToFinal(stagingTour)); + } + } + + return { + deleted: deletedTours.map(({ operationId }) => operationId), + updated: updatedTours, + created: newTours, + }; + } + + private static tourStagingToFinal(tour: TourStaging, ticketId?: string) { + const { + done, + direction, + carName, + operationId, + driverId, + driverName, + codriverId, + codriverName, + patientId, + patientName, + patientSurname, + patientStreet, + patientZip, + patientCity, + healthInsurance, + healthInsuranceNumber, + type, + category, + transportType, + ordinanceType, + rangeEndDate, + startInstitution, + startStreet, + startZip, + startCity, + targetInstitution, + targetStreet, + targetZip, + targetCity, + startDate, + check, + occupiedKm, + totalKm, + startBegin, + startEnd, + target, + targetBegin, + empty, + hasInfection, + infectionName, + revenueDispoLive, + revenue, + revenueDeviation, + consumptionCosts, + billDate, + billNumber, + createdAt, + } = tour; + + return { + done, + direction, + carName, + operationId, + driverId, + driverName, + codriverId, + codriverName, + patientId, + patientName, + patientSurname, + patientStreet, + patientZip, + patientCity, + healthInsurance, + healthInsuranceNumber, + type, + category, + transportType, + ordinanceType, + rangeEndDate, + startInstitution, + startStreet, + startZip, + startCity, + targetInstitution, + targetStreet, + targetZip, + targetCity, + startDate, + check, + occupiedKm, + totalKm, + startBegin, + startEnd, + target, + targetBegin, + empty, + hasInfection, + infectionName, + revenueDispoLive, + revenue, + revenueDeviation, + consumptionCosts, + billDate, + billNumber, + createdAt, + ...(ticketId ? { ticketId } : {}), + }; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-loaders/loaders/work-time.loader.ts b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/work-time.loader.ts new file mode 100644 index 0000000..4453396 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-loaders/loaders/work-time.loader.ts @@ -0,0 +1,70 @@ +import { Prisma, WorkTime, WorkTimeStaging } from '@prisma/client'; + +export class WorkTimeLoader { + public static determineLoadOperations( + stagingWorkTimes: WorkTimeStaging[], + finalWorkTimes: WorkTime[], + ): { + deleted: string[]; + updated: Prisma.WorkTimeCreateInput[]; + created: Prisma.WorkTimeCreateInput[]; + } { + const deletedWorkTimes = finalWorkTimes.filter( + (finalWorkTime) => + !stagingWorkTimes.find( + (stagingWorkTime) => `${stagingWorkTime.id}` === finalWorkTime.id, + ), + ); + const workTimesToInsert = stagingWorkTimes.filter( + ({ id }) => + !!id && + !deletedWorkTimes.find( + (deletedWorkTime) => deletedWorkTime.id === `${id}`, + ), + ); + const newWorkTimes: Prisma.WorkTimeCreateInput[] = []; + const updatedWorkTimes: Prisma.WorkTimeCreateInput[] = []; + for (const stagingWorkTime of workTimesToInsert) { + const finalWorkTime = finalWorkTimes.find( + (finalWorkTime) => finalWorkTime.id === `${stagingWorkTime.id}`, + ); + if (finalWorkTime) { + updatedWorkTimes.push(this.workTimeStagingToFinal(stagingWorkTime)); + } else { + newWorkTimes.push(this.workTimeStagingToFinal(stagingWorkTime)); + } + } + + return { + deleted: deletedWorkTimes.map(({ id }) => id), + updated: updatedWorkTimes, + created: newWorkTimes, + }; + } + + private static workTimeStagingToFinal( + worktime: WorkTimeStaging, + ): Prisma.WorkTimeCreateInput { + const { + id, + employeeId, + firstname, + surname, + departmentName, + startDate, + endDate, + } = worktime; + + return { + firstname, + surname, + startDate, + endDate, + id: `${id}`, + userId: `${employeeId}`, + department: departmentName, + pause: 0, + duration: 0, + }; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/adapter/tours-calculation.adapter.ts b/service/data-hub/data-connector/src/modules/feat-transformers/adapter/tours-calculation.adapter.ts new file mode 100644 index 0000000..a100651 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/adapter/tours-calculation.adapter.ts @@ -0,0 +1,54 @@ +// import { CalculationService } from '@dks/avicenna/api-util-calculation'; +// import { Injectable } from '@nestjs/common'; +// import { Tour } from '@prisma/client'; +// import { chunk } from 'src/utils'; +// import { TourPreStaging } from '../staging.interface'; + +// @Injectable() +// export class ToursCalculationAdapater { +// constructor(private readonly calculationService: CalculationService) {} + +// public async addRevenueAndConsumptionCostsToTours( +// tours: TourPreStaging[], +// ): Promise { +// const resultTours = []; +// for (const tourChunk of chunk(tours, 5)) { +// resultTours.push( +// await Promise.all( +// tourChunk.map((tour) => +// this.addRevenueAndConsumptionCostsToTour(tour), +// ), +// ), +// ); +// } +// return resultTours.flat(); +// } + +// public async addRevenueAndConsumptionCostsToTour( +// tour: TourPreStaging, +// ): Promise { +// if (!tour?.ordinanceType) { +// return tour; +// } +// const revenue = await this.calculationService.calculateTourRevenue( +// tour as unknown as Tour, +// ); +// const revenueDeviation = +// tour.revenueDispoLive > 0 +// ? Math.floor((revenue - tour.revenueDispoLive) * 100) / 100 +// : 0; + +// const { consumptionCostsPerTour } = +// this.calculationService.getBillingInformationByOrdinanceType( +// tour.ordinanceType.match(/([A-Z])\w+/).at(0), +// tour.startDate, +// ); + +// return { +// ...tour, +// revenue, +// revenueDeviation, +// consumptionCosts: consumptionCostsPerTour, +// }; +// } +// } diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/data-access-transformers.module.ts b/service/data-hub/data-connector/src/modules/feat-transformers/data-access-transformers.module.ts new file mode 100644 index 0000000..08d82ad --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/data-access-transformers.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +// import { ToursCalculationAdapater } from './adapter/tours-calculation.adapter'; +import { StagingRepository } from './staging.repository'; +import { TransformersService } from './transformers.service'; +import { PrismaModule } from 'src/core/database/prisma.module'; +import { DataAccessExtractorsModule } from '../feat-extractors/data-access-extractors.module'; + +@Module({ + imports: [PrismaModule, DataAccessExtractorsModule], + providers: [TransformersService, StagingRepository], + exports: [TransformersService], +}) +export class DataAccessTransformersModule {} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/staging.interface.ts b/service/data-hub/data-connector/src/modules/feat-transformers/staging.interface.ts new file mode 100644 index 0000000..4561d3c --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/staging.interface.ts @@ -0,0 +1,117 @@ +export interface TourPreStaging { + id?: string; + done: boolean; + direction: string; + carName?: string; + operationId: string; + + driverId?: string; + driverName?: string; + codriverId?: string; + codriverName?: string; + + patientId?: string; + patientName?: string; + patientSurname?: string; + patientStreet?: string; + patientZip?: string; + patientCity?: string; + healthInsurance?: string; + healthInsuranceNumber?: string; + + type?: string; + category?: string; + transportType?: string; + ordinanceType?: string; + rangeEndDate?: string; + + startInstitution?: string; + startStreet: string; + startZip: string; + startCity: string; + targetInstitution?: string; + targetStreet: string; + targetZip: string; + targetCity: string; + + startDate: Date; + + check: number; + occupiedKm?: number; + totalKm?: number; + + startBegin?: string; + startEnd?: string; + target?: string; + targetBegin?: string; + empty?: string; + + hasInfection: boolean; + infectionName?: string; + + revenueDispoLive?: number; + billDate?: Date; + billNumber?: string; + revenue?: number; + revenueDeviation?: number; + consumptionCosts?: number; + + createdAt?: Date; +} + +export interface PatientPreStaging { + id?: string; + surname?: string; + name?: string; + street?: string; + zip?: string; + city?: string; + + birthday?: Date; + healthinsuranceId?: string; + healthinsurance?: string; + careDegree?: string; + disabilityMark1?: string; + disabilityMark2?: string; +} + +export interface AttendanceRegistrationPreStaging { + attendanceRegistrationId: number; + employeeId: number; + dateTime: Date; + event: string; +} + +export interface PlannedTimePreStaging { + id: number; + employeeId: number; + firstname?: string; + surname?: string; + departmentName: string; + startDate: Date; + endDate: Date; + deleted: boolean; + mark?: string; +} + +export interface WorkTimePreStaging { + id: number; + employeeId: number; + firstname: string; + surname: string; + departmentName: string; + startDate: Date; + endDate: Date; +} + +export interface EmployeePreStaging { + id: number; + firstname: string; + surname: string; + contractStart: Date; + contractEnd: Date; + contractHoursWeek: number; + contractSalaryHour: number; + personTransportCertificate?: Date; + qualification?: string; +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/staging.repository.ts b/service/data-hub/data-connector/src/modules/feat-transformers/staging.repository.ts new file mode 100644 index 0000000..136e313 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/staging.repository.ts @@ -0,0 +1,118 @@ +import { Injectable } from '@nestjs/common'; +import { + AttendanceRegistrationStaging, + PatientStaging, + PlannedTimeStaging, + Prisma, + TourStaging, + WorkTimeStaging, +} from '@prisma/client'; +import { PrismaService } from 'src/core/database/prisma.service'; +import { + AttendanceRegistrationPreStaging, + EmployeePreStaging, + PatientPreStaging, + PlannedTimePreStaging, + TourPreStaging, + WorkTimePreStaging, +} from './staging.interface'; + +@Injectable() +export class StagingRepository { + constructor(private readonly prisma: PrismaService) {} + + clearStaging(): Promise { + return Promise.all([ + this.prisma.tourStaging.deleteMany(), + this.prisma.employeeStaging.deleteMany(), + this.prisma.workTimeStaging.deleteMany(), + this.prisma.plannedTimeStaging.deleteMany(), + this.prisma.attendanceRegistrationStaging.deleteMany(), + this.prisma.patientStaging.deleteMany(), + ]); + } + + createPatients(patients: PatientPreStaging[]): Promise { + return this.prisma.patientStaging.createMany({ + data: patients, + }); + } + + createPlannedTimes( + plannedTimes: PlannedTimePreStaging[], + ): Promise { + return this.prisma.plannedTimeStaging.createMany({ + data: plannedTimes, + }); + } + + createWorkTimes( + workTimes: WorkTimePreStaging[], + ): Promise { + return this.prisma.workTimeStaging.createMany({ + data: workTimes, + }); + } + + createAttendanceRegistrations( + attendanceRegistrations: AttendanceRegistrationPreStaging[], + ): Promise { + return this.prisma.attendanceRegistrationStaging.createMany({ + data: attendanceRegistrations, + }); + } + + createTours(tours: TourPreStaging[]): Promise { + return this.prisma.tourStaging.createMany({ + data: tours, + }); + } + + createEmployees( + employees: EmployeePreStaging[], + ): Promise { + return this.prisma.employeeStaging.createMany({ + data: employees, + }); + } + + findPatients(): Promise { + return this.prisma.patientStaging.findMany({ + orderBy: { + surname: 'asc', + }, + }); + } + + findTours(): Promise { + return this.prisma.tourStaging.findMany({ + orderBy: { + startDate: 'asc', + }, + }); + } + + findAttendanceRegistrations(): Promise { + return this.prisma.attendanceRegistrationStaging.findMany({ + orderBy: { + dateTime: 'asc', + }, + }); + } + + findPlannedTimes(): Promise { + return this.prisma.plannedTimeStaging.findMany({ + orderBy: { + startDate: 'asc', + }, + }); + } + + findWorkTimes(): Promise { + return this.prisma.workTimeStaging.findMany({ + orderBy: { + startDate: 'asc', + }, + }); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformer.functions.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformer.functions.ts new file mode 100644 index 0000000..cc89a55 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformer.functions.ts @@ -0,0 +1,54 @@ +import { parse, parseISO } from 'date-fns'; +import { fromZonedTime } from 'date-fns-tz'; + +export const parseDateTime = (format: string) => (value: string) => + parse(value, format, new Date()); +export const parseIsoDateTime = (value: string) => parseISO(value); + +export const parseZonedDateTime = + (tz: string) => (format: string) => (value: string) => + fromZonedTime(parse(value, format, new Date()), tz); + +export const parseCETDateTime = parseZonedDateTime('Europe/Berlin'); + +export const parseInteger = (value = '') => parseInt(value); + +export const parseFloat = (value = '') => + Number((value ?? '').replace(',', '.')); + +export const parseBoolean = (value = '') => value === '1' || value === 'true'; + +export const pick = + >(keys: (keyof T)[]) => + (value: Record) => + keys.reduce( + (obj, key) => { + if (value && Object.prototype.hasOwnProperty.call(value, key)) { + obj[key] = value[key]; + } + return obj; + }, + {} as Record, + ); + +export const pick2 = >( + keys: (keyof T)[], + value: T, +) => + keys.reduce((obj, key) => { + if (value && Object.prototype.hasOwnProperty.call(value, key)) { + obj[key] = value[key]; + } + return obj; + }, {} as T); + +export const transformProperties = + (transformer: (value: T) => U) => + (value: Record) => + Object.entries(value).reduce( + (obj, [key, value]) => ({ + ...obj, + [key]: transformer(value), + }), + {} as Record, + ); diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers.service.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers.service.ts new file mode 100644 index 0000000..6372a40 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers.service.ts @@ -0,0 +1,149 @@ +import { Injectable } from '@nestjs/common'; +import { + AttendanceRegistrationStaging, + EmployeeDispoLiveSource, + EmployeeDyflexisSource, + PatientStaging, + PlannedTimeStaging, + Prisma, + TourStaging, + WorkTimeStaging, +} from '@prisma/client'; +import { combineLatest, from, map, switchMap } from 'rxjs'; + +import { ExtractorsService } from '../feat-extractors/extractors.service'; +// import { ToursCalculationAdapater } from './adapter/tours-calculation.adapter'; +import { StagingRepository } from './staging.repository'; +import { AttendanceRegistrationTransformer } from './transformers/attendance-registration.transformer'; +import { EmployeeTransformer } from './transformers/employee.transformer'; +import { PatientTransformer } from './transformers/patient.transformer'; +import { PlannedTimeTransformer } from './transformers/planned-time.transformer'; +import { TourTransformer } from './transformers/tour.transformer'; +import { WorkTimeTransformer } from './transformers/work-time.transformer'; + +@Injectable() +export class TransformersService { + constructor( + private readonly extractorsService: ExtractorsService, + private readonly repository: StagingRepository, + // private readonly tourCalculationAdapter: ToursCalculationAdapater, + ) {} + + clearStaging(): Promise { + return this.repository.clearStaging(); + } + + transformPlannedTimes(jobId: string) { + return from(this.extractorsService.findPlannedTimesByJobId(jobId)).pipe( + map((plannedTimes) => + plannedTimes.map((plannedTime) => + PlannedTimeTransformer.transform(plannedTime), + ), + ), + switchMap((plannedTimes) => + this.repository.createPlannedTimes(plannedTimes), + ), + ); + } + + transformWorkTimes(jobId: string) { + return from(this.extractorsService.findWorkTimesByJobId(jobId)).pipe( + map((workTimes) => + workTimes.map((workTime) => WorkTimeTransformer.transform(workTime)), + ), + switchMap((workTimes) => this.repository.createWorkTimes(workTimes)), + ); + } + + transformAttendanceRegistrations(jobId: string) { + return from( + this.extractorsService.findAttendanceRegistrationsByJobId(jobId), + ).pipe( + map((attendanceRegistrations) => + attendanceRegistrations.map((attendanceRegistration) => + AttendanceRegistrationTransformer.transform(attendanceRegistration), + ), + ), + switchMap((attendanceRegistrations) => + this.repository.createAttendanceRegistrations(attendanceRegistrations), + ), + ); + } + + transformTours(jobId: string) { + return from(this.extractorsService.findToursByJobId(jobId)).pipe( + // TODO: Add validation filter + map((tours) => + tours + .filter(({ einsatzNummer }) => !!einsatzNummer) + .map((tour) => TourTransformer.transform(tour)), + ), + // switchMap((tours) => + // from( + // this.tourCalculationAdapter.addRevenueAndConsumptionCostsToTours( + // tours, + // ), + // ), + // ), + switchMap((tours) => this.repository.createTours(tours)), + ); + } + + transformEmployees(jobId: string) { + return combineLatest([ + from(this.extractorsService.findEmployeesDispoLiveByJobId(jobId)), + from(this.extractorsService.findEmployeesDyflexisByJobId(jobId)), + ]).pipe( + map(([dispoLiveEmployees, dyflexisEmployees]) => + dyflexisEmployees + .map((dyflexisEmployee) => { + const dispoLiveEmployee = dispoLiveEmployees.find( + (e) => e.importPersId === dyflexisEmployee.id, + ); + if (!dispoLiveEmployee) return undefined; + + return [dyflexisEmployee, dispoLiveEmployee]; + }) + .filter(Boolean), + ), + map((employees) => + employees.map(([dyflexisEmployee, dispoLiveEmployee]) => + EmployeeTransformer.transform( + dyflexisEmployee as EmployeeDyflexisSource, + dispoLiveEmployee as EmployeeDispoLiveSource, + ), + ), + ), + switchMap((employees) => this.repository.createEmployees(employees)), + ); + } + + transformPatients(jobId: string) { + return from(this.extractorsService.findPatientsByJobId(jobId)).pipe( + map((patients) => + patients.map((patient) => PatientTransformer.transform(patient)), + ), + switchMap((patients) => this.repository.createPatients(patients)), + ); + } + + findPatients(): Promise { + return this.repository.findPatients(); + } + + findTours(): Promise { + return this.repository.findTours(); + } + + findAttendanceRegistrations(): Promise { + return this.repository.findAttendanceRegistrations(); + } + + findPlannedTimes(): Promise { + return this.repository.findPlannedTimes(); + } + + findWorkTimes(): Promise { + return this.repository.findWorkTimes(); + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/attendance-registration.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/attendance-registration.transformer.ts new file mode 100644 index 0000000..7646044 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/attendance-registration.transformer.ts @@ -0,0 +1,36 @@ +import { AttendanceRegistrationSource } from '@prisma/client'; +import { AttendanceRegistrationPreStaging } from '../staging.interface'; +import { + parseInteger, + parseIsoDateTime, + pick, + transformProperties, +} from '../transformer.functions'; + +export class AttendanceRegistrationTransformer { + public static transform( + attendanceRegistrationSource: AttendanceRegistrationSource, + ): AttendanceRegistrationPreStaging { + const attendanceRegistrationParsed = { + ...attendanceRegistrationSource, + ...transformProperties(parseInteger)( + pick(['attendanceRegistrationId', 'employeeId'])( + attendanceRegistrationSource, + ), + ), + ...transformProperties(parseIsoDateTime)( + pick(['dateTime'])(attendanceRegistrationSource), + ), + }; + + const { attendanceRegistrationId, employeeId, dateTime, event } = + attendanceRegistrationParsed; + + return { + attendanceRegistrationId, + employeeId, + dateTime, + event, + } as unknown as AttendanceRegistrationPreStaging; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/employee.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/employee.transformer.ts new file mode 100644 index 0000000..f9e0383 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/employee.transformer.ts @@ -0,0 +1,67 @@ +import { + EmployeeDispoLiveSource, + EmployeeDyflexisSource, +} from '@prisma/client'; + +import { isValidDate } from 'src/utils'; +import { EmployeePreStaging } from '../staging.interface'; +import { + parseDateTime, + parseFloat, + parseInteger, + pick, + transformProperties, +} from '../transformer.functions'; + +export class EmployeeTransformer { + public static transform( + dyflexisEmployeeSource: EmployeeDyflexisSource, + dispoLiveEmployeeSource: EmployeeDispoLiveSource, + ): EmployeePreStaging { + const dyflexisEmployeeParsed = { + ...dyflexisEmployeeSource, + ...transformProperties(parseFloat)( + pick(['contract_hours_week', 'contract_salary_hour'])( + dyflexisEmployeeSource, + ), + ), + ...transformProperties(parseInteger)( + pick(['id'])(dyflexisEmployeeSource), + ), + ...transformProperties(parseDateTime('yyyy-MM-dd'))( + pick(['contract_start', 'contract_end'])(dyflexisEmployeeSource), + ), + }; + const dispoLiveEmployeeParsed = { + ...dispoLiveEmployeeSource, + ...transformProperties(parseDateTime('yyyy-MM-dd'))( + pick(['pScheinAblaufDatum'])(dispoLiveEmployeeSource), + ), + }; + + const { + id, + firstname, + surname, + contract_start, + contract_end, + contract_hours_week, + contract_salary_hour, + } = dyflexisEmployeeParsed; + const { pScheinAblaufDatum, qualifikation } = dispoLiveEmployeeParsed; + + return { + id, + firstname, + surname, + contractStart: contract_start, + contractEnd: contract_end, + contractHoursWeek: contract_hours_week, + contractSalaryHour: contract_salary_hour, + personTransportCertificate: isValidDate(new Date(pScheinAblaufDatum)) + ? pScheinAblaufDatum + : undefined, + qualification: qualifikation, + } as unknown as EmployeePreStaging; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/patient.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/patient.transformer.ts new file mode 100644 index 0000000..1393261 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/patient.transformer.ts @@ -0,0 +1,50 @@ +import { PatientSource } from '@prisma/client'; +import { isValidDate } from 'src/utils'; +import { PatientPreStaging } from '../staging.interface'; +import { + parseDateTime, + pick, + transformProperties, +} from '../transformer.functions'; + +export class PatientTransformer { + public static transform(patientSource: PatientSource): PatientPreStaging { + const patientParsed = { + ...patientSource, + ...transformProperties(parseDateTime('yyyy-MM-dd'))( + pick(['birthday'])(patientSource), + ), + }; + + const { + id, + surname, + name, + street, + zip, + city, + + birthday, + kkId, + krankenkasse, + pflegestufe, + behinderung1, + behinderung2, + } = patientParsed; + + return { + id, + surname, + name, + street, + zip, + city, + birthday: isValidDate(new Date(birthday)) ? birthday : undefined, + healthinsurance: krankenkasse, + healthinsuranceId: kkId, + careDegree: pflegestufe, + disabilityMark1: behinderung1, + disabilityMark2: behinderung2, + } as unknown as PatientPreStaging; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/planned-time.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/planned-time.transformer.ts new file mode 100644 index 0000000..d73c1a7 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/planned-time.transformer.ts @@ -0,0 +1,52 @@ +import { PlannedTimeSource } from '@prisma/client'; +import { PlannedTimePreStaging } from '../staging.interface'; +import { + parseBoolean, + parseCETDateTime, + parseInteger, + pick, + transformProperties, +} from '../transformer.functions'; + +export class PlannedTimeTransformer { + public static transform( + plannedTimeSource: PlannedTimeSource, + ): PlannedTimePreStaging { + const plannedTimeParsed = { + ...plannedTimeSource, + ...transformProperties(parseInteger)( + pick(['id', 'user_id'])(plannedTimeSource), + ), + ...transformProperties(parseCETDateTime('yyyy-MM-dd HH:mm:ss'))( + pick(['start_date', 'end_date'])(plannedTimeSource), + ), + ...transformProperties(parseBoolean)( + pick(['deleted'])(plannedTimeSource), + ), + }; + + const { + id, + user_id, + firstname, + surname, + department_name, + start_date, + end_date, + deleted, + mark, + } = plannedTimeParsed; + + return { + id, + firstname, + surname, + deleted, + mark, + employeeId: user_id, + departmentName: department_name, + startDate: start_date, + endDate: end_date, + } as unknown as PlannedTimePreStaging; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/tour.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/tour.transformer.ts new file mode 100644 index 0000000..318fc50 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/tour.transformer.ts @@ -0,0 +1,142 @@ +import { TourSource } from '@prisma/client'; +import { TourPreStaging } from '../staging.interface'; +import { + parseBoolean, + parseCETDateTime, + parseDateTime, + parseFloat, + parseInteger, + parseIsoDateTime, + pick, + transformProperties, +} from '../transformer.functions'; + +export class TourTransformer { + public static transform(tourSource: TourSource): TourPreStaging { + const tourParsed = { + ...tourSource, + ...transformProperties(parseInteger)(pick(['check'])(tourSource)), + ...transformProperties(parseFloat)( + pick(['besetztKm', 'gesamtPreis'])(tourSource), + ), + ...transformProperties(parseIsoDateTime)(pick(['date_iso'])(tourSource)), + ...(tourSource.abrDate + ? transformProperties(parseDateTime('yyyy-MM-dd'))( + pick(['abrDate'])(tourSource), + ) + : undefined), + ...(tourSource.endgen + ? transformProperties(parseDateTime('yyyy-MM-dd'))( + pick(['endgen'])(tourSource), + ) + : undefined), + ...(tourSource.createTime.length > 10 + ? transformProperties(parseCETDateTime('yyyy-MM-dd HH:mm:ss'))( + pick(['createTime'])(tourSource), + ) + : transformProperties(parseCETDateTime('yyyy-MM-dd'))( + pick(['createTime'])(tourSource), + )), + ...transformProperties(parseBoolean)( + pick(['infektion', 'gefahren'])(tourSource), + ), + }; + + const { + id, + gefahren, + direction, + carName, + einsatzNummer, + fahrer, + fahrerName, + beifahrer, + beifahrerName, + patId, + patName, + patSurname, + patStreet, + patZip, + patCity, + krankenkasse, + kkId, + type, + serienKategorName, + transportKrz, + verordnungsName, + endgen, + startInstitution, + startStreet, + startZip, + startCity, + targetInstitution, + targetStreet, + targetZip, + targetCity, + date_iso, + check, + besetztKm, + startBegin, + startEnd, + ziel, + zielBegin, + frei, + infektion, + itKrz, + gesamtPreis, + abrDate, + abrNum, + createTime, + } = tourParsed; + + return { + id, + direction, + carName, + startInstitution, + startStreet, + startZip, + startCity, + targetInstitution, + targetStreet, + targetZip, + targetCity, + check, + type, + done: gefahren, + operationId: einsatzNummer, + driverId: fahrer, + driverName: fahrerName, + codriverId: beifahrer, + codriverName: beifahrerName, + patientId: patId, + patientName: patName, + patientSurname: patSurname, + patientStreet: patStreet, + patientZip: patZip, + patientCity: patCity, + healthInsurance: krankenkasse, + healthInsuranceNumber: kkId, + category: serienKategorName, + transportType: transportKrz, + ordinanceType: verordnungsName, + rangeEndDate: endgen, + startDate: date_iso, + occupiedKm: besetztKm, + totalKm: besetztKm, + startBegin, + startEnd, + target: ziel, + targetBegin: zielBegin, + empty: frei, + hasInfection: infektion, + infectionName: itKrz, + revenueDispoLive: gesamtPreis, + billDate: abrDate, + billNumber: abrNum, + createdAt: createTime, + revenue: 0, + consumptionCosts: 0, + } as unknown as TourPreStaging; + } +} diff --git a/service/data-hub/data-connector/src/modules/feat-transformers/transformers/work-time.transformer.ts b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/work-time.transformer.ts new file mode 100644 index 0000000..7e3d554 --- /dev/null +++ b/service/data-hub/data-connector/src/modules/feat-transformers/transformers/work-time.transformer.ts @@ -0,0 +1,42 @@ +import { WorkTimeSource } from '@prisma/client'; +import { WorkTimePreStaging } from '../staging.interface'; +import { + parseCETDateTime, + parseInteger, + pick, + transformProperties, +} from '../transformer.functions'; + +export class WorkTimeTransformer { + public static transform(workTimeSource: WorkTimeSource): WorkTimePreStaging { + const workTimeParsed = { + ...workTimeSource, + ...transformProperties(parseInteger)( + pick(['id', 'user_id'])(workTimeSource), + ), + ...transformProperties(parseCETDateTime('yyyy-MM-dd HH:mm:ss'))( + pick(['start_date', 'end_date'])(workTimeSource), + ), + }; + + const { + id, + user_id, + firstname, + surname, + department_name, + start_date, + end_date, + } = workTimeParsed; + + return { + id, + firstname, + surname, + employeeId: user_id, + departmentName: department_name, + startDate: start_date, + endDate: end_date, + } as unknown as WorkTimePreStaging; + } +} diff --git a/service/data-hub/data-connector/src/utils/chunk.spec.ts b/service/data-hub/data-connector/src/utils/chunk.spec.ts new file mode 100644 index 0000000..8a9afa6 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/chunk.spec.ts @@ -0,0 +1,19 @@ +import { chunk } from './chunk'; +import { range } from './range'; + +describe('chunk', () => { + it('should split an array with 10 elements into two chunks with 5 elements', () => { + const given = range(10); + const when = chunk(given, 5); + expect(when).toHaveLength(2); + when.map((sub) => expect(sub).toHaveLength(5)); + }); + + it('should split an array with 8 elements into two chunks, first with 5 elements, second with 3', () => { + const given = range(8); + const when = chunk(given, 5); + expect(when).toHaveLength(2); + expect(when[0]).toHaveLength(5); + expect(when[1]).toHaveLength(3); + }); +}); diff --git a/service/data-hub/data-connector/src/utils/chunk.ts b/service/data-hub/data-connector/src/utils/chunk.ts new file mode 100644 index 0000000..9f15b71 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/chunk.ts @@ -0,0 +1,9 @@ +export function chunk(input: T[], size: number): T[][] { + return input.reduce( + (array, item, index) => + index % size === 0 + ? [...array, [item]] + : [...array.slice(0, -1), [...array.slice(-1)[0], item]], + [] as T[][] + ); +} diff --git a/service/data-hub/data-connector/src/utils/index.ts b/service/data-hub/data-connector/src/utils/index.ts new file mode 100644 index 0000000..36cedd1 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/index.ts @@ -0,0 +1,3 @@ +export * from './chunk'; +export * from './is-valid-date'; +export * from './range'; diff --git a/service/data-hub/data-connector/src/utils/is-valid-date.spec.ts b/service/data-hub/data-connector/src/utils/is-valid-date.spec.ts new file mode 100644 index 0000000..e32a0a1 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/is-valid-date.spec.ts @@ -0,0 +1,12 @@ +import { isValidDate } from './is-valid-date'; + +describe('isValidDate', () => { + it('should return false if date is invalid', () => { + const given = new Date(NaN); + expect(isValidDate(given)).toBe(false); + }); + it('should return true if date is valid', () => { + const given = new Date(); + expect(isValidDate(given)).toBe(true); + }); +}); diff --git a/service/data-hub/data-connector/src/utils/is-valid-date.ts b/service/data-hub/data-connector/src/utils/is-valid-date.ts new file mode 100644 index 0000000..2340757 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/is-valid-date.ts @@ -0,0 +1,3 @@ +export function isValidDate(date: Date): boolean { + return date instanceof Date && !isNaN(date.getTime()); +} diff --git a/service/data-hub/data-connector/src/utils/range.spec.ts b/service/data-hub/data-connector/src/utils/range.spec.ts new file mode 100644 index 0000000..ec20e2e --- /dev/null +++ b/service/data-hub/data-connector/src/utils/range.spec.ts @@ -0,0 +1,9 @@ +import { range } from './range'; + +describe('range', () => { + it('should generate an array with the supplied length and numbers ascending from 0', () => { + const when = range(10); + expect(when).toHaveLength(10); + when.map((item, index) => expect(item).toBe(index)); + }); +}); diff --git a/service/data-hub/data-connector/src/utils/range.ts b/service/data-hub/data-connector/src/utils/range.ts new file mode 100644 index 0000000..1ecc007 --- /dev/null +++ b/service/data-hub/data-connector/src/utils/range.ts @@ -0,0 +1,3 @@ +export function range(length: number): number[] { + return Array.from({ length }, (_, i) => i); +}