(dc): change logging to json

service/data-hub/data-connector/nest-cli.json

@@ -3,9 +3,6 @@
   "collection": "@nestjs/schematics",
   "sourceRoot": "src",
   "compilerOptions": {
-    "deleteOutDir": true,
+    "deleteOutDir": true
-    "assets": [
-      "**/instrument.js"
-    ]
   }
 }

service/data-hub/data-connector/package.json

@@ -23,59 +23,59 @@
     "seed": "ts-node prisma/seed/seed.ts"
   },
   "dependencies": {
-    "@apollo/server": "^4.10.5",
+    "@apollo/server": "^4.12.0",
-    "@faker-js/faker": "^9.0.1",
+    "@faker-js/faker": "^9.6.0",
-    "@nestjs/apollo": "^12.2.0",
+    "@nestjs/apollo": "^13.0.4",
-    "@nestjs/axios": "^3.0.2",
+    "@nestjs/axios": "^4.0.0",
-    "@nestjs/common": "^10.0.0",
+    "@nestjs/common": "^11.0.1",
-    "@nestjs/config": "^3.2.3",
+    "@nestjs/config": "^4.0.2",
-    "@nestjs/core": "^10.0.0",
+    "@nestjs/core": "^11.0.15",
-    "@nestjs/event-emitter": "^2.0.4",
+    "@nestjs/event-emitter": "^3.0.1",
-    "@nestjs/graphql": "^12.2.0",
+    "@nestjs/graphql": "^13.0.4",
-    "@nestjs/platform-express": "^10.0.0",
+    "@nestjs/platform-express": "^11.0.15",
-    "@nestjs/schedule": "^4.1.0",
+    "@nestjs/schedule": "^5.0.1",
     "@opentelemetry/api": "^1.9.0",
-    "@opentelemetry/auto-instrumentations-node": "^0.56.1",
+    "@opentelemetry/auto-instrumentations-node": "^0.57.1",
     "@opentelemetry/exporter-metrics-otlp-proto": "^0.200.0",
     "@opentelemetry/exporter-trace-otlp-proto": "^0.200.0",
     "@opentelemetry/resources": "^2.0.0",
     "@opentelemetry/sdk-node": "^0.200.0",
-    "@opentelemetry/semantic-conventions": "^1.30.0",
+    "@opentelemetry/semantic-conventions": "^1.31.0",
-    "@prisma/client": "^5.16.2",
+    "@prisma/client": "^6.6.0",
-    "date-fns": "^3.6.0",
+    "date-fns": "^4.1.0",
-    "date-fns-tz": "^3.1.3",
+    "date-fns-tz": "^3.2.0",
-    "graphql": "^16.9.0",
+    "graphql": "^16.10.0",
-    "graphql-subscriptions": "^2.0.0",
+    "graphql-subscriptions": "^3.0.0",
-    "graphql-ws": "^5.16.0",
+    "graphql-ws": "^6.0.4",
-    "kysely": "^0.27.4",
+    "kysely": "^0.27.6",
-    "pg": "^8.12.0",
+    "pg": "^8.14.1",
     "prisma-kysely": "^1.8.0",
-    "reflect-metadata": "^0.2.0",
+    "reflect-metadata": "^0.2.2",
-    "rxjs": "^7.8.1"
+    "rxjs": "^7.8.2"
   },
   "devDependencies": {
-    "@nestjs/cli": "^10.0.0",
+    "@nestjs/cli": "^11.0.6",
-    "@nestjs/schematics": "^10.0.0",
+    "@nestjs/schematics": "^11.0.5",
-    "@nestjs/testing": "^10.0.0",
+    "@nestjs/testing": "^11.0.15",
-    "@types/express": "^4.17.17",
+    "@types/express": "^5.0.1",
-    "@types/jest": "^29.5.2",
+    "@types/jest": "^29.5.14",
-    "@types/node": "^20.3.1",
+    "@types/node": "^22.14.0",
-    "@types/supertest": "^6.0.0",
+    "@types/supertest": "^6.0.3",
-    "@typescript-eslint/eslint-plugin": "^7.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.29.1",
-    "@typescript-eslint/parser": "^7.0.0",
+    "@typescript-eslint/parser": "^8.29.1",
-    "eslint": "^8.42.0",
+    "eslint": "^9.24.0",
-    "eslint-config-prettier": "^9.0.0",
+    "eslint-config-prettier": "^10.1.2",
-    "eslint-plugin-prettier": "^5.0.0",
+    "eslint-plugin-prettier": "^5.2.6",
-    "jest": "^29.5.0",
+    "jest": "^29.7.0",
-    "prettier": "^3.0.0",
+    "prettier": "^3.5.3",
-    "prisma": "^5.16.2",
+    "prisma": "^6.6.0",
     "source-map-support": "^0.5.21",
-    "supertest": "^7.0.0",
+    "supertest": "^7.1.0",
-    "ts-jest": "^29.1.0",
+    "ts-jest": "^29.3.1",
-    "ts-loader": "^9.4.3",
+    "ts-loader": "^9.5.2",
-    "ts-node": "^10.9.1",
+    "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.7.2"
+    "typescript": "^5.8.3"
   },
   "jest": {
     "moduleFileExtensions": [

service/data-hub/data-connector/src/app.controller.ts

@@ -1,4 +1,4 @@
-import { Controller, Get, Query } from '@nestjs/common';
+import { Controller, Get, Logger, Query } from '@nestjs/common';
 import { PrismaService } from './core/database/prisma.service';
 import { AnomaliesService } from './modules/feat-anomalies/anomalies.service';
 import { CalculationService } from './modules/feat-calculation/calculation.service';
@@ -7,6 +7,8 @@ import { TicketService } from './modules/feat-tickets/tickets.service';
 
 @Controller()
 export class AppController {
+  private logger = new Logger(AppController.name);
+
   constructor(
     private readonly importer: ImporterService,
     private readonly anomaliesService: AnomaliesService,
@@ -17,6 +19,8 @@ export class AppController {
 
   @Get('etl')
   etl(@Query('from') from: string, @Query('to') to: string) {
+    this.logger.log(`Start ETL for interval from=${from} to=${to}`);
+
     if (!from || !to) {
       return {
         error: 'from and to must be defined',
@@ -146,7 +150,6 @@ export class AppController {
         revenue: await this.calculationService.calculateTourRevenue(tour),
       });
     }
-    console.log(this.calculationService.getHealthInsurancesWithNoTenant());
     return toursWithRevenue;
   }
 }

service/data-hub/data-connector/src/instrument.js

@@ -1,38 +0,0 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-/*instrumentation.ts*/
-const { NodeSDK } = require('@opentelemetry/sdk-node');
-const {
-  getNodeAutoInstrumentations,
-} = require('@opentelemetry/auto-instrumentations-node');
-const {
-  OTLPTraceExporter,
-} = require('@opentelemetry/exporter-trace-otlp-proto'); // Annahme: gRPC/Proto Exporter für Backend
-const {
-  OTLPMetricExporter,
-} = require('@opentelemetry/exporter-metrics-otlp-proto'); // Annahme: gRPC/Proto Exporter für Backend
-const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
-const { resourceFromAttributes } = require('@opentelemetry/resources');
-const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
-
-const otelEndpoint =
-  process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ??
-  'http://monitoring_alloy:4318';
-
-const sdk = new NodeSDK({
-  resource: resourceFromAttributes({
-    [ATTR_SERVICE_NAME]: 'avicenna.data-connector',
-    namespace: 'avicenna',
-  }),
-  traceExporter: new OTLPTraceExporter({
-    url: `${otelEndpoint}/v1/traces`,
-  }),
-  metricReader: new PeriodicExportingMetricReader({
-    exporter: new OTLPMetricExporter({
-      url: `${otelEndpoint}/v1/metrics`,
-    }),
-  }),
-  instrumentations: [getNodeAutoInstrumentations()],
-});
-
-sdk.start();
-console.log('OTel SDK started');

service/data-hub/data-connector/src/main.ts

@@ -1,8 +1,11 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
+import { ConsoleLogger } from '@nestjs/common';
 
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule);
+  const app = await NestFactory.create(AppModule, {
+    logger: new ConsoleLogger({ json: true }),
+  });
   app.enableCors();
 
   await app.listen(3000);

service/data-hub/data-connector/src/modules/app-manager/manager.resolver.ts

@@ -39,7 +39,7 @@ export class ManagerResolver {
 
   @Subscription(() => TimePerTourUpdateObjectType)
   timePerTourUpdated() {
-    return this.timePerTourKpiService.pubSub.asyncIterator(
+    return this.timePerTourKpiService.pubSub.asyncIterableIterator(
       'timePerTourUpdated',
     );
   }

service/data-hub/data-connector/src/modules/app-ticket-system/ticket-lifecycle.service.ts

@@ -92,7 +92,6 @@ export class TicketLifecycleService {
 
   private async validateCreatedTickets(): Promise<void> {
     const tickets = await this.repository.findAllCreated();
-    this.logger.debug(`Found ${tickets.length} with state CREATED`);
 
     for (const ticket of tickets) {
       const {

service/data-hub/data-connector/src/modules/app-ticket-system/ticket.repository.ts

@@ -362,7 +362,6 @@ export class TicketRepository {
 
     const dwellTimesForTicketState = ticketWithState
       .map(({ state }) => {
-        console.log(state);
         const stateToCalculateIdx = state.findIndex(
           ({ state }) => state === ticketState,
         );

service/data-hub/data-connector/src/modules/app-ticket-system/tickets.resolver.ts

@@ -77,6 +77,6 @@ export class TicketsResolver {
 
   @Subscription(() => TicketsUpdatedObjectType)
   ticketsUpdated() {
-    return this.ticketsService.pubSub.asyncIterator('ticketsUpdated');
+    return this.ticketsService.pubSub.asyncIterableIterator('ticketsUpdated');
   }
 }

service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/driver-certificate.detector.ts

@@ -82,7 +82,7 @@ export class DriverCertificateDetector extends AnomalyDetector {
         };
       }
     } catch (error) {
-      this.logger.error(error, anomaly.id);
+      this.logger.error(error, { anomalyId: anomaly.id });
       return anomaly;
     }
   }

service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/employees-logged-in.detector.ts

@@ -68,7 +68,7 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
         return anomaly;
       }
     } catch (error) {
-      this.logger.error(error, anomaly.id);
+      this.logger.error(error, { anomalyId: anomaly.id });
       return anomaly;
     }
   }
@@ -200,12 +200,11 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
       return false;
     }
     if (!employee.personnelNumber) {
-      this.logger.warn(
+      this.logger.warn(`Employee has no personnelNumber`, {
-        `Employee has no personnelNumber`,
+        id: employee.id,
-        employee.id,
+        surname: employee.surname,
-        employee.surname,
+        name: employee.name,
-        employee.name,
+      });
-      );
       return false;
     }
     return true;
@@ -313,13 +312,12 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
         return [false, undefined];
       }
     } catch (error) {
-      this.logger.error(
+      this.logger.error(error, {
-        error,
+        timeInQuestion: datetimeToCheck.toISOString(),
-        datetimeToCheck.toISOString(),
         workTime,
         firstClockedIn,
         lastClockedOut,
-      );
+      });
       return [false, undefined];
     }
   }

service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/one-tour-at-a-time.detector.ts

@@ -131,12 +131,11 @@ export class OneTourAtATimeDetector extends AnomalyDetector {
                       tourListInterval,
                     );
                   } catch (error) {
-                    this.logger.error(
+                    this.logger.error(error, {
-                      'tourListInterval',
+                      what: 'tourListInterval',
-                      error,
                       tourListInterval,
                       tourInterval,
-                    );
+                    });
                     return false;
                   }
                 },
@@ -299,7 +298,7 @@ export class OneTourAtATimeDetector extends AnomalyDetector {
       }
       return { ...anomaly };
     } catch (error) {
-      this.logger.error(error, anomaly.id);
+      this.logger.error(error, { anomalyId: anomaly.id });
       return anomaly;
     }
   }

service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/working-employee-not-in-distribution-system.detector.ts

@@ -66,7 +66,7 @@ export class WorkingEmployeeNotInDistributionSystemDetector extends AnomalyDetec
           } catch (error) {
             this.logger.warn(
               `Employee does not exist! PersonnelNumber: ${personnelNumber}`,
-              error,
+              { error },
             );
             return;
           }

service/data-hub/data-connector/src/modules/feat-calculation/calculation.service.ts

@@ -51,7 +51,7 @@ export class CalculationService {
       new Date(startDate),
     );
     if (!tariff) {
-      this.logger.error(
+      this.logger.warn(
         `No tariff found for tour ${tour.operationId} (${tour.healthInsurance})`,
       );
       return 0;
@@ -163,11 +163,10 @@ export class CalculationService {
       ),
     );
     if (!billingInformation) {
-      this.logger.error(
+      this.logger.warn("Couldn't find billing information", {
-        "Couldn't find billing information",
+        date: date.toISOString(),
         ordinanceType,
-        date.toISOString(),
+      });
-      );
       throw new Error("Couldn't find billing information");
     }
     return billingInformation[1];

service/data-hub/data-connector/src/modules/feat-etl/feat-loaders/loaders/planned-time.loader.ts

@@ -12,7 +12,6 @@ export class PlannedTimeLoader {
     const deletedPlannedTimes = stagingPlannedTimes.filter(
       ({ deleted }) => deleted,
     );
-    console.log(deletedPlannedTimes.length);
     const plannedTimesToInsert = stagingPlannedTimes.filter(
       ({ id }) =>
         !!id &&