marndt
/
avicenna
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(dc): change logging to json

This commit is contained in:
Marcel Arndt 2025-04-11 11:58:19 +02:00
parent 46631db5b2
commit dde00abe54
16 changed files with 3674 additions and 3092 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
service/data-hub/data-connector/nest-cli.json
View File

@ -3,9 +3,6 @@
"collection": "@nestjs/schematics",
"sourceRoot": "src",
"compilerOptions": {
"deleteOutDir": true,
"assets": [
"**/instrument.js"
]
"deleteOutDir": true
}
}

6576
service/data-hub/data-connector/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

86
service/data-hub/data-connector/package.json
View File

@ -23,59 +23,59 @@
"seed": "ts-node prisma/seed/seed.ts"
},
"dependencies": {
"@apollo/server": "^4.10.5",
"@faker-js/faker": "^9.0.1",
"@nestjs/apollo": "^12.2.0",
"@nestjs/axios": "^3.0.2",
"@nestjs/common": "^10.0.0",
"@nestjs/config": "^3.2.3",
"@nestjs/core": "^10.0.0",
"@nestjs/event-emitter": "^2.0.4",
"@nestjs/graphql": "^12.2.0",
"@nestjs/platform-express": "^10.0.0",
"@nestjs/schedule": "^4.1.0",
"@apollo/server": "^4.12.0",
"@faker-js/faker": "^9.6.0",
"@nestjs/apollo": "^13.0.4",
"@nestjs/axios": "^4.0.0",
"@nestjs/common": "^11.0.1",
"@nestjs/config": "^4.0.2",
"@nestjs/core": "^11.0.15",
"@nestjs/event-emitter": "^3.0.1",
"@nestjs/graphql": "^13.0.4",
"@nestjs/platform-express": "^11.0.15",
"@nestjs/schedule": "^5.0.1",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.56.1",
"@opentelemetry/auto-instrumentations-node": "^0.57.1",
"@opentelemetry/exporter-metrics-otlp-proto": "^0.200.0",
"@opentelemetry/exporter-trace-otlp-proto": "^0.200.0",
"@opentelemetry/resources": "^2.0.0",
"@opentelemetry/sdk-node": "^0.200.0",
"@opentelemetry/semantic-conventions": "^1.30.0",
"@prisma/client": "^5.16.2",
"date-fns": "^3.6.0",
"date-fns-tz": "^3.1.3",
"graphql": "^16.9.0",
"graphql-subscriptions": "^2.0.0",
"graphql-ws": "^5.16.0",
"kysely": "^0.27.4",
"pg": "^8.12.0",
"@opentelemetry/semantic-conventions": "^1.31.0",
"@prisma/client": "^6.6.0",
"date-fns": "^4.1.0",
"date-fns-tz": "^3.2.0",
"graphql": "^16.10.0",
"graphql-subscriptions": "^3.0.0",
"graphql-ws": "^6.0.4",
"kysely": "^0.27.6",
"pg": "^8.14.1",
"prisma-kysely": "^1.8.0",
"reflect-metadata": "^0.2.0",
"rxjs": "^7.8.1"
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.2"
},
"devDependencies": {
"@nestjs/cli": "^10.0.0",
"@nestjs/schematics": "^10.0.0",
"@nestjs/testing": "^10.0.0",
"@types/express": "^4.17.17",
"@types/jest": "^29.5.2",
"@types/node": "^20.3.1",
"@types/supertest": "^6.0.0",
"@typescript-eslint/eslint-plugin": "^7.0.0",
"@typescript-eslint/parser": "^7.0.0",
"eslint": "^8.42.0",
"eslint-config-prettier": "^9.0.0",
"eslint-plugin-prettier": "^5.0.0",
"jest": "^29.5.0",
"prettier": "^3.0.0",
"prisma": "^5.16.2",
"@nestjs/cli": "^11.0.6",
"@nestjs/schematics": "^11.0.5",
"@nestjs/testing": "^11.0.15",
"@types/express": "^5.0.1",
"@types/jest": "^29.5.14",
"@types/node": "^22.14.0",
"@types/supertest": "^6.0.3",
"@typescript-eslint/eslint-plugin": "^8.29.1",
"@typescript-eslint/parser": "^8.29.1",
"eslint": "^9.24.0",
"eslint-config-prettier": "^10.1.2",
"eslint-plugin-prettier": "^5.2.6",
"jest": "^29.7.0",
"prettier": "^3.5.3",
"prisma": "^6.6.0",
"source-map-support": "^0.5.21",
"supertest": "^7.0.0",
"ts-jest": "^29.1.0",
"ts-loader": "^9.4.3",
"ts-node": "^10.9.1",
"supertest": "^7.1.0",
"ts-jest": "^29.3.1",
"ts-loader": "^9.5.2",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"typescript": "^5.7.2"
"typescript": "^5.8.3"
},
"jest": {
"moduleFileExtensions": [

7
service/data-hub/data-connector/src/app.controller.ts
View File

@ -1,4 +1,4 @@
import { Controller, Get, Query } from '@nestjs/common';
import { Controller, Get, Logger, Query } from '@nestjs/common';
import { PrismaService } from './core/database/prisma.service';
import { AnomaliesService } from './modules/feat-anomalies/anomalies.service';
import { CalculationService } from './modules/feat-calculation/calculation.service';
@ -7,6 +7,8 @@ import { TicketService } from './modules/feat-tickets/tickets.service';
@Controller()
export class AppController {
private logger = new Logger(AppController.name);
constructor(
private readonly importer: ImporterService,
private readonly anomaliesService: AnomaliesService,
@ -17,6 +19,8 @@ export class AppController {
@Get('etl')
etl(@Query('from') from: string, @Query('to') to: string) {
this.logger.log(`Start ETL for interval from=${from} to=${to}`);
if (!from || !to) {
return {
error: 'from and to must be defined',
@ -146,7 +150,6 @@ export class AppController {
revenue: await this.calculationService.calculateTourRevenue(tour),
});
}
console.log(this.calculationService.getHealthInsurancesWithNoTenant());
return toursWithRevenue;
}
}

38
service/data-hub/data-connector/src/instrument.js
View File

@ -1,38 +0,0 @@
/* eslint-disable @typescript-eslint/no-var-requires */
/*instrumentation.ts*/
const { NodeSDK } = require('@opentelemetry/sdk-node');
const {
getNodeAutoInstrumentations,
} = require('@opentelemetry/auto-instrumentations-node');
const {
OTLPTraceExporter,
} = require('@opentelemetry/exporter-trace-otlp-proto'); // Annahme: gRPC/Proto Exporter für Backend
const {
OTLPMetricExporter,
} = require('@opentelemetry/exporter-metrics-otlp-proto'); // Annahme: gRPC/Proto Exporter für Backend
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { resourceFromAttributes } = require('@opentelemetry/resources');
const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions');
const otelEndpoint =
process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ??
'http://monitoring_alloy:4318';
const sdk = new NodeSDK({
resource: resourceFromAttributes({
[ATTR_SERVICE_NAME]: 'avicenna.data-connector',
namespace: 'avicenna',
}),
traceExporter: new OTLPTraceExporter({
url: `${otelEndpoint}/v1/traces`,
}),
metricReader: new PeriodicExportingMetricReader({
exporter: new OTLPMetricExporter({
url: `${otelEndpoint}/v1/metrics`,
}),
}),
instrumentations: [getNodeAutoInstrumentations()],
});
sdk.start();
console.log('OTel SDK started');

5
service/data-hub/data-connector/src/main.ts
View File

@ -1,8 +1,11 @@
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
import { ConsoleLogger } from '@nestjs/common';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
const app = await NestFactory.create(AppModule, {
logger: new ConsoleLogger({ json: true }),
});
app.enableCors();
await app.listen(3000);

2
service/data-hub/data-connector/src/modules/app-manager/manager.resolver.ts
View File

@ -39,7 +39,7 @@ export class ManagerResolver {
@Subscription(() => TimePerTourUpdateObjectType)
timePerTourUpdated() {
return this.timePerTourKpiService.pubSub.asyncIterator(
return this.timePerTourKpiService.pubSub.asyncIterableIterator(
'timePerTourUpdated',
);
}

1
service/data-hub/data-connector/src/modules/app-ticket-system/ticket-lifecycle.service.ts
View File

@ -92,7 +92,6 @@ export class TicketLifecycleService {
private async validateCreatedTickets(): Promise<void> {
const tickets = await this.repository.findAllCreated();
this.logger.debug(`Found ${tickets.length} with state CREATED`);
for (const ticket of tickets) {
const {

1
service/data-hub/data-connector/src/modules/app-ticket-system/ticket.repository.ts
View File

@ -362,7 +362,6 @@ export class TicketRepository {
const dwellTimesForTicketState = ticketWithState
.map(({ state }) => {
console.log(state);
const stateToCalculateIdx = state.findIndex(
({ state }) => state === ticketState,
);

2
service/data-hub/data-connector/src/modules/app-ticket-system/tickets.resolver.ts
View File

@ -77,6 +77,6 @@ export class TicketsResolver {
@Subscription(() => TicketsUpdatedObjectType)
ticketsUpdated() {
return this.ticketsService.pubSub.asyncIterator('ticketsUpdated');
return this.ticketsService.pubSub.asyncIterableIterator('ticketsUpdated');
}
}

2
service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/driver-certificate.detector.ts
View File

@ -82,7 +82,7 @@ export class DriverCertificateDetector extends AnomalyDetector {
};
}
} catch (error) {
this.logger.error(error, anomaly.id);
this.logger.error(error, { anomalyId: anomaly.id });
return anomaly;
}
}

20
service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/employees-logged-in.detector.ts
View File

@ -68,7 +68,7 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
return anomaly;
}
} catch (error) {
this.logger.error(error, anomaly.id);
this.logger.error(error, { anomalyId: anomaly.id });
return anomaly;
}
}
@ -200,12 +200,11 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
return false;
}
if (!employee.personnelNumber) {
this.logger.warn(
`Employee has no personnelNumber`,
employee.id,
employee.surname,
employee.name,
);
this.logger.warn(`Employee has no personnelNumber`, {
id: employee.id,
surname: employee.surname,
name: employee.name,
});
return false;
}
return true;
@ -313,13 +312,12 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
return [false, undefined];
}
} catch (error) {
this.logger.error(
error,
datetimeToCheck.toISOString(),
this.logger.error(error, {
timeInQuestion: datetimeToCheck.toISOString(),
workTime,
firstClockedIn,
lastClockedOut,
);
});
return [false, undefined];
}
}

9
service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/one-tour-at-a-time.detector.ts
View File

@ -131,12 +131,11 @@ export class OneTourAtATimeDetector extends AnomalyDetector {
tourListInterval,
);
} catch (error) {
this.logger.error(
'tourListInterval',
error,
this.logger.error(error, {
what: 'tourListInterval',
tourListInterval,
tourInterval,
);
});
return false;
}
},
@ -299,7 +298,7 @@ export class OneTourAtATimeDetector extends AnomalyDetector {
}
return { ...anomaly };
} catch (error) {
this.logger.error(error, anomaly.id);
this.logger.error(error, { anomalyId: anomaly.id });
return anomaly;
}
}

2
service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/working-employee-not-in-distribution-system.detector.ts
View File

@ -66,7 +66,7 @@ export class WorkingEmployeeNotInDistributionSystemDetector extends AnomalyDetec
} catch (error) {
this.logger.warn(
`Employee does not exist! PersonnelNumber: ${personnelNumber}`,
error,
{ error },
);
return;
}

9
service/data-hub/data-connector/src/modules/feat-calculation/calculation.service.ts
View File

@ -51,7 +51,7 @@ export class CalculationService {
new Date(startDate),
);
if (!tariff) {
this.logger.error(
this.logger.warn(
`No tariff found for tour ${tour.operationId} (${tour.healthInsurance})`,
);
return 0;
@ -163,11 +163,10 @@ export class CalculationService {
),
);
if (!billingInformation) {
this.logger.error(
"Couldn't find billing information",
this.logger.warn("Couldn't find billing information", {
date: date.toISOString(),
ordinanceType,
date.toISOString(),
);
});
throw new Error("Couldn't find billing information");
}
return billingInformation[1];

1
service/data-hub/data-connector/src/modules/feat-etl/feat-loaders/loaders/planned-time.loader.ts
View File

@ -12,7 +12,6 @@ export class PlannedTimeLoader {
const deletedPlannedTimes = stagingPlannedTimes.filter(
({ deleted }) => deleted,
);
console.log(deletedPlannedTimes.length);
const plannedTimesToInsert = stagingPlannedTimes.filter(
({ id }) =>
!!id &&