marndt/avicenna
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add log aggregation with HyperDX

Browse Source
This commit is contained in:
Marcel Arndt
2025-12-12 12:00:22 +01:00
parent 0951282f11
commit 62e663d053
21 changed files with 3463 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/data-hub/data-connector/Dockerfile
+2 -2
View File
@@ -28,9 +28,9 @@ WORKDIR /app
COPY --from=builder /app/builder/node_modules ./node_modules
COPY --from=builder /app/builder/prisma ./prisma
COPY --from=builder /app/builder/dist .
COPY --from=builder /app/builder/dist ./dist
COPY --from=builder /app/builder/docker-entrypoint.sh .
ENTRYPOINT [ "./docker-entrypoint.sh" ]
CMD ["node", "./src/main.js" ]
CMD ["node", "./dist/src/main.js" ]
service/data-hub/data-connector/package-lock.json
Generated
+2302 -37
View File
File diff suppressed because it is too large Load Diff
service/data-hub/data-connector/package.json
+3
View File
@@ -26,6 +26,8 @@
"dependencies": {
"@apollo/server": "^4.12.0",
"@faker-js/faker": "^9.6.0",
"@hyperdx/node-logger": "^0.4.0",
"@hyperdx/node-opentelemetry": "^0.10.2",
"@nestjs-modules/mailer": "^2.0.2",
"@nestjs/apollo": "^13.0.4",
"@nestjs/axios": "^4.0.0",
@@ -40,6 +42,7 @@
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/auto-instrumentations-node": "^0.60.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.202.0",
"@opentelemetry/instrumentation-runtime-node": "^0.22.0",
"@opentelemetry/sdk-node": "^0.202.0",
"@prisma/client": "^6.6.0",
"bullmq": "^5.56.0",
service/data-hub/data-connector/src/app.controller.ts
+5 -4
View File
@@ -1,4 +1,5 @@
import { Controller, Get, Logger, Query } from '@nestjs/common';
import { endOfDay, startOfDay } from 'date-fns';
import { PrismaService } from './core/database/prisma.service';
import { AnomaliesService } from './modules/feat-anomalies/anomalies.service';
import { CalculationService } from './modules/feat-calculation/calculation.service';
@@ -7,9 +8,8 @@ import { TicketService } from './modules/feat-tickets/tickets.service';
@Controller()
export class AppController {
private logger = new Logger(AppController.name);
constructor(
private readonly logger: Logger,
private readonly importer: ImporterService,
private readonly anomaliesService: AnomaliesService,
private readonly ticketService: TicketService,
@@ -30,6 +30,7 @@ export class AppController {
start: new Date(from),
end: new Date(to),
};
return this.importer.runImportForRange(interval);
}
@@ -41,8 +42,8 @@ export class AppController {
};
}
const interval = {
start: new Date(from),
end: new Date(to),
start: startOfDay(new Date(from)),
end: endOfDay(new Date(to)),
};
return this.ticketService.createTickets(interval);
}
service/data-hub/data-connector/src/core/database/schema.gql
+1 -1
View File
@@ -69,7 +69,7 @@ type AttendanceRegistration {
dateTime: DateTime!
employeeId: Int!
event: String!
personnelNumber: String!
personnelNumber: String
}
"""
service/data-hub/data-connector/src/core/middleware/request-logger.middleware.ts
+25
View File
@@ -0,0 +1,25 @@
import { Injectable, NestMiddleware, Logger } from '@nestjs/common';
import { Request, Response, NextFunction } from 'express';
@Injectable()
export class RequestLoggerMiddleware implements NestMiddleware {
private logger = new Logger('HTTP');
use(request: Request, response: Response, next: NextFunction): void {
const { ip, method, originalUrl } = request;
const userAgent = request.get('user-agent') || '';
const start = Date.now();
response.on('finish', () => {
const { statusCode } = response;
const contentLength = response.get('content-length');
const duration = Date.now() - start;
this.logger.log(
`${method} ${originalUrl} ${statusCode} ${contentLength} - ${userAgent} ${ip} +${duration}ms`
);
});
next();
}
}
service/data-hub/data-connector/src/instrumentation.ts
+22
View File
@@ -0,0 +1,22 @@
import * as HyperDX from '@hyperdx/node-opentelemetry';
import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node';
HyperDX.init({
apiKey: '71addf09-bcd2-4e83-8158-97458402137b',
service: 'avicenna-data-connector',
instrumentations: {
'@opentelemetry/instrumentation-http': { enabled: true },
'@opentelemetry/instrumentation-express': { enabled: true },
'@opentelemetry/instrumentation-nestjs-core': { enabled: true },
'@opentelemetry/instrumentation-graphql': { enabled: true },
'@opentelemetry/instrumentation-pg': { enabled: true },
},
additionalInstrumentations: [
//@ts-ignore
new RuntimeNodeInstrumentation({
monitoringPrecision: 5000,
}),
],
});
service/data-hub/data-connector/src/main.ts
+11 -4
View File
@@ -1,16 +1,23 @@
// const tracer = require('./tracer');
import tracer from './tracer'
// import tracer from './tracer'
import { HyperDXNestLoggerModule } from '@hyperdx/node-logger';
import * as HyperDX from '@hyperdx/node-opentelemetry';
import { NestFactory } from '@nestjs/core';
import { AppModule } from './app.module';
import { ConsoleLogger } from '@nestjs/common';
import './instrumentation';
async function bootstrap() {
await tracer.start();
const logger = HyperDXNestLoggerModule.createLogger({
apiKey: '71addf09-bcd2-4e83-8158-97458402137b',
service: 'avicenna-data-connector',
maxLevel: 'info',
});
const app = await NestFactory.create(AppModule, {
logger: new ConsoleLogger({ json: true }),
logger: logger,
});
app.enableCors();
HyperDX.setupExpressErrorHandler(app);
await app.listen(3000);
}
service/data-hub/data-connector/src/modules/app-control-center/anomaly.object-type.ts
+2 -2
View File
@@ -33,7 +33,7 @@ export class PlannedTimeObjectType implements PlannedTime {
pause: number;
@Field(() => Int)
duration: number;
@Field({nullable: true})
@Field({ nullable: true })
note: string;
}
@@ -83,7 +83,7 @@ export class EmployeeObjectType implements Employee {
export class AttendanceRegistrationObjectType
implements AttendanceRegistration
{
@Field()
@Field({ nullable: true })
personnelNumber: string;
@Field(() => Int)
attendanceRegistrationId: number;
service/data-hub/data-connector/src/modules/feat-anomalies/anomaly-detectors/employees-logged-in.detector.ts
+1 -1
View File
@@ -217,7 +217,7 @@ export class EmployeesLoggedInDetector extends AnomalyDetector {
return Promise.resolve(undefined);
}
private validateEmployee(employee: Employee | undefined) {
private validateEmployee(employee: Employee | undefined): employee is Employee {
if (!employee) {
this.logger.warn(`Employee does not exist`);
return false;
service/data-hub/data-connector/src/modules/feat-business-objects/entities/tour/tour.repository.ts
-62
View File
@@ -14,68 +14,6 @@ type TourEb = ExpressionBuilder<DB, 'Tour'>;
const PAUSE_PATIENT_ID = 'b7424477-a4b2-42f7-b7e7-a4bef24f6979';
const FEIERABEND_PATIENT_ID = '3554e1a9-9c07-45e5-b53c-692a77e4fa9a';
function onlyRealTours(): Prisma.TourWhereInput[] {
return [
{
OR: [
{
carName: null,
},
{
AND: [
{
carName: {
not: 'Storno',
},
},
{
carName: {
not: 'Löschen',
},
},
{
carName: {
not: 'Abgabe',
},
},
],
},
],
},
{
check: {
not: 3,
},
},
{
deletedAt: null,
},
{
OR: [
{
patientId: null,
},
{
AND: [
{
patientId: {
// Pause
not: PAUSE_PATIENT_ID,
},
},
{
patientId: {
// Feierabend
not: FEIERABEND_PATIENT_ID,
},
},
],
},
],
},
];
}
function onlyRealToursKysely(eb: TourEb) {
return eb.and([
eb.or([
service/data-hub/data-connector/src/modules/feat-etl/feat-importers/feature-importer.module.ts
+5 -5
View File
@@ -1,20 +1,20 @@
import { BullModule } from '@nestjs/bullmq';
import { Module } from '@nestjs/common';
import { MailModule } from 'src/core/mail/mail.module';
import { MAILING_QUEUE } from 'src/core/mail/mail.tokens';
import { TOURS_QUEUE } from 'src/modules/feat-business-objects/entities/tour/tour.tokens';
import { FeatBusinessObjectsModule } from 'src/modules/feat-business-objects/feat-business-objects.module';
import { DataAccessExtractorsModule } from '../feat-extractors/data-access-extractors.module';
import { DataAccessLoadersModule } from '../feat-loaders/data-access-loaders.module';
import { DataAccessTransformersModule } from '../feat-transformers/data-access-transformers.module';
import { ImporterService } from './importer.service';
import { MailConsumer } from './processors/mail.consumer';
import { SchichtplanungConsumer } from './processors/schichtplanung.consumer';
import { TourFileUploadConsumer } from './processors/tour-file-upload.consumer';
import { ToursConsumer } from './processors/tours.consumer';
import { SchichtplanungSynchronizer } from './producer/schichtplanung';
import { TourUpdateChecker } from './producer/tour-update';
import { SCHICHTPLANUNG_QUEUE } from './tokens';
import { TourFileUploadConsumer } from './processors/tour-file-upload.consumer';
import { MAILING_QUEUE } from 'src/core/mail/mail.tokens';
import { MailConsumer } from './processors/mail.consumer';
import { MailModule } from 'src/core/mail/mail.module';
@Module({
imports: [
@@ -34,7 +34,7 @@ import { MailModule } from 'src/core/mail/mail.module';
SchichtplanungSynchronizer,
SchichtplanungConsumer,
TourFileUploadConsumer,
MailConsumer
MailConsumer,
],
exports: [ImporterService],
})
service/data-hub/data-connector/src/modules/feat-etl/feat-importers/importer.service.ts
+4 -4
View File
@@ -4,6 +4,7 @@ import { randomUUID } from 'crypto';
import { Interval } from 'date-fns';
import { Observable, combineLatest, from, map, switchMap, tap } from 'rxjs';
import { DyflexisUserAccessExtractorsService } from '../feat-extractors/dyflexis-user-access-extractors.service';
import { ExtractorsService } from '../feat-extractors/extractors.service';
import { LoadersService } from '../feat-loaders/loaders.service';
import { TransformersService } from '../feat-transformers/transformers.service';
@@ -11,9 +12,8 @@ import { ETLEventTypes } from './events/event-types.enum';
import { ExtractionDoneEvent } from './events/extraction-done.event';
import { LoadingDoneEvent } from './events/loading-done.event';
import { TransformationDoneEvent } from './events/transformation-done.event';
import { TourUpdateChecker } from './producer/tour-update';
import { SchichtplanungSynchronizer } from './producer/schichtplanung';
import { DyflexisUserAccessExtractorsService } from '../feat-extractors/dyflexis-user-access-extractors.service';
import { TourUpdateChecker } from './producer/tour-update';
@Injectable()
export class ImporterService {
@@ -25,11 +25,11 @@ export class ImporterService {
private readonly eventEmitter: EventEmitter2,
private readonly tourUpdateChecker: TourUpdateChecker,
private readonly schichtplanungSynchronizer: SchichtplanungSynchronizer,
private readonly dyflexisUserAccessService: DyflexisUserAccessExtractorsService
private readonly dyflexisUserAccessService: DyflexisUserAccessExtractorsService,
) {}
runDyflexisUserAccessTest() {
return this.dyflexisUserAccessService.loadData()
return this.dyflexisUserAccessService.loadData();
}
runTourUpdateCheck(interval: Interval) {
service/data-hub/data-connector/src/modules/feat-etl/feat-importers/producer/schichtplanung.ts
+23 -12
View File
@@ -1,10 +1,15 @@
import { InjectQueue } from '@nestjs/bullmq';
import { Injectable, Logger } from '@nestjs/common';
import { Queue } from 'bullmq';
import { addMinutes, endOfDay, getMonth, getYear, isSameDay, startOfDay } from 'date-fns';
import {
lastValueFrom
} from 'rxjs';
addMinutes,
endOfDay,
getMonth,
getYear,
isSameDay,
startOfDay,
} from 'date-fns';
import { lastValueFrom } from 'rxjs';
import { EmployeeService } from 'src/modules/feat-business-objects/entities/employee/employee.service';
import { DispoLiveApiExtractorsService } from '../../feat-extractors/dispo-live-api-extractors.service';
import { DyflexisExtractorsService } from '../../feat-extractors/dyflexis-extractors.service';
@@ -17,7 +22,7 @@ import { SCHICHTPLANUNG_QUEUE } from '../tokens';
@Injectable()
export class SchichtplanungSynchronizer {
private logger = new Logger(SchichtplanungSynchronizer.name)
private logger = new Logger(SchichtplanungSynchronizer.name);
constructor(
private readonly dlApiService: DispoLiveApiExtractorsService,
@@ -42,7 +47,9 @@ export class SchichtplanungSynchronizer {
.filter(({ user_id }) => user_id != '0')
.filter(
({ department_name }) =>
department_name !== 'Büro' && department_name !== 'Leitstelle',
department_name !== 'Büro' &&
department_name !== 'Leitstelle' &&
department_name !== 'Fortbildung',
)
.filter(({ start_date }) => isSameDay(new Date(start_date), day));
@@ -63,7 +70,9 @@ export class SchichtplanungSynchronizer {
plannedTime.user_id,
);
if (!employee) {
this.logger.error(`couldn\'t find employee with id ${plannedTime.user_id}`);
this.logger.error(
`couldn\'t find employee with id ${plannedTime.user_id}`,
);
continue;
}
@@ -71,7 +80,7 @@ export class SchichtplanungSynchronizer {
this.isCarNameForDepartment(plannedTime.department_name, kfzNr),
);
if (!car) {
console.error(
this.logger.error(
`couldn\'t find car with department ${plannedTime.department_name}`,
);
continue;
@@ -83,13 +92,13 @@ export class SchichtplanungSynchronizer {
end: end_date.toISOString(),
group: car._id,
mitarbeiterId: employee.id,
}
if(car.kfzNr.indexOf("KTW") > -1) {
};
if (car.kfzNr.indexOf('KTW') > -1) {
planning = {
...planning,
start: addMinutes(new Date(planning.start), 15).toISOString(),
}
};
}
schichtplanung.push(planning);
@@ -126,7 +135,9 @@ export class SchichtplanungSynchronizer {
}
if (
!!plannedTime &&
(plannedTime?.start !== plan.start || plannedTime?.end !== plan.end || plannedTime?.group !== plan.group)
(plannedTime?.start !== plan.start ||
plannedTime?.end !== plan.end ||
plannedTime?.group !== plan.group)
) {
toUpdate.push({ ...plan, _id: plannedTime._id });
}
service/data-hub/data-connector/src/modules/feat-tickets/ticket-lifecycle.service.ts
+1 -1
View File
@@ -122,7 +122,7 @@ export class TicketLifecycleService {
private async validateCreatedTickets(): Promise<void> {
const tickets = await this.ticketService.findAllCreated();
this.logger.debug(`Found ${tickets.length} with state CREATED`);
this.logger.log(`validating ${tickets.length} created tickets`);
for (const ticket of tickets) {
const {