(data-hub): add etl feature

service/data-hub/data-connector/src/core/config/config.module.ts

@@ -0,0 +1,11 @@
+import { Global, Module } from '@nestjs/common';
+import { ConfigModule as NestConfigModule } from '@nestjs/config';
+import { ConfigService } from './config.service';
+
+@Global()
+@Module({
+  imports: [NestConfigModule.forRoot()],
+  providers: [ConfigService],
+  exports: [ConfigService],
+})
+export class ConfigModule {}

service/data-hub/data-connector/src/core/config/config.service.ts

@@ -0,0 +1,27 @@
+import { Injectable } from '@nestjs/common';
+import { ConfigService as NestConfigService } from '@nestjs/config';
+
+@Injectable()
+export class ConfigService extends NestConfigService {
+  dispoliveBaseUrl(): string {
+    return this.get('DISPO_LIVE_BASE_URL');
+  }
+  dispoliveUsername(): string {
+    return this.get('DISPO_LIVE_USERNAME');
+  }
+  dispolivePassword(): string {
+    return this.get('DISPO_LIVE_PASSWORD');
+  }
+  dyflexisV0BaseUrl(): string {
+    return this.get('DYFLEXIS_V0_BASE_URL');
+  }
+  dyflexisV0ApiKey(): string {
+    return this.get('DYFLEXIS_V0_API_KEY');
+  }
+  dyflexisV2BaseUrl(): string {
+    return this.get('DYFLEXIS_V2_BASE_URL');
+  }
+  dyflexisV2ApiKey(): string {
+    return this.get('DYFLEXIS_V2_API_KEY');
+  }
+}

service/data-hub/data-connector/src/core/database/constants.ts

@@ -0,0 +1 @@
+export const PRISMA_SERVICE_OPTIONS = 'PRISMA_SERVICE_OPTIONS';

service/data-hub/data-connector/src/core/database/pagination.interface.ts

@@ -0,0 +1,9 @@
+export interface PaginationInterface<T> {
+  items: T[];
+  meta: {
+    totalItems: number;
+    items: number;
+    totalPages: number;
+    page: number;
+  };
+}

service/data-hub/data-connector/src/core/database/prisma-module.options.ts

@@ -0,0 +1,51 @@
+import { ModuleMetadata, Type } from "@nestjs/common";
+import { Prisma } from "@prisma/client";
+
+export interface PrismaModuleOptions {
+  /**
+   * If true, registers `PrismaModule` as a global module.
+   * See: https://docs.nestjs.com/modules#global-modules
+   */
+  isGlobal?: boolean;
+  prismaServiceOptions?: PrismaServiceOptions;
+}
+
+export interface PrismaServiceOptions {
+  /**
+   * Pass options directly to the `PrismaClient`.
+   * See: https://www.prisma.io/docs/reference/api-reference/prisma-client-reference/#prismaclient
+   */
+  prismaOptions?: Prisma.PrismaClientOptions;
+
+  /**
+   * If true, `PrismaClient` explicitly creates a connection pool and your first query will respond instantly.
+   *
+   * For most use cases the lazy connect behavior of `PrismaClient` will do. The first query of `PrismaClient` creates the connection pool.
+   * See: https://www.prisma.io/docs/concepts/components/prisma-client/working-with-prismaclient/connection-management
+   */
+  explicitConnect?: boolean;
+
+  /**
+   * Apply Prisma middlewares to perform actions before or after db queries.
+   *
+   * See: https://www.prisma.io/docs/concepts/components/prisma-client/middleware
+   */
+  middlewares?: Array<Prisma.Middleware>;
+}
+
+export interface PrismaOptionsFactory {
+  createPrismaOptions(): Promise<PrismaServiceOptions> | PrismaServiceOptions;
+}
+
+export interface PrismaModuleAsyncOptions
+  extends Pick<ModuleMetadata, 'imports'> {
+  isGlobal?: boolean;
+  useExisting?: Type<PrismaOptionsFactory>;
+  useClass?: Type<PrismaOptionsFactory>;
+  useFactory?: (
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ...args: any[]
+  ) => Promise<PrismaServiceOptions> | PrismaServiceOptions;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  inject?: any[];
+}

service/data-hub/data-connector/src/core/database/prisma.module.ts

@@ -0,0 +1,100 @@
+import { DynamicModule, Module, Provider } from '@nestjs/common';
+import { PRISMA_SERVICE_OPTIONS } from './constants';
+import {
+  PrismaModuleAsyncOptions,
+  PrismaModuleOptions,
+  PrismaOptionsFactory,
+} from './prisma-module.options';
+import { PrismaService } from './prisma.service';
+
+@Module({
+  providers: [PrismaService],
+  exports: [PrismaService],
+})
+export class PrismaModule {
+  static forRoot(options: PrismaModuleOptions = {}): DynamicModule {
+    return {
+      global: options.isGlobal,
+      module: PrismaModule,
+      providers: [
+        {
+          provide: PRISMA_SERVICE_OPTIONS,
+          useValue: options.prismaServiceOptions,
+        },
+      ],
+    };
+  }
+
+  static forRootAsync(options: PrismaModuleAsyncOptions): DynamicModule {
+    return {
+      global: options.isGlobal,
+      module: PrismaModule,
+      imports: options.imports || [],
+      providers: this.createAsyncProviders(options),
+    };
+  }
+
+  private static createAsyncProviders(
+    options: PrismaModuleAsyncOptions,
+  ): Provider[] {
+    if (options.useExisting || options.useFactory) {
+      return this.createAsyncOptionsProvider(options);
+    }
+
+    if (options.useClass) {
+      return [
+        ...this.createAsyncOptionsProvider(options),
+        {
+          provide: options.useClass,
+          useClass: options.useClass,
+        },
+      ];
+    }
+    return [...this.createAsyncOptionsProvider(options)];
+  }
+
+  private static createAsyncOptionsProvider(
+    options: PrismaModuleAsyncOptions,
+  ): Provider[] {
+    if (options.useFactory) {
+      return [
+        {
+          provide: PRISMA_SERVICE_OPTIONS,
+          useFactory: options.useFactory,
+          inject: options.inject || [],
+        },
+      ];
+    }
+
+    if (options.useExisting) {
+      return [
+        {
+          provide: PRISMA_SERVICE_OPTIONS,
+          useFactory: async (optionsFactory: PrismaOptionsFactory) =>
+            await optionsFactory.createPrismaOptions(),
+          inject: [options.useExisting],
+        },
+      ];
+    }
+
+    if (options.useClass) {
+      return [
+        {
+          provide: PRISMA_SERVICE_OPTIONS,
+          useFactory: async (optionsFactory: PrismaOptionsFactory) =>
+            await optionsFactory.createPrismaOptions(),
+          inject: [options.useClass],
+        },
+      ];
+    }
+
+    return [
+      {
+        provide: PRISMA_SERVICE_OPTIONS,
+        useFactory: async (optionsFactory: PrismaOptionsFactory) =>
+          await optionsFactory.createPrismaOptions(),
+        inject: [],
+      },
+    ];
+  }
+}

service/data-hub/data-connector/src/core/database/prisma.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test } from '@nestjs/testing';
+import { PrismaService } from './prisma.service';
+
+describe('PrismaService', () => {
+  let service: PrismaService;
+
+  beforeEach(async () => {
+    const module = await Test.createTestingModule({
+      providers: [PrismaService],
+    }).compile();
+
+    service = module.get(PrismaService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeTruthy();
+  });
+});

service/data-hub/data-connector/src/core/database/prisma.service.ts

@@ -0,0 +1,62 @@
+import {
+  Inject,
+  Injectable,
+  Logger,
+  OnModuleInit,
+  Optional,
+} from '@nestjs/common';
+import { PrismaClient } from '@prisma/client';
+import { PrismaClientOptions } from '@prisma/client/runtime/library';
+
+import { PRISMA_SERVICE_OPTIONS } from './constants';
+import { PrismaServiceOptions } from './prisma-module.options';
+
+@Injectable()
+export class PrismaService extends PrismaClient implements OnModuleInit {
+  private readonly logger = new Logger('Prisma');
+
+  constructor(
+    @Optional()
+    @Inject(PRISMA_SERVICE_OPTIONS)
+    private readonly prismaServiceOptions: PrismaServiceOptions = {},
+  ) {
+    super({
+      ...prismaServiceOptions.prismaOptions,
+      // Make sure queries are always emitted as events, so that we can log them.
+      log: [
+        { emit: 'event', level: 'query' },
+        ...(prismaServiceOptions.prismaOptions?.log
+          ? prismaServiceOptions.prismaOptions.log
+          : (['info', 'warn', 'error'] as PrismaClientOptions['log'])),
+      ],
+    });
+
+    if (this.prismaServiceOptions.middlewares) {
+      this.prismaServiceOptions.middlewares.forEach((middleware) =>
+        this.$use(middleware),
+      );
+    }
+
+    /* 
+      Sadly typescript does not recognize the Prisma Client configuration passed in the super call.
+      As Prisma defines the events generically, based on the configuration, the event type, typescript inferred, is wrong. 
+     */
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    //@ts-ignore
+    this.$on('query', (e: QueryEvent) => {
+      const { timestamp, query, params, duration } = e;
+      const message = `${timestamp.toISOString()} ${query} ${params} ${duration}`;
+      // this.logger.log(message);
+    });
+  }
+
+  async onModuleInit() {
+    if (this.prismaServiceOptions.explicitConnect) {
+      await this.$connect();
+    }
+  }
+
+  // async enableShutdownHooks(app: INestApplication) {
+  //   this.$on('beforeExit', async () => await app.close());
+  // }
+}