From 98aeb82ff63e6827a785c8402e8067fac21abfc5 Mon Sep 17 00:00:00 2001 From: Marcel Arndt Date: Thu, 10 Apr 2025 17:31:33 +0200 Subject: [PATCH] persist infrastructure and plattform --- .gitignore | 6 +- infrastructure/Pulumi.prod.yaml | 3 + infrastructure/index.ts | 73 ++++++++++--------- plattform/docker-compose.deploy.yml | 79 +++++++++++++++++++++ plattform/docker-compose.yml | 104 ++++++++++++++++++++++++++++ plattform/docker/airflow/.gitkeep | 0 plattform/docker/spark/.gitkeep | 0 7 files changed, 230 insertions(+), 35 deletions(-) create mode 100644 infrastructure/Pulumi.prod.yaml create mode 100644 plattform/docker-compose.deploy.yml create mode 100644 plattform/docker-compose.yml delete mode 100644 plattform/docker/airflow/.gitkeep delete mode 100644 plattform/docker/spark/.gitkeep diff --git a/.gitignore b/.gitignore index f81e5d7..e735689 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -.local/ \ No newline at end of file +.local/ +.vscode/ + +.env +.DS_Store diff --git a/infrastructure/Pulumi.prod.yaml b/infrastructure/Pulumi.prod.yaml new file mode 100644 index 0000000..1fea73b --- /dev/null +++ b/infrastructure/Pulumi.prod.yaml @@ -0,0 +1,3 @@ +config: + avicenna-infrastructure:hcloudToken: + secure: AAABAIi5Eso+isiy15TT2VC45yG73iLmuwhaDFi8180Gm0Vd9zdVSmh9HRPXQMlSU+NxzQDqOx5qbgqvMDiyew4AHkEgUJvJUA0cV0Mn1B4WeGlhzzYmbgdHeVobgv8i diff --git a/infrastructure/index.ts b/infrastructure/index.ts index dfb7cf2..e281439 100644 --- a/infrastructure/index.ts +++ b/infrastructure/index.ts @@ -22,8 +22,13 @@ const sshKey = new tls.PrivateKey('sshKey', { export const privateKey = sshKey.privateKeyOpenssh; +const env = pulumi.getStack(); + +if (!fs.existsSync(`./.local/${env}`)) { + fs.mkdirSync(`./.local/${env}`); +} privateKey.apply((privKey) => - fs.writeFileSync('././local/private_key', privKey, { mode: '0600' }) + fs.writeFileSync(`./.local/${env}/private_key`, privKey, { mode: '0600' }) ); const hetznerSshKey = new hcloud.SshKey( @@ -163,40 +168,40 @@ async function createSwarmCluster(nodeCount: number) { return vm; } -const geniusceo = digitalocean.Domain.get( - 'geniusceo', - 'genius.ceo', -); +const doToken = config.getSecret('doToken'); +if (doToken) { + const geniusceo = digitalocean.Domain.get('geniusceo', 'genius.ceo'); -const wwwSubDomain = geniusceo.id.apply( - (id) => - new digitalocean.DnsRecord('www', { - domain: id, - type: 'A', - value: swarmMaster.ipv4Address, - name: 'www', - }) -); + const wwwSubDomain = geniusceo.id.apply( + (id) => + new digitalocean.DnsRecord('www', { + domain: id, + type: 'A', + value: swarmMaster.ipv4Address, + name: 'www', + }) + ); -const noSubDomain = geniusceo.id.apply( - (id) => - new digitalocean.DnsRecord('noSub', { - domain: id, - type: 'A', - value: swarmMaster.ipv4Address, - name: '@', - }) -); + const noSubDomain = geniusceo.id.apply( + (id) => + new digitalocean.DnsRecord('noSub', { + domain: id, + type: 'A', + value: swarmMaster.ipv4Address, + name: '@', + }) + ); -const wildcardSubDomain = geniusceo.id.apply( - (id) => - new digitalocean.DnsRecord('*', { - domain: id, - type: 'A', - value: swarmMaster.ipv4Address, - name: '*', - }) -); + const wildcardSubDomain = geniusceo.id.apply( + (id) => + new digitalocean.DnsRecord('*', { + domain: id, + type: 'A', + value: swarmMaster.ipv4Address, + name: '*', + }) + ); +} const installAnsibleDepsCmd = new command.remote.Command( 'installAnsibleDepsCmd', @@ -213,7 +218,7 @@ const installAnsibleDepsCmd = new command.remote.Command( const deployDockerStackCmd = new command.local.Command( 'deployDockerStackCmd', { - create: pulumi.interpolate`ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -u root -i "${swarmMaster.ipv4Address}," --private-key ./private_key ansible/playbook.yml`, + create: pulumi.interpolate`ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -u root -i "${swarmMaster.ipv4Address}," --private-key ./.local/${env}/private_key ansible/playbook.yml`, }, { dependsOn: [installAnsibleDepsCmd], @@ -233,4 +238,4 @@ const restartCmd = new command.remote.Command( { dependsOn: [deployDockerStackCmd], } -); \ No newline at end of file +); diff --git a/plattform/docker-compose.deploy.yml b/plattform/docker-compose.deploy.yml new file mode 100644 index 0000000..869a0fe --- /dev/null +++ b/plattform/docker-compose.deploy.yml @@ -0,0 +1,79 @@ +version: '3.3' + +networks: + data-hub_net: + traefik_public: + external: true + +volumes: + pgdata: + +services: + data: + image: thedevilisdero/avic-data-connector:latest + environment: + DATABASE_PORT: ${DATABASE_PORT} + DATABASE_HOST: ${DATABASE_HOST} + DATABASE_USERNAME: ${DATABASE_USERNAME} + DATABASE_PASSWORD: ${DATABASE_PASSWORD} + DATABASE_DATABASE: ${DATABASE_DATABASE} + DATABASE_URL: ${DATABASE_URL} + DISPO_LIVE_BASE_URL: ${DISPO_LIVE_BASE_URL} + DISPO_LIVE_USERNAME: ${DISPO_LIVE_USERNAME} + DISPO_LIVE_PASSWORD: ${DISPO_LIVE_PASSWORD} + DYFLEXIS_V0_BASE_URL: ${DYFLEXIS_V0_BASE_URL} + DYFLEXIS_V2_BASE_URL: ${DYFLEXIS_V2_BASE_URL} + DYFLEXIS_V0_API_KEY: ${DYFLEXIS_V0_API_KEY} + DYFLEXIS_V2_API_KEY: ${DYFLEXIS_V2_API_KEY} + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: "http://monitoring_alloy:4318/v1/traces" + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://monitoring_alloy:4318/v1/metrics" + OTEL_SERVICE_NAME: "avicenna.data-connector" + networks: + - data-hub_net + - traefik_public + deploy: + labels: + - "traefik.enable=true" + - "traefik.swarm.network=traefik_public" + - "traefik.http.routers.avic-data-con.rule=(Host(`avicenna.genius.ceo`) && PathPrefix(`/api/`)) || (Host(`avicenna.genius.ceo`) && Path(`/graphql`))" + - "traefik.http.routers.avic-data-con.entrypoints=https" + - "traefik.http.routers.avic-data-con.tls=true" + - "traefik.http.routers.avic-data-con.tls.certresolver=main" + - "traefik.http.services.avic-data-con.loadbalancer.server.port=3000" + + web: + image: thedevilisdero/avic-app-dashboard:latest + networks: + - traefik_public + environment: + - COLLECTOR_SECRET + - OTLP_ENDPOINT + - OTLP_SERVICE_NAME + - GRAPHQL_ENDPOINT + - GRAPHQL_WS_ENDPOINT + - MS_AUTHENTICATION_CLIENT_ID + - MS_AUTHENTICATION_AUTHORITY + - MS_AUTHENTICATION_REDIRECT_URI + - MS_AUTHENTICATION_POST_LOGOUT_REDIRECT_URI + deploy: + labels: + - "traefik.enable=true" + - "traefik.swarm.network=traefik_public" + - "traefik.http.routers.avicenna-web.rule=Host(`avicenna.genius.ceo`)" + - "traefik.http.routers.avicenna-web.entrypoints=https" + - "traefik.http.routers.avicenna-web.tls=true" + - "traefik.http.routers.avicenna-web.tls.certresolver=main" + - "traefik.http.services.avicenna-web.loadbalancer.server.port=80" + + db: + image: postgres:16.3 + environment: + POSTGRES_HOST: ${DATABASE_HOST} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} + POSTGRES_USER: ${DATABASE_USERNAME} + POSTGRES_DB: ${DATABASE_DATABASE} + PG_DATA: /var/lib/postgresql/data + volumes: + - pgdata:/var/lib/postgresql/data + networks: + - data-hub_net diff --git a/plattform/docker-compose.yml b/plattform/docker-compose.yml new file mode 100644 index 0000000..3e457eb --- /dev/null +++ b/plattform/docker-compose.yml @@ -0,0 +1,104 @@ +version: '3.3' + +networks: + data-hub_net: + +services: +# Automation + kestra-db: + image: postgres + volumes: + - ./.local/data/postgres:/var/lib/postgresql/data + env_file: + - .env + environment: + POSTGRES_DB: ${KESTRA_POSTGRES_DB} + POSTGRES_USER: ${KESTRA_POSTGRES_USER} + POSTGRES_PASSWORD: ${KESTRA_POSTGRES_PASSWORD} + networks: + data-hub_net: + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${KESTRA_POSTGRES_DB} -U $${KESTRA_POSTGRES_USER}"] + interval: 30s + timeout: 10s + retries: 10 + kestra: + image: kestra/kestra:latest-full + pull_policy: always + # Note that this is meant for development only. Refer to the documentation for production deployments of Kestra which runs without a root user. + user: "root" + command: server standalone --worker-thread=128 + networks: + data-hub_net: + volumes: + - ./.local/data/kestra:/app/storage + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/kestra-wd:/tmp/kestra-wd + environment: + KESTRA_CONFIGURATION: | + datasources: + postgres: + url: jdbc:postgresql://kestra-db:5432/kestra + driverClassName: org.postgresql.Driver + username: kestra + password: k3str4 + kestra: + server: + basic-auth: + enabled: false + username: "admin@kestra.io" # it must be a valid email address + password: kestra + repository: + type: postgres + storage: + type: local + local: + base-path: "/app/storage" + queue: + type: postgres + tasks: + tmp-dir: + path: /tmp/kestra-wd/tmp + url: http://localhost:8080/ + ports: + - "8080:8080" + - "8081:8081" + depends_on: + kestra-db: + condition: service_started + + avicenna-data: + image: thedevilisdero/avic-data-connector + ports: + - 3000:3000 + env_file: .env + environment: + DATABASE_PORT: ${DATABASE_PORT} + DATABASE_HOST: ${DATABASE_HOST} + DATABASE_USERNAME: ${DATABASE_USERNAME} + DATABASE_PASSWORD: ${DATABASE_PASSWORD} + DATABASE_DATABASE: ${DATABASE_DATABASE} + DATABASE_URL: ${DATABASE_URL} + DISPO_LIVE_BASE_URL: ${DISPO_LIVE_BASE_URL} + DISPO_LIVE_USERNAME: ${DISPO_LIVE_USERNAME} + DISPO_LIVE_PASSWORD: ${DISPO_LIVE_PASSWORD} + DYFLEXIS_V0_BASE_URL: ${DYFLEXIS_V0_BASE_URL} + DYFLEXIS_V2_BASE_URL: ${DYFLEXIS_V2_BASE_URL} + DYFLEXIS_V0_API_KEY: ${DYFLEXIS_V0_API_KEY} + DYFLEXIS_V2_API_KEY: ${DYFLEXIS_V2_API_KEY} + networks: + - data-hub_net + + avicenna-db: + image: postgres:16.3 + env_file: .env + environment: + POSTGRES_HOST: ${DATABASE_HOST} + POSTGRES_PASSWORD: ${DATABASE_PASSWORD} + POSTGRES_USER: ${DATABASE_USERNAME} + POSTGRES_DB: ${DATABASE_DATABASE} + PG_DATA: /var/lib/postgresql/data + volumes: + - ./.local/pgdata:/var/lib/postgresql/data + networks: + - data-hub_net diff --git a/plattform/docker/airflow/.gitkeep b/plattform/docker/airflow/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/plattform/docker/spark/.gitkeep b/plattform/docker/spark/.gitkeep deleted file mode 100644 index e69de29..0000000