64 lines
2.3 KiB
YAML
64 lines
2.3 KiB
YAML
id: avic-etl
|
|
namespace: dev
|
|
description: Run Avicenna ETL
|
|
|
|
labels:
|
|
env: dev
|
|
project: avicenna
|
|
|
|
triggers:
|
|
- id: schedule
|
|
type: io.kestra.plugin.core.trigger.Schedule
|
|
cron: "*/5 * * * *"
|
|
|
|
tasks:
|
|
- id: getPartitions
|
|
type: io.kestra.plugin.scripts.python.Script
|
|
taskRunner:
|
|
type: io.kestra.plugin.scripts.runner.docker.Docker
|
|
containerImage: ghcr.io/kestra-io/pydata:latest
|
|
beforeCommands:
|
|
- pip install -U arrow
|
|
script: |
|
|
from kestra import Kestra
|
|
import arrow
|
|
partitions = []
|
|
|
|
now = arrow.utcnow() #.shift(months=-1)
|
|
partitions.append("from={}&to={}".format(now.span('month')[0].format('YYYY-MM-DD'), now.shift(days=14).format('YYYY-MM-DD')))
|
|
lastMonth = now.shift(months=-1)
|
|
partitions.append("from={}&to={}".format(lastMonth.span('month')[0].format('YYYY-MM-DD'), lastMonth.span('month')[-1].format('YYYY-MM-DD')))
|
|
monthBeforeLast = lastMonth.shift(months=-1)
|
|
partitions.append("from={}&to={}".format(monthBeforeLast.span('month')[0].format('YYYY-MM-DD'), monthBeforeLast.span('month')[-1].format('YYYY-MM-DD')))
|
|
monthBeforeLast = monthBeforeLast.shift(months=-1)
|
|
partitions.append("from={}&to={}".format(monthBeforeLast.span('month')[0].format('YYYY-MM-DD'), monthBeforeLast.span('month')[-1].format('YYYY-MM-DD')))
|
|
Kestra.outputs({'partitions': partitions})
|
|
- id: loadThreeMonths
|
|
type: io.kestra.plugin.core.flow.EachSequential
|
|
value: '{{ outputs.getPartitions.vars.partitions }}'
|
|
tasks:
|
|
- id: "request"
|
|
type: "io.kestra.plugin.core.http.Request"
|
|
uri: "http://avicenna_data:3000/etl?{{ taskrun.value }}"
|
|
headers:
|
|
user-agent: "kestra-io"
|
|
options:
|
|
readTimeout: "PT180S"
|
|
method: "GET"
|
|
# - id: checkForAnomalies
|
|
# type: "io.kestra.plugin.core.http.Request"
|
|
# uri: "http://avicenna_avicenna-data:3000/anomalies?from=2024-07-01&to=2024-07-31"
|
|
# headers:
|
|
# user-agent: "kestra-io"
|
|
# options:
|
|
# readTimeout: "PT90S"
|
|
# method: "GET"
|
|
# - id: updateTickets
|
|
# type: "io.kestra.plugin.core.http.Request"
|
|
# uri: "http://avicenna_avicenna-data:3000/tickets?from=2024-07-01&to=2024-07-31"
|
|
# headers:
|
|
# user-agent: "kestra-io"
|
|
# options:
|
|
# readTimeout: "PT90S"
|
|
# method: "GET"
|