From 51a2eedca145241cc7d802f59ffacff64fa4a2ed Mon Sep 17 00:00:00 2001 From: "Ruben L. Mendoza" Date: Tue, 16 May 2023 14:24:52 -0500 Subject: [PATCH] Osm simple metrics container (#291) * Add osm-simple-metrics container * Add chartpress section for osm-simple-metrics and helm template * Set fixed port for DB in osm-metrics * Fix mapping values - context, job metrics * Fix path for aws bucket name * Get metrics code from main branch --- chartpress.yaml | 4 +- compose/osm-simple-metrics.yml | 10 +++ images/osm-simple-metrics/Dockerfile | 11 +++ images/osm-simple-metrics/README.md | 18 +++++ images/osm-simple-metrics/start.sh | 14 ++++ .../jobs/osm-simple-metrics-job.yaml | 68 +++++++++++++++++++ osm-seed/values.yaml | 19 ++++++ 7 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 compose/osm-simple-metrics.yml create mode 100644 images/osm-simple-metrics/Dockerfile create mode 100644 images/osm-simple-metrics/README.md create mode 100755 images/osm-simple-metrics/start.sh create mode 100644 osm-seed/templates/jobs/osm-simple-metrics-job.yaml diff --git a/chartpress.yaml b/chartpress.yaml index 763629a2..1300479f 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -37,4 +37,6 @@ charts: overpass-api: valuesPath: overpassApi.image taginfo: - valuesPath: taginfo.image \ No newline at end of file + valuesPath: taginfo.image + osm-simple-metrics: + valuesPath: osmSimpleMetrics.image \ No newline at end of file diff --git a/compose/osm-simple-metrics.yml b/compose/osm-simple-metrics.yml new file mode 100644 index 00000000..bb206b71 --- /dev/null +++ b/compose/osm-simple-metrics.yml @@ -0,0 +1,10 @@ +version: '3' +services: + osm-simple-metrics: + image: osm-simple-metrics:v1 + build: + context: ../images/osm-simple-metrics + dockerfile: Dockerfile + env_file: + - ../envs/.env.db + - ../envs/.env.cloudprovider diff --git a/images/osm-simple-metrics/Dockerfile b/images/osm-simple-metrics/Dockerfile new file mode 100644 index 00000000..0e065d04 --- /dev/null +++ b/images/osm-simple-metrics/Dockerfile @@ -0,0 +1,11 @@ +FROM node:18 + +# Create app directory +WORKDIR /usr/src/app + +# Clone +RUN git clone https://github.com/developmentseed/osm-simple-metrics.git ./ + +RUN npm install && npm link +COPY start.sh / +CMD /start.sh diff --git a/images/osm-simple-metrics/README.md b/images/osm-simple-metrics/README.md new file mode 100644 index 00000000..050bbf2e --- /dev/null +++ b/images/osm-simple-metrics/README.md @@ -0,0 +1,18 @@ +# OSM Simple Metrics + +This is a container that allows to extract simple metrics calculations for an OSM database. + +Repo: https://github.com/developmentseed/osm-simple-metrics + +#### Building the container + + +```sh +docker compose -f compose/osm-simple-metrics.yml build +``` + +#### Access the container + +```sh +docker-compose -f compose/osm-simple-metrics.yml run osm-simple-metrics bash +``` diff --git a/images/osm-simple-metrics/start.sh b/images/osm-simple-metrics/start.sh new file mode 100755 index 00000000..149ac6a7 --- /dev/null +++ b/images/osm-simple-metrics/start.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +conString=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:5432/${POSTGRES_DB} +bucket_name="${AWS_S3_BUCKET#s3://}" + +CLIS=(cumulative_elements current_elements changeset_counts) +for cli in "${CLIS[@]}"; do + echo "Star running... $cli" + if [ $CLOUDPROVIDER == "aws" ]; then + osm-simple-metrics $cli \ + $conString \ + -csv \ + -s3 $bucket_name + fi +done diff --git a/osm-seed/templates/jobs/osm-simple-metrics-job.yaml b/osm-seed/templates/jobs/osm-simple-metrics-job.yaml new file mode 100644 index 00000000..cb8f0f57 --- /dev/null +++ b/osm-seed/templates/jobs/osm-simple-metrics-job.yaml @@ -0,0 +1,68 @@ +{{- if .Values.osmSimpleMetrics.enabled -}} +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-osm-simple-metrics-job + labels: + app: {{ template "osm-seed.name" . }} + component: osm-simple-metrics-job + environment: {{ .Values.environment }} + release: {{ .Release.Name }} +spec: + schedule: {{ quote .Values.osmSimpleMetrics.schedule }} + startingDeadlineSeconds: 100 + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 2 + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: {{ .Release.Name }}-osm-simple-metrics-job + image: {{ .Values.osmSimpleMetrics.image.name }}:{{ .Values.osmSimpleMetrics.image.tag }} + {{- if .Values.osmSimpleMetrics.resources.enabled }} + resources: + requests: + memory: {{ .Values.osmSimpleMetrics.resources.requests.memory }} + cpu: {{ .Values.osmSimpleMetrics.resources.requests.cpu }} + limits: + memory: {{ .Values.osmSimpleMetrics.resources.limits.memory }} + cpu: {{ .Values.osmSimpleMetrics.resources.limits.cpu }} + {{- end }} + env: + - name: POSTGRES_HOST + value: {{ .Release.Name }}-db + - name: POSTGRES_DB + value: {{ .Values.db.env.POSTGRES_DB }} + - name: POSTGRES_PASSWORD + value: {{ quote .Values.db.env.POSTGRES_PASSWORD }} + - name: POSTGRES_USER + value: {{ .Values.db.env.POSTGRES_USER }} + - name: CLOUDPROVIDER + value: {{ .Values.cloudProvider }} + # In case cloudProvider=aws + {{- if eq .Values.cloudProvider "aws" }} + - name: AWS_S3_BUCKET + value: {{ .Values.AWS_S3_BUCKET }} + {{- end }} + # In case cloudProvider=gcp + {{- if eq .Values.cloudProvider "gcp" }} + - name: GCP_STORAGE_BUCKET + value: {{ .Values.GCP_STORAGE_BUCKET }} + {{- end }} + # In case cloudProvider=azure + {{- if eq .Values.cloudProvider "azure" }} + - name: AZURE_STORAGE_ACCOUNT + value: {{ .Values.AZURE_STORAGE_ACCOUNT }} + - name: AZURE_CONTAINER_NAME + value: {{ .Values.AZURE_CONTAINER_NAME }} + - name: AZURE_STORAGE_CONNECTION_STRING + value: {{ .Values.AZURE_STORAGE_CONNECTION_STRING }} + {{- end }} + restartPolicy: OnFailure + {{- if .Values.osmSimpleMetrics.nodeSelector.enabled }} + nodeSelector: + {{ .Values.osmSimpleMetrics.nodeSelector.label_key }}: {{ .Values.osmSimpleMetrics.nodeSelector.label_value }} + {{- end }} +{{- end }} diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 30d9c561..330cc8dc 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -646,3 +646,22 @@ taginfo: cpu: '2' nodeSelector: enabled: false +# ==================================================================================================== +# Variables for osm-simple-metrics +# ==================================================================================================== +osmSimpleMetrics: + enabled: false + image: + name: '' + tag: '' + schedule: '* */1 * * *' + resources: + enabled: false + requests: + memory: '1Gi' + cpu: '2' + limits: + memory: '2Gi' + cpu: '2' + nodeSelector: + enabled: false