From 09a0d60360b51caa779743c981f072759c07d3dc Mon Sep 17 00:00:00 2001 From: 7va Date: Tue, 28 Oct 2025 13:59:14 -0400 Subject: [PATCH 01/49] add helm folder --- helm/.helmignore | 44 ++++++++++++++++++++++++++ helm/Chart.yaml | 7 ++++ helm/README.md | 17 ++++++++++ helm/templates/NOTES.txt | 9 ++++++ helm/templates/_helpers.tpl | 15 +++++++++ helm/templates/backend-deployment.yaml | 24 ++++++++++++++ helm/templates/backend-service.yaml | 14 ++++++++ helm/templates/db-service.yaml | 14 ++++++++ helm/templates/db-statefulset.yaml | 41 ++++++++++++++++++++++++ helm/templates/grafana-deployment.yaml | 26 +++++++++++++++ helm/templates/grafana-service.yaml | 14 ++++++++ helm/templates/ingress.yaml | 20 ++++++++++++ helm/templates/migrations-job.yaml | 21 ++++++++++++ helm/templates/pvc.yaml | 15 +++++++++ helm/values.yaml | 44 ++++++++++++++++++++++++++ 15 files changed, 325 insertions(+) create mode 100644 helm/.helmignore create mode 100644 helm/Chart.yaml create mode 100644 helm/README.md create mode 100644 helm/templates/NOTES.txt create mode 100644 helm/templates/_helpers.tpl create mode 100644 helm/templates/backend-deployment.yaml create mode 100644 helm/templates/backend-service.yaml create mode 100644 helm/templates/db-service.yaml create mode 100644 helm/templates/db-statefulset.yaml create mode 100644 helm/templates/grafana-deployment.yaml create mode 100644 helm/templates/grafana-service.yaml create mode 100644 helm/templates/ingress.yaml create mode 100644 helm/templates/migrations-job.yaml create mode 100644 helm/templates/pvc.yaml create mode 100644 helm/values.yaml diff --git a/helm/.helmignore b/helm/.helmignore new file mode 100644 index 0000000..5ea7d00 --- /dev/null +++ b/helm/.helmignore @@ -0,0 +1,44 @@ +# Common VCS directories +.git/ +.gitignore +.github/ + +# Mac / Windows system files +.DS_Store +Thumbs.db + +# Editor files +*.swp +*.bak +*.tmp +*.orig +*.log + +# Python / build artifacts +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ + +# Tests and CI/CD +tests/ +*.test +*.coverage +.coverage +.env +.vscode/ +.idea/ + +# Docs and misc +docs/ +*.md +LICENSE +CHANGELOG.md +README-PACKAGE.md + +# Docker and Compose files +docker-compose.yaml +Dockerfile +*.Dockerfile diff --git a/helm/Chart.yaml b/helm/Chart.yaml new file mode 100644 index 0000000..2abb772 --- /dev/null +++ b/helm/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: opensampl +description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) +type: application +version: 0.1.0 +appVersion: "1.0.0" +icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..4f56c30 --- /dev/null +++ b/helm/README.md @@ -0,0 +1,17 @@ +charts/ +└── opensampl/ + ├── Chart.yaml + ├── values.yaml + ├── templates/ + │ ├── _helpers.tpl + │ ├── db-statefulset.yaml + │ ├── db-service.yaml + │ ├── grafana-deployment.yaml + │ ├── grafana-service.yaml + │ ├── backend-deployment.yaml + │ ├── backend-service.yaml + │ ├── migrations-job.yaml + │ ├── pvc.yaml + │ ├── ingress.yaml (optional) + │ └── NOTES.txt + └── .helmignore \ No newline at end of file diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt new file mode 100644 index 0000000..44279c2 --- /dev/null +++ b/helm/templates/NOTES.txt @@ -0,0 +1,9 @@ +Thank you for installing OpenSAMPL! + +To access your services: +- Backend: ClusterIP service "{{ include "opensampl.name" . }}-backend" on port {{ .Values.backend.port }} +- Grafana: ClusterIP service "{{ include "opensampl.name" . }}-grafana" on port {{ .Values.grafana.port }} +- Database: StatefulSet "{{ include "opensampl.name" . }}-db" with {{ .Values.db.storage }} storage + +If ingress is enabled, access OpenSAMPL at: +http://{{ .Values.ingress.host }} diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl new file mode 100644 index 0000000..5ee2189 --- /dev/null +++ b/helm/templates/_helpers.tpl @@ -0,0 +1,15 @@ +{{/* Common naming helpers */}} +{{- define "opensampl.name" -}} +opensampl +{{- end -}} + + +{{- define "opensampl.fullname" -}} +{{ include "opensampl.name" . }}-{{ .Release.Name }} +{{- end -}} + + +{{- define "opensampl.labels" -}} +app.kubernetes.io/name: {{ include "opensampl.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/backend-deployment.yaml b/helm/templates/backend-deployment.yaml new file mode 100644 index 0000000..e0dc245 --- /dev/null +++ b/helm/templates/backend-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.name" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.backend.replicas }} + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-backend + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-backend + spec: + containers: + - name: backend + image: {{ .Values.backend.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.backend.port }} + env: + {{- toYaml .Values.backend.env | nindent 12 }} diff --git a/helm/templates/backend-service.yaml b/helm/templates/backend-service.yaml new file mode 100644 index 0000000..692a613 --- /dev/null +++ b/helm/templates/backend-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.backend.port }} + targetPort: {{ .Values.backend.port }} + name: http + selector: + app: {{ include "opensampl.name" . }}-backend diff --git a/helm/templates/db-service.yaml b/helm/templates/db-service.yaml new file mode 100644 index 0000000..b7a13c7 --- /dev/null +++ b/helm/templates/db-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.db.port }} + targetPort: {{ .Values.db.port }} + name: postgres + selector: + app: {{ include "opensampl.name" . }}-db diff --git a/helm/templates/db-statefulset.yaml b/helm/templates/db-statefulset.yaml new file mode 100644 index 0000000..ed717f4 --- /dev/null +++ b/helm/templates/db-statefulset.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "opensampl.fullname" . }}-db +spec: + serviceName: {{ include "opensampl.fullname" . }}-db + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-db + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-db + spec: + containers: + - name: db + image: {{ .Values.db.image.repository }}:{{ .Values.db.image.tag }} + env: + - name: POSTGRES_DB + value: {{ .Values.db.database }} + - name: POSTGRES_USER + value: {{ .Values.db.user }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: password + ports: + - containerPort: 5432 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.db.persistence.size }} diff --git a/helm/templates/grafana-deployment.yaml b/helm/templates/grafana-deployment.yaml new file mode 100644 index 0000000..3c6813e --- /dev/null +++ b/helm/templates/grafana-deployment.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.name" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-grafana + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-grafana + spec: + containers: + - name: grafana + image: {{ .Values.grafana.image }} + ports: + - containerPort: {{ .Values.grafana.port }} + env: + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.grafana.adminUser }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.grafana.adminPassword }} diff --git a/helm/templates/grafana-service.yaml b/helm/templates/grafana-service.yaml new file mode 100644 index 0000000..67a5d9f --- /dev/null +++ b/helm/templates/grafana-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.name" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.grafana.port }} + targetPort: {{ .Values.grafana.port }} + name: http + selector: + app: {{ include "opensampl.name" . }}-grafana diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml new file mode 100644 index 0000000..25e1964 --- /dev/null +++ b/helm/templates/ingress.yaml @@ -0,0 +1,20 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "opensampl.name" . }}-ingress + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.className }} +spec: + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "opensampl.name" . }}-backend + port: + number: {{ .Values.backend.port }} +{{- end }} diff --git a/helm/templates/migrations-job.yaml b/helm/templates/migrations-job.yaml new file mode 100644 index 0000000..f5fc367 --- /dev/null +++ b/helm/templates/migrations-job.yaml @@ -0,0 +1,21 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "opensampl.fullname" . }}-migrations + labels: + {{- include "opensampl.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install,post-upgrade +spec: + template: + metadata: + labels: + app: {{ include "opensampl.name" . }} + spec: + restartPolicy: OnFailure + containers: + - name: migrations + image: {{ .Values.migrations.image }} + env: + - name: DATABASE_URL + values: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file diff --git a/helm/templates/pvc.yaml b/helm/templates/pvc.yaml new file mode 100644 index 0000000..acfe1e6 --- /dev/null +++ b/helm/templates/pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "opensampl.name" . }}-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.size }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml new file mode 100644 index 0000000..e108869 --- /dev/null +++ b/helm/values.yaml @@ -0,0 +1,44 @@ +# Global settings +global: + imagePullPolicy: IfNotPresent + +# PostgreSQL Database +db: + image: postgres:14 + storage: 10Gi + username: opensampl + password: opensampl + database: opensampl + port: 5432 + +# Backend service +backend: + image: ghcr.io/ornl/opensampl-backend:latest + replicas: 1 + port: 8000 + env: + - name: DATABASE_URL + value: postgresql://opensampl:opensampl@opensampl-db:5432/opensampl + +# Grafana +grafana: + image: grafana/grafana:10.0.0 + port: 3000 + adminUser: admin + adminPassword: admin + +# Migrations +migrations: + image: ghcr.io/ornl/opensampl-migrations:latest + +# Persistence +persistence: + enabled: true + size: 10Gi + storageClass: "" + +# Ingress +ingress: + enabled: false + className: nginx + host: opensampl.local From d026a59f70373adc4f6b2442a55a53d16ad63c6e Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 11:07:47 -0400 Subject: [PATCH 02/49] add github actions workflow to build and push db image to ornl/opensampl --- .github/workflows/docker-publish.yml | 54 +++++++++++++++++++ .github/workflows/helm-chart.yaml | 33 ++++++++++++ helm/templates/_helpers.tpl | 21 ++++++-- helm/templates/backend-deployment.yaml | 24 --------- helm/templates/backend/deployment.yaml | 44 +++++++++++++++ .../service.yaml} | 10 ++-- helm/templates/db-service.yaml | 14 ----- helm/templates/db-statefulset.yaml | 41 -------------- helm/templates/db/configmap.yaml | 9 ++++ helm/templates/db/secret.yaml | 11 ++++ helm/templates/db/service.yaml | 16 ++++++ helm/templates/db/statefulset.yaml | 53 ++++++++++++++++++ helm/templates/grafana-deployment.yaml | 26 --------- helm/templates/grafana/deployment.yaml | 52 ++++++++++++++++++ .../service.yaml} | 10 ++-- helm/templates/ingress.yaml | 25 +++++++-- .../{ => migrations}/migrations-job.yaml | 3 +- helm/values.yaml | 49 +++++++++++------ opensampl/db/Dockerfile | 7 +++ opensampl/db/create-grafana.sh | 4 ++ 20 files changed, 368 insertions(+), 138 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/helm-chart.yaml delete mode 100644 helm/templates/backend-deployment.yaml create mode 100644 helm/templates/backend/deployment.yaml rename helm/templates/{backend-service.yaml => backend/service.yaml} (68%) delete mode 100644 helm/templates/db-service.yaml delete mode 100644 helm/templates/db-statefulset.yaml create mode 100644 helm/templates/db/configmap.yaml create mode 100644 helm/templates/db/secret.yaml create mode 100644 helm/templates/db/service.yaml create mode 100644 helm/templates/db/statefulset.yaml delete mode 100644 helm/templates/grafana-deployment.yaml create mode 100644 helm/templates/grafana/deployment.yaml rename helm/templates/{grafana-service.yaml => grafana/service.yaml} (68%) rename helm/templates/{ => migrations}/migrations-job.yaml (67%) create mode 100644 opensampl/db/Dockerfile create mode 100644 opensampl/db/create-grafana.sh diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..e84ab2e --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,54 @@ +name: Build and Push DB Image + +on: + push: + branches: + - main + paths: + - 'opensampl/db/**' + - '.github/workflows/docker-publish-db.yml' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ornl/opensampl-db + +jobs: + build-and-push-db: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest + type=sha,prefix=main- + + - name: Build and push DB Docker image + uses: docker/build-push-action@v5 + with: + context: ./db + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/helm-chart.yaml b/.github/workflows/helm-chart.yaml new file mode 100644 index 0000000..d25bce9 --- /dev/null +++ b/.github/workflows/helm-chart.yaml @@ -0,0 +1,33 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + helm: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 5ee2189..6c3d8de 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -1,15 +1,28 @@ -{{/* Common naming helpers */}} +{{/* Base name */}} {{- define "opensampl.name" -}} opensampl {{- end -}} - +{{/* Full release-qualified name */}} {{- define "opensampl.fullname" -}} -{{ include "opensampl.name" . }}-{{ .Release.Name }} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" (include "opensampl.name" .) .Release.Name | trunc 63 | trimSuffix "-" }} +{{- end }} {{- end -}} +{{/* Component-specific name */}} +{{- define "opensampl.componentname" -}} +{{- printf "%s-%s" (include "opensampl.name" .) .component | trunc 63 | trimSuffix "-" }} +{{- end -}} +{{/* Common labels */}} {{- define "opensampl.labels" -}} app.kubernetes.io/name: {{ include "opensampl.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} \ No newline at end of file +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .component }} +app.kubernetes.io/component: {{ .component }} +{{- end }} +{{- end -}} diff --git a/helm/templates/backend-deployment.yaml b/helm/templates/backend-deployment.yaml deleted file mode 100644 index e0dc245..0000000 --- a/helm/templates/backend-deployment.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "opensampl.name" . }}-backend - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.backend.replicas }} - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-backend - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-backend - spec: - containers: - - name: backend - image: {{ .Values.backend.image }} - imagePullPolicy: {{ .Values.global.imagePullPolicy }} - ports: - - containerPort: {{ .Values.backend.port }} - env: - {{- toYaml .Values.backend.env | nindent 12 }} diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml new file mode 100644 index 0000000..7974e3d --- /dev/null +++ b/helm/templates/backend/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.fullname" . }}-backend + labels: + {{- include "opensampl.labels" . | nindent 4 }} + role: backend +spec: + replicas: {{ .Values.backend.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend + spec: + initContainers: + - name: wait-for-db + image: busybox + command: + - sh + - -c + - > + until nc -z {{ include "opensampl.fullname" . }}-db {{ .Values.db.port }}; + do echo "Waiting for DB..."; + sleep 3; + done; + containers: + - name: backend + image: {{ .Values.backend.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.backend.port }} + env: + - name: DATABASE_URL + value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" + {{- if .Values.backend.env }} + {{- toYaml .Values.backend.env | nindent 12 }} + {{- end }} diff --git a/helm/templates/backend-service.yaml b/helm/templates/backend/service.yaml similarity index 68% rename from helm/templates/backend-service.yaml rename to helm/templates/backend/service.yaml index 692a613..2f091c5 100644 --- a/helm/templates/backend-service.yaml +++ b/helm/templates/backend/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "opensampl.name" . }}-backend + name: {{ include "opensampl.fullname" . }}-backend labels: {{- include "opensampl.labels" . | nindent 4 }} spec: type: ClusterIP - ports: - - port: {{ .Values.backend.port }} - targetPort: {{ .Values.backend.port }} - name: http selector: app: {{ include "opensampl.name" . }}-backend + ports: + - name: http + port: {{ .Values.backend.port }} + targetPort: {{ .Values.backend.port }} diff --git a/helm/templates/db-service.yaml b/helm/templates/db-service.yaml deleted file mode 100644 index b7a13c7..0000000 --- a/helm/templates/db-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "opensampl.name" . }}-db - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - type: ClusterIP - ports: - - port: {{ .Values.db.port }} - targetPort: {{ .Values.db.port }} - name: postgres - selector: - app: {{ include "opensampl.name" . }}-db diff --git a/helm/templates/db-statefulset.yaml b/helm/templates/db-statefulset.yaml deleted file mode 100644 index ed717f4..0000000 --- a/helm/templates/db-statefulset.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "opensampl.fullname" . }}-db -spec: - serviceName: {{ include "opensampl.fullname" . }}-db - replicas: 1 - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-db - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-db - spec: - containers: - - name: db - image: {{ .Values.db.image.repository }}:{{ .Values.db.image.tag }} - env: - - name: POSTGRES_DB - value: {{ .Values.db.database }} - - name: POSTGRES_USER - value: {{ .Values.db.user }} - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "opensampl.fullname" . }}-db-secret - key: password - ports: - - containerPort: 5432 - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ .Values.db.persistence.size }} diff --git a/helm/templates/db/configmap.yaml b/helm/templates/db/configmap.yaml new file mode 100644 index 0000000..17deed1 --- /dev/null +++ b/helm/templates/db/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensampl.fullname" . }}-db-config + labels: + {{- include "opensampl.labels" . | nindent 4 }} +data: + POSTGRES_DB: {{ .Values.db.database | quote }} + POSTGRES_PORT: "{{ .Values.db.port }}" diff --git a/helm/templates/db/secret.yaml b/helm/templates/db/secret.yaml new file mode 100644 index 0000000..79a49c5 --- /dev/null +++ b/helm/templates/db/secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "opensampl.fullname" . }}-db-secret + labels: + {{- include "opensampl.labels" . | nindent 4 }} +type: Opaque +stringData: + POSTGRES_USER: {{ .Values.db.username | quote }} + POSTGRES_PASSWORD: {{ .Values.db.password | quote }} + GF_SECURITY_ADMIN_PASSWORD: {{ .Values.db.grafanaPassword | quote }} diff --git a/helm/templates/db/service.yaml b/helm/templates/db/service.yaml new file mode 100644 index 0000000..6d3d185 --- /dev/null +++ b/helm/templates/db/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "opensampl.fullname" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - name: postgres + port: {{ .Values.db.port }} + targetPort: {{ .Values.db.port }} + selector: + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml new file mode 100644 index 0000000..27ab052 --- /dev/null +++ b/helm/templates/db/statefulset.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "opensampl.name" . }}-db + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + serviceName: {{ include "opensampl.name" . }}-db + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-db + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-db + spec: + securityContext: + runAsUser: 999 + fsGroup: 999 + containers: + - name: db + image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.db.port }} + name: postgres + envFrom: + - secretRef: + name: {{ include "opensampl.fullname" . }}-db-secret + - configMapRef: + name: {{ include "opensampl.fullname" . }}-db-config + volumeMounts: + - name: db-storage + mountPath: /var/lib/postgresql/data + - name: init-script + mountPath: /docker-entrypoint-initdb.d/020_create_grafana_user.sh + subPath: 020_create_grafana_user.sh + volumes: + - name: init-script + configMap: + name: {{ include "opensampl.fullname" . }}-db-init + volumeClaimTemplates: + - metadata: + name: db-storage + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.db.persistence.size }} + {{- if .Values.db.persistence.storageClass }} + storageClassName: {{ .Values.db.persistence.storageClass }} + {{- end }} diff --git a/helm/templates/grafana-deployment.yaml b/helm/templates/grafana-deployment.yaml deleted file mode 100644 index 3c6813e..0000000 --- a/helm/templates/grafana-deployment.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "opensampl.name" . }}-grafana - labels: - {{- include "opensampl.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "opensampl.name" . }}-grafana - template: - metadata: - labels: - app: {{ include "opensampl.name" . }}-grafana - spec: - containers: - - name: grafana - image: {{ .Values.grafana.image }} - ports: - - containerPort: {{ .Values.grafana.port }} - env: - - name: GF_SECURITY_ADMIN_USER - value: {{ .Values.grafana.adminUser }} - - name: GF_SECURITY_ADMIN_PASSWORD - value: {{ .Values.grafana.adminPassword }} diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml new file mode 100644 index 0000000..5c3eca4 --- /dev/null +++ b/helm/templates/grafana/deployment.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "opensampl.fullname" . }}-grafana + labels: + {{- include "opensampl.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "opensampl.name" . }}-grafana + template: + metadata: + labels: + app: {{ include "opensampl.name" . }}-grafana + spec: + containers: + - name: grafana + image: {{ .Values.grafana.image }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.grafana.port }} + env: + - name: GF_SECURITY_ADMIN_USER + value: {{ .Values.grafana.adminUser }} + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ .Values.grafana.adminPassword }} + - name: GF_DATABASE_TYPE + value: postgres + - name: GF_DATABASE_HOST + value: {{ include "opensampl.name" . }}-db:{{ .Values.db.port }} + - name: GF_DATABASE_NAME + value: {{ .Values.db.database }} + - name: GF_DATABASE_USER + value: {{ .Values.db.username }} + - name: GF_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: POSTGRES_PASSWORD + readinessProbe: + httpGet: + path: /login + port: {{ .Values.grafana.port }} + initialDelaySeconds: 10 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /api/health + port: {{ .Values.grafana.port }} + initialDelaySeconds: 30 + periodSeconds: 10 diff --git a/helm/templates/grafana-service.yaml b/helm/templates/grafana/service.yaml similarity index 68% rename from helm/templates/grafana-service.yaml rename to helm/templates/grafana/service.yaml index 67a5d9f..6070f7f 100644 --- a/helm/templates/grafana-service.yaml +++ b/helm/templates/grafana/service.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "opensampl.name" . }}-grafana + name: {{ include "opensampl.fullname" . }}-grafana labels: {{- include "opensampl.labels" . | nindent 4 }} spec: type: ClusterIP - ports: - - port: {{ .Values.grafana.port }} - targetPort: {{ .Values.grafana.port }} - name: http selector: app: {{ include "opensampl.name" . }}-grafana + ports: + - name: http + port: {{ .Values.grafana.port }} + targetPort: {{ .Values.grafana.port }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 25e1964..40bdfa4 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,19 +2,38 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.name" . }}-ingress + name: {{ include "opensampl.fullname" . }}-ingress + labels: + {{- include "opensampl.labels" . | nindent 4 }} annotations: kubernetes.io/ingress.class: {{ .Values.ingress.className }} + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: {{ include "opensampl.fullname" . }}-tls rules: - host: {{ .Values.ingress.host }} http: paths: - - path: / + # Backend API + - path: /api pathType: Prefix backend: service: - name: {{ include "opensampl.name" . }}-backend + name: {{ include "opensampl.fullname" . }}-backend port: number: {{ .Values.backend.port }} + + # Grafana UI + - path: /grafana + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} {{- end }} diff --git a/helm/templates/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml similarity index 67% rename from helm/templates/migrations-job.yaml rename to helm/templates/migrations/migrations-job.yaml index f5fc367..0fb36e8 100644 --- a/helm/templates/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -6,6 +6,7 @@ metadata: {{- include "opensampl.labels" . | nindent 4 }} annotations: "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: metadata: @@ -18,4 +19,4 @@ spec: image: {{ .Values.migrations.image }} env: - name: DATABASE_URL - values: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file + value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index e108869..276f8fb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -4,40 +4,59 @@ global: # PostgreSQL Database db: - image: postgres:14 - storage: 10Gi + image: + #repository: ghcr.io/mzinternallab/opensampl-dbc + repository: ghcr.io/ornl/opensampl-db + tag: latest username: opensampl password: opensampl + grafanaPassword: grafana123 database: opensampl port: 5432 + storage: 10Gi + persistence: + enabled: true + size: 10Gi + storageClass: "" # Backend service backend: - image: ghcr.io/ornl/opensampl-backend:latest + image: + repository: ghcr.io/ornl/opensampl-backend + tag: latest replicas: 1 port: 8000 env: - - name: DATABASE_URL - value: postgresql://opensampl:opensampl@opensampl-db:5432/opensampl + # Additional backend environment variables (merged automatically) + LOG_LEVEL: INFO + USE_API_KEY: "false" -# Grafana +# Grafana service grafana: - image: grafana/grafana:10.0.0 + image: + repository: grafana/grafana + tag: "10.0.0" + replicas: 1 port: 3000 adminUser: admin adminPassword: admin + database: + enabled: true + type: postgres + host: opensampl-db + port: 5432 + name: opensampl + user: opensampl -# Migrations +# Migrations (optional) migrations: - image: ghcr.io/ornl/opensampl-migrations:latest - -# Persistence -persistence: enabled: true - size: 10Gi - storageClass: "" + image: + repository: ghcr.io/ornl/opensampl-migrations + tag: latest + env: {} -# Ingress +# Ingress (optional) ingress: enabled: false className: nginx diff --git a/opensampl/db/Dockerfile b/opensampl/db/Dockerfile new file mode 100644 index 0000000..a16469b --- /dev/null +++ b/opensampl/db/Dockerfile @@ -0,0 +1,7 @@ +FROM timescale/timescaledb-ha:pg16.2-ts2.14.2-all + +ENV DEBIAN_FRONTEND=noninteractive + +COPY create-grafana.sh /docker-entrypoint-initdb.d/020_create_grafana_user.sh + +CMD ["postgres"] \ No newline at end of file diff --git a/opensampl/db/create-grafana.sh b/opensampl/db/create-grafana.sh new file mode 100644 index 0000000..daa9694 --- /dev/null +++ b/opensampl/db/create-grafana.sh @@ -0,0 +1,4 @@ +#!/bin/bash +psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" < Date: Thu, 30 Oct 2025 12:09:54 -0400 Subject: [PATCH 03/49] build and push db image to ghcr/ornl/opensampl --- .github/workflows/docker-publish.yml | 54 ---------------------- .github/workflows/helm-chart.yaml | 33 -------------- .github/workflows/helm-chart.yml | 62 ++++++++++++++++++++++++++ helm/templates/backend/deployment.yaml | 4 +- helm/templates/db/pvc.yaml | 15 +++++++ helm/templates/pvc.yaml | 15 ------- helm/values.yaml | 8 ++-- 7 files changed, 84 insertions(+), 107 deletions(-) delete mode 100644 .github/workflows/docker-publish.yml delete mode 100644 .github/workflows/helm-chart.yaml create mode 100644 .github/workflows/helm-chart.yml create mode 100644 helm/templates/db/pvc.yaml delete mode 100644 helm/templates/pvc.yaml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index e84ab2e..0000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Build and Push DB Image - -on: - push: - branches: - - main - paths: - - 'opensampl/db/**' - - '.github/workflows/docker-publish-db.yml' - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ornl/opensampl-db - -jobs: - build-and-push-db: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest - type=sha,prefix=main- - - - name: Build and push DB Docker image - uses: docker/build-push-action@v5 - with: - context: ./db - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max \ No newline at end of file diff --git a/.github/workflows/helm-chart.yaml b/.github/workflows/helm-chart.yaml deleted file mode 100644 index d25bce9..0000000 --- a/.github/workflows/helm-chart.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Helm Chart CI/CD - -on: - push: - branches: [ helm-deploy ] - pull_request: - branches: [ helm-deploy ] - workflow_dispatch: - -jobs: - helm: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v3 - with: - version: v3.14.0 - - - name: Helm Lint - run: helm lint ./helm - - - name: Package Chart - run: helm package ./helm - - - name: Push to GHCR - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml new file mode 100644 index 0000000..61ec37c --- /dev/null +++ b/.github/workflows/helm-chart.yml @@ -0,0 +1,62 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + build-docker-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push DB image + uses: docker/build-push-action@v5 + with: + context: ./db + push: true + tags: ghcr.io/ornl/opensampl-db:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + helm: + runs-on: ubuntu-latest + needs: build-docker-images + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index 7974e3d..ef2dd75 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -39,6 +39,6 @@ spec: env: - name: DATABASE_URL value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" - {{- if .Values.backend.env }} - {{- toYaml .Values.backend.env | nindent 12 }} + {{- with .Values.backend.env }} + {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/templates/db/pvc.yaml b/helm/templates/db/pvc.yaml new file mode 100644 index 0000000..b049791 --- /dev/null +++ b/helm/templates/db/pvc.yaml @@ -0,0 +1,15 @@ +{{- if .Values.db.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "opensampl.name" . }}-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.db.persistence.size }} + {{- if .Values.db.persistence.storageClass }} + storageClassName: {{ .Values.db.persistence.storageClass }} + {{- end }} +{{- end }} diff --git a/helm/templates/pvc.yaml b/helm/templates/pvc.yaml deleted file mode 100644 index acfe1e6..0000000 --- a/helm/templates/pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if .Values.persistence.enabled }} -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "opensampl.name" . }}-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.persistence.size }} - {{- if .Values.persistence.storageClass }} - storageClassName: {{ .Values.persistence.storageClass }} - {{- end }} -{{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 276f8fb..8fcede9 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -27,9 +27,11 @@ backend: replicas: 1 port: 8000 env: - # Additional backend environment variables (merged automatically) - LOG_LEVEL: INFO - USE_API_KEY: "false" + # Additional backend environment variables + - name: LOG_LEVEL + value: INFO + - name: USE_API_KEY + value: "false" # Grafana service grafana: From 5fa3c989d31c5d24d50c6e94c05367e9d072daa8 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 12:15:55 -0400 Subject: [PATCH 04/49] use workflow to build and push db image to ghcr.io.ornl/opensampl --- .github/workflows/helm-chart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 61ec37c..9b15183 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -30,7 +30,7 @@ jobs: - name: Build and push DB image uses: docker/build-push-action@v5 with: - context: ./db + context: ./opensampl/db push: true tags: ghcr.io/ornl/opensampl-db:latest cache-from: type=gha From d3b01d7ffc2063eddaa5d63dd5f01d9597ab224b Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:33:38 -0400 Subject: [PATCH 05/49] build and push backend and grafana to ghcr for testing helm deployment - minus migratins --- .github/workflows/helm-chart.yml | 13 ++ helm/values.yaml | 1 - opensampl/backend/Dockerfile | 9 + opensampl/backend/README.d | 2 + opensampl/backend/main.py | 353 +++++++++++++++++++++++++++++ opensampl/backend/requirements.txt | 8 + 6 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 opensampl/backend/Dockerfile create mode 100644 opensampl/backend/README.d create mode 100644 opensampl/backend/main.py create mode 100644 opensampl/backend/requirements.txt diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 9b15183..5b6db77 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -13,6 +13,19 @@ jobs: permissions: contents: read packages: write + + strategy: + matrix: + service: + - name: db + context: ./opensampl/db + - name: backend + context: ./opensampl/backend + - name: grafana + context: ./opensampl/server/grafana + #- name: migrations + # context: ./opensampl/migrations + steps: - name: Checkout uses: actions/checkout@v4 diff --git a/helm/values.yaml b/helm/values.yaml index 8fcede9..77f7ed0 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -5,7 +5,6 @@ global: # PostgreSQL Database db: image: - #repository: ghcr.io/mzinternallab/opensampl-dbc repository: ghcr.io/ornl/opensampl-db tag: latest username: opensampl diff --git a/opensampl/backend/Dockerfile b/opensampl/backend/Dockerfile new file mode 100644 index 0000000..d1ab545 --- /dev/null +++ b/opensampl/backend/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11 +WORKDIR /tmp +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && \ + rm requirements.txt +ENV ROUTE_TO_BACKEND=false +COPY main.py . +CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"] +# CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/opensampl/backend/README.d b/opensampl/backend/README.d new file mode 100644 index 0000000..f46eed8 --- /dev/null +++ b/opensampl/backend/README.d @@ -0,0 +1,2 @@ +CAST Backend +http://localhost:8000/docs - swagger endpoint \ No newline at end of file diff --git a/opensampl/backend/main.py b/opensampl/backend/main.py new file mode 100644 index 0000000..26fbb7c --- /dev/null +++ b/opensampl/backend/main.py @@ -0,0 +1,353 @@ +import io +import json +import os +import sys +import time +from typing import Any, Dict, Callable, Optional +from datetime import datetime, timedelta +import pandas as pd +from fastapi import FastAPI, HTTPException, File, UploadFile, Form, Request, Response, Security, Depends +from fastapi.security.api_key import APIKeyHeader +from fastapi.responses import JSONResponse, RedirectResponse +from loguru import logger +from pydantic import BaseModel +from sqlalchemy import create_engine, text, select, or_, and_ +from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from sqlalchemy.orm import sessionmaker, Session +from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST +from opensampl.db.access_orm import APIAccessKey +from opensampl import load_data +from opensampl.db.orm import ProbeMetadata +from opensampl.vendors.constants import VendorType, ProbeKey +from opensampl.metrics import METRICS, MetricType +from opensampl.references import REF_TYPES, CompoundReferenceType, ReferenceType +import psycopg2 + +class TimeDataPoint(BaseModel): + time: str + value: float + + +class WriteTablePayload(BaseModel): + table: str + data: Dict[str, Any] + if_exists: load_data.conflict_actions = 'update' + + +class ProbeMetadataPayload(BaseModel): + vendor: VendorType + probe_key: ProbeKey + data: Dict[str, Any] + + +DATABASE_URI = os.getenv("DATABASE_URL") +engine = create_engine(DATABASE_URI) + +loglevel = os.getenv("BACKEND_LOG_LEVEL", "INFO") +app = FastAPI() + + +REQUEST_COUNT = Counter( + "http_requests_total", + "Total number of HTTP requests", + ["method", "endpoint", "http_status"] +) + +REQUEST_LATENCY = Histogram( + "http_request_duration_seconds", + "Duration of HTTP requests in seconds", + ["method", "endpoint"] +) + +EXCLUDED_PATHS = {"/metrics", "/healthcheck", "/healthcheck_database", "/healthcheck_metadata"} + +logger.configure(handlers=[{"sink": sys.stderr, "level": loglevel}]) + +USE_API_KEY = os.getenv("USE_API_KEY", "false").lower() == "true" +API_KEY_NAME = "access-key" + +api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) + +def get_keys(): + env_keys = os.getenv('API_KEYS', '').strip() + keys = [k.strip() for k in env_keys.split(',') if k.strip()] + if keys: + logger.debug("api access keys loaded from env") + return keys + try: + Session = sessionmaker(bind=engine) + with Session() as session: + now = datetime.utcnow() + stmt = select(APIAccessKey.key).where( + or_( + APIAccessKey.expires_at == None, + APIAccessKey.expires_at > now + ) + ) + result = session.execute(stmt) + keys = [row[0] for row in result.all()] + logger.debug("api access keys loaded from db") + return keys + except Exception as e: + logger.debug(f"exception attempting to load api access keys from db: {e}") + return [] + +def validate_api_key(api_key: str = Security(api_key_header)): + if not USE_API_KEY: + return # Security is disabled + if api_key not in get_keys(): + raise HTTPException(status_code=403, detail="Invalid or missing API key") + return api_key + +def get_db(): + Session = sessionmaker(bind=engine) + try: + session = Session() + yield session + finally: + session.close() + +@app.middleware("http") +async def metrics_middleware(request: Request, call_next: Callable) -> Response: + """Middleware to track request metrics.""" + if request.url.path in EXCLUDED_PATHS: + return await call_next(request) + start_time = time.time() + response: Response = await call_next(request) + duration = time.time() - start_time + + REQUEST_COUNT.labels( + method=request.method, + endpoint=request.url.path, + http_status=response.status_code + ).inc() + + REQUEST_LATENCY.labels( + method=request.method, + endpoint=request.url.path + ).observe(duration) + + return response + + +# add route to docs from / to /docs +@app.get("/", include_in_schema=False) +async def docs_redirect(): + return RedirectResponse(url='/docs') + + +@app.get("/setloglevel") +def set_log_level(newloglevel: str, api_key: str = Depends(validate_api_key)): + """ + change visible log level in backend container + """ + newloglevel = newloglevel.upper() + logger.configure(handlers=[{"sink": sys.stderr, "level": newloglevel}]) + return {"loglevel": newloglevel} + + +@app.get("/checkloglevel") +def check_log_level(api_key: str = Depends(validate_api_key)): + """ + This is to check which log levels are visible in backend container + """ + logger.debug('Debug test') + logger.info('Info test') + logger.warning('Warning test') + logger.error('Error test') + current_level = list(logger._core.handlers.values())[0]["level"].name + return {"loglevel": current_level} + + +@app.post("/write_to_table") +def write_to_table(payload: WriteTablePayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + try: + load_data.write_to_table(table=payload.table, data=payload.data, if_exists=payload.if_exists, session=session) + logger.debug(f'Successfully wrote to {payload.table} using: {payload.data}') + return JSONResponse(content={"message": f"Succeeded loading data into {payload.table}"}, status_code=200) + except IntegrityError as e: + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.post("/load_time_data") +async def load_time_data(probe_key_str: str = Form(...), + metric_type_str: Optional[str] = Form(None), + reference_type_str: Optional[str] = Form(None), + compound_key_str: Optional[str] = Form(None), + file: UploadFile = File(...), + api_key: str = Depends(validate_api_key), + session: Session = Depends(get_db)): + try: + + probe_key = ProbeKey(**json.loads(probe_key_str)) + + if metric_type_str is not None: + metric_type_dict = json.loads(metric_type_str) + metric_type = MetricType(**metric_type_dict) + else: + metric_type = METRICS.UNKNOWN + + if reference_type_str is not None: + reference_type_dict = json.loads(reference_type_str) + if 'reference_table' in reference_type_dict: + reference_type = CompoundReferenceType(**reference_type_dict) + else: + reference_type = ReferenceType(**reference_type_dict) + else: + reference_type = REF_TYPES.UNKNOWN + + compound_key = None if compound_key_str is None else json.loads(compound_key_str) + + content = await file.read() + df = pd.read_csv(io.BytesIO(content)) + logger.info(df.head()) + # Convert time strings back to datetime + df['time'] = pd.to_datetime(df['time']) + + # Convert value strings back to float64 + # df['value'] = df['value'].astype('float64') + + # Use the same load_time_data function as before + load_data.load_time_data( + probe_key=probe_key, + metric_type=metric_type, + reference_type=reference_type, + compound_key=compound_key, + data=df, + session=session + ) + + return JSONResponse( + content={"message": f"Successfully loaded {len(df)} data points"}, + status_code=200 + ) + except IntegrityError as e: + if session: + session.rollback() + session.close() + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'Database error: {e}') + if session: + session.rollback() + session.close() + raise HTTPException(status_code=500, detail=f"Database error: {str(e)}") + except Exception as e: + logger.error(f'Unexpected error: {e}') + if session: + session.rollback() + session.close() + raise HTTPException(status_code=500, detail=f"Error processing time series data: {str(e)}") + + +@app.post("/load_probe_metadata") +def load_probe_metadata(payload: ProbeMetadataPayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + logger.debug(f"Received payload: {payload.model_dump()}") + + try: + load_data.load_probe_metadata(vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, + session=session) + logger.debug( + f'Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}') + return JSONResponse(content={"message": f"Succeeded loaded metadata for {payload.probe_key}"}, status_code=200) + except IntegrityError as e: + session.rollback() + if isinstance(e.orig, psycopg2.errors.UniqueViolation): + return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) + return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.exception(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.get("/create_new_tables") +def create_new_tables(create_schema: bool = True, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): + try: + load_data.create_new_tables(create_schema=create_schema, session=session) + return JSONResponse(content={"message": f"Succeeded in creating any new tables"}, status_code=200) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except json.JSONDecodeError as e: + logger.error(f'JSON decode error: {e}') + return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) + + +@app.get("/gen_api_key") +def generate_api_key(expire_after: Optional[int] = None, session: Session = Depends(get_db)): + try: + + new_key = APIAccessKey() + new_key.generate_key() + if expire_after: + new_key.expires_at = datetime.utcnow() + timedelta(days=expire_after) + + session.add(new_key) + + session.commit() + return JSONResponse(content={"message": f"Succeeded in creating new access key"}, status_code=200) + except SQLAlchemyError as e: + logger.error(f'SQLAlchemy error: {e}') + return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) + except Exception as e: + logger.error(f'Unexpected error: {e}') + return JSONResponse(content={"message": f"Failed to create new access key: {e}"}, status_code=500) + + +@app.get("/healthcheck") +def healthcheck(): + return {"status": "OK"} + +@app.get("/healthcheck_database") +def healthcheck_db(): + try: + with engine.connect() as connection: + connection.execute(text("SELECT 1")) + return {"status": "OK"} + except SQLAlchemyError: + raise HTTPException(status_code=503, detail="Database connection error") + + +@app.get("/healthcheck_metadata") +def healthcheck_metadata(): + try: + with engine.connect() as connection: + result = connection.execute( + text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'castdb';")) + schema_exists = result.fetchone() is not None + if schema_exists: + return {"status": "OK"} + else: + raise HTTPException(status_code=500, detail="Schema 'castdb' does not exist") + except SQLAlchemyError as e: + raise HTTPException(status_code=503, detail=f"Database connection error: {str(e)}") + + +@app.get("/metrics", include_in_schema=False) +def metrics(): + """ + Expose Prometheus metrics. + """ + return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) diff --git a/opensampl/backend/requirements.txt b/opensampl/backend/requirements.txt new file mode 100644 index 0000000..ca91da0 --- /dev/null +++ b/opensampl/backend/requirements.txt @@ -0,0 +1,8 @@ +psycopg2-binary +fastapi +uvicorn +sqlalchemy +loguru +python-multipart +prometheus-client +opensampl==1.1.5 From 39889ed1eb12d816ecfc333041502fda0937b760 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:50:48 -0400 Subject: [PATCH 06/49] build backend and grafana to test helm deployment --- .github/workflows/helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 5b6db77..42e325e 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -72,4 +72,6 @@ jobs: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts + + \ No newline at end of file From 86db46acae1c3d88f35206f49ca4ee95b20bbb11 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 13:55:39 -0400 Subject: [PATCH 07/49] corrected the build and push phase to use the matrix build for all images --- .github/workflows/helm-chart.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 42e325e..6e234c0 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -40,14 +40,14 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push DB image + - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: - context: ./opensampl/db + context: ${{ matrix.service.context }} push: true - tags: ghcr.io/ornl/opensampl-db:latest - cache-from: type=gha - cache-to: type=gha,mode=max + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + cache-from: type=gha,scope=${{ matrix.service.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} helm: runs-on: ubuntu-latest @@ -74,4 +74,3 @@ jobs: helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - \ No newline at end of file From e8f24272ba3ba470df8cdd669a9538435115a9a9 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 14:44:19 -0400 Subject: [PATCH 08/49] testing helm deploy and minikube test --- .github/workflows/helm-chart.old | 76 ++++++++++++++++++++++++++++++++ .github/workflows/helm-chart.yml | 69 ++++++++++++++++++++++++++--- .github/workflows/helm-readme.md | 27 ++++++++++++ helm/values.yaml | 4 +- 4 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/helm-chart.old create mode 100644 .github/workflows/helm-readme.md diff --git a/.github/workflows/helm-chart.old b/.github/workflows/helm-chart.old new file mode 100644 index 0000000..6e234c0 --- /dev/null +++ b/.github/workflows/helm-chart.old @@ -0,0 +1,76 @@ +name: Helm Chart CI/CD + +on: + push: + branches: [ helm-deploy ] + pull_request: + branches: [ helm-deploy ] + workflow_dispatch: + +jobs: + build-docker-images: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + strategy: + matrix: + service: + - name: db + context: ./opensampl/db + - name: backend + context: ./opensampl/backend + - name: grafana + context: ./opensampl/server/grafana + #- name: migrations + # context: ./opensampl/migrations + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ matrix.service.name }} image + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.service.context }} + push: true + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + cache-from: type=gha,scope=${{ matrix.service.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} + + helm: + runs-on: ubuntu-latest + needs: build-docker-images + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.14.0 + + - name: Helm Lint + run: helm lint ./helm + + - name: Package Chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts + diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 6e234c0..889a967 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -40,24 +40,84 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/ornl/opensampl-${{ matrix.service.name }} + tags: | + type=ref,event=pr + type=ref,event=branch + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} push: true - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - helm: + test-helm-chart: + name: Test Helm Chart Deployment runs-on: ubuntu-latest needs: build-docker-images + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start minikube + uses: medyagh/setup-minikube@latest + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Install Helm Chart + working-directory: helm/ + run: | + PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') + + helm install test-opensampl . \ + --set db.image.tag=pr-${PR_NUMBER} \ + --set backend.image.tag=pr-${PR_NUMBER} \ + --set grafana.image.tag=pr-${PR_NUMBER} \ + --wait \ + --timeout 5m + + - name: Verify Deployment + run: | + kubectl get pods + kubectl get services + kubectl get pvc + + - name: Show Pod Logs on Failure + if: failure() + run: | + echo "=== Pod Status ===" + kubectl get pods + echo "=== Pod Descriptions ===" + kubectl describe pods + echo "=== Pod Logs ===" + for pod in $(kubectl get pods -o name); do + echo "Logs for $pod:" + kubectl logs $pod --all-containers=true || true + done + + helm-package: + name: Package and Push Helm Chart + runs-on: ubuntu-latest + needs: build-docker-images + if: github.event_name != 'pull_request' steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v4 with: version: v3.14.0 @@ -72,5 +132,4 @@ jobs: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file diff --git a/.github/workflows/helm-readme.md b/.github/workflows/helm-readme.md new file mode 100644 index 0000000..4580128 --- /dev/null +++ b/.github/workflows/helm-readme.md @@ -0,0 +1,27 @@ +### 1. **Smart Image Tagging** +Images now get tagged with: +- `pr-123` for pull requests +- `helm-deploy-abc123` for branch commits +- `latest` for main branch + +### 2. **New Test Job** (`test-helm-chart`) +- **Only runs on PRs** (not regular pushes) +- Spins up local Kubernetes (minikube) +- Installs your Helm chart with PR-tagged images +- Verifies pods start successfully +- Shows detailed logs if anything fails + +### 3. **Separated Helm Packaging** +- `helm-package` job only runs on **direct pushes** (not PRs) +- Lints, packages, and pushes the chart + +## Workflow Flow + +**On Pull Request:** +``` +Build Images (with pr-123 tags) → Test in Kubernetes → ✓ Pass/Fail +``` + +**On Push to helm-deploy:** +``` +Build Images (with latest tag) → Package & Push Helm Chart \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 77f7ed0..dbae2f6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -35,8 +35,8 @@ backend: # Grafana service grafana: image: - repository: grafana/grafana - tag: "10.0.0" + repository: ghcr.io/ornl/opensampl-grafana + tag: latest replicas: 1 port: 3000 adminUser: admin From f855e019264bfec19ec2d13753e9c54064aeca15 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 14:59:22 -0400 Subject: [PATCH 09/49] test helm deploy and minikube testing --- .github/workflows/helm-chart.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 889a967..38e52c7 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -4,7 +4,9 @@ on: push: branches: [ helm-deploy ] pull_request: - branches: [ helm-deploy ] + branches: + - helm-deploy + - main workflow_dispatch: jobs: From c2ad499e2f6d0bcd5e1f33c86008e3481b8d0fe2 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:04:37 -0400 Subject: [PATCH 10/49] helm deployment and minikube test --- .github/workflows/helm-chart.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 38e52c7..2da49c2 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -25,8 +25,6 @@ jobs: context: ./opensampl/backend - name: grafana context: ./opensampl/server/grafana - #- name: migrations - # context: ./opensampl/migrations steps: - name: Checkout @@ -42,27 +40,25 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata (tags, labels) - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/ornl/opensampl-${{ matrix.service.name }} - tags: | - type=ref,event=pr - type=ref,event=branch - type=sha,prefix={{branch}}- - type=raw,value=latest,enable={{is_default_branch}} + - name: Set image tags + id: tags + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + PR_NUMBER=$(echo ${{ github.ref }} | awk -F'/' '{print $3}') + echo "tag=pr-${PR_NUMBER}" >> $GITHUB_OUTPUT + else + echo "tag=latest" >> $GITHUB_OUTPUT + fi - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.tags.outputs.tag }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - + test-helm-chart: name: Test Helm Chart Deployment runs-on: ubuntu-latest From ba6a158a6cf49b30f99b7c329e03a79b7d23c3ca Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:39:48 -0400 Subject: [PATCH 11/49] Add missing DB init configmap and fixed image references --- helm/templates/backend/deployment.yaml | 2 +- helm/templates/db/db-init-configmap.yaml | 16 ++++++++++++++++ helm/templates/grafana/deployment.yaml | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 helm/templates/db/db-init-configmap.yaml diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index ef2dd75..efe2d88 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -32,7 +32,7 @@ spec: done; containers: - name: backend - image: {{ .Values.backend.image }} + image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - containerPort: {{ .Values.backend.port }} diff --git a/helm/templates/db/db-init-configmap.yaml b/helm/templates/db/db-init-configmap.yaml new file mode 100644 index 0000000..9dd7f5e --- /dev/null +++ b/helm/templates/db/db-init-configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensampl.fullname" . }}-db-init + labels: + {{- include "opensampl.labels" . | nindent 4 }} +data: + 020_create_grafana_user.sh: | + #!/bin/bash + set -e + + # This script creates additional database setup for Grafana + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + -- Grant necessary privileges + GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER}; + EOSQL \ No newline at end of file diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 5c3eca4..fa3a5a6 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: grafana - image: {{ .Values.grafana.image }} + image: "{{ .Values.grafana.image.repository }}:{{ .Values.grafana.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - containerPort: {{ .Values.grafana.port }} From 8ee95055d1a0d55d3528eb28cd66b929e8d1f127 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 15:58:28 -0400 Subject: [PATCH 12/49] Fix helm template naming format from .name to .fullname --- helm/templates/backend/deployment.yaml | 2 +- helm/templates/db/statefulset.yaml | 5 ++++- helm/templates/grafana/deployment.yaml | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index efe2d88..a9f1708 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -38,7 +38,7 @@ spec: - containerPort: {{ .Values.backend.port }} env: - name: DATABASE_URL - value: "postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }}" + value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} {{- with .Values.backend.env }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 27ab052..0ed53bf 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "opensampl.labels" . | nindent 4 }} spec: - serviceName: {{ include "opensampl.name" . }}-db + serviceName: {{ include "opensampl.fullname" . }}-db replicas: 1 selector: matchLabels: @@ -25,6 +25,9 @@ spec: ports: - containerPort: {{ .Values.db.port }} name: postgres + env: + - name: PGDATA + value: /var/lib/postgres/data/pgdata envFrom: - secretRef: name: {{ include "opensampl.fullname" . }}-db-secret diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index fa3a5a6..82f0b3a 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -28,7 +28,7 @@ spec: - name: GF_DATABASE_TYPE value: postgres - name: GF_DATABASE_HOST - value: {{ include "opensampl.name" . }}-db:{{ .Values.db.port }} + value: {{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }} - name: GF_DATABASE_NAME value: {{ .Values.db.database }} - name: GF_DATABASE_USER From 1ebc47c3f72943477bb01d3588603502ccff1cd8 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 16:16:45 -0400 Subject: [PATCH 13/49] fix typos in db deployment --- helm/templates/db/statefulset.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 0ed53bf..5a33183 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -9,11 +9,15 @@ spec: replicas: 1 selector: matchLabels: - app: {{ include "opensampl.name" . }}-db + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db template: metadata: labels: - app: {{ include "opensampl.name" . }}-db + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: db spec: securityContext: runAsUser: 999 @@ -27,7 +31,7 @@ spec: name: postgres env: - name: PGDATA - value: /var/lib/postgres/data/pgdata + value: /var/lib/postgresql/data/pgdata envFrom: - secretRef: name: {{ include "opensampl.fullname" . }}-db-secret From aa7b6fcc820dd1eb6c510bc9e3bd9b845961b418 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:04:05 -0400 Subject: [PATCH 14/49] fix volume write permisson for DB --- helm/templates/db/statefulset.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 5a33183..1724be1 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -20,8 +20,8 @@ spec: role: db spec: securityContext: - runAsUser: 999 fsGroup: 999 + fsGroupChangePolicy: "OnRootMismatch" containers: - name: db image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag }}" From 54447e92201a9facb9501012032378569de2efa5 Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:32:00 -0400 Subject: [PATCH 15/49] diable migrations helm deploy --- .github/workflows/helm-chart.yml | 1 + helm/templates/migrations/migrations-job.yaml | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 2da49c2..191faaf 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -83,6 +83,7 @@ jobs: --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ + --set migrations.enabled=false \ --wait \ --timeout 5m diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 0fb36e8..3882065 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -1,22 +1,20 @@ +{{- if .Values.migrations.enabled }} apiVersion: batch/v1 kind: Job metadata: name: {{ include "opensampl.fullname" . }}-migrations - labels: - {{- include "opensampl.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: - metadata: - labels: - app: {{ include "opensampl.name" . }} spec: - restartPolicy: OnFailure containers: - - name: migrations - image: {{ .Values.migrations.image }} - env: - - name: DATABASE_URL - value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.name" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} \ No newline at end of file + - name: migrations + image: "{{ .Values.migrations.image.repository }}:{{ .Values.migrations.image.tag }}" + env: + - name: DATABASE_URL + value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + envFrom: + - secretRef: + name: {{ include "opensampl.fullname" . }}-db-secret + - configMapRef: + name: {{ include "opensampl.fullname" . }}-db-config + restartPolicy: OnFailure \ No newline at end of file From 3c0f2fae012ef3bb72078c44b63cea9a4ca8eaed Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 30 Oct 2025 17:36:13 -0400 Subject: [PATCH 16/49] deploy helm and test with minikube --- helm/templates/migrations/migrations-job.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 3882065..8edd519 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -17,4 +17,5 @@ spec: name: {{ include "opensampl.fullname" . }}-db-secret - configMapRef: name: {{ include "opensampl.fullname" . }}-db-config - restartPolicy: OnFailure \ No newline at end of file + restartPolicy: OnFailure +{{- end }} \ No newline at end of file From 49d745ba6dbfb56c8fa901b597cac6decf92e3a2 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 11:49:42 -0400 Subject: [PATCH 17/49] fixed ingress hosts --- helm/templates/ingress.yaml | 44 +++++++++++++------------- helm/values-prod.yaml | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 helm/values-prod.yaml diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 40bdfa4..daf2d68 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -5,35 +5,37 @@ metadata: name: {{ include "opensampl.fullname" . }}-ingress labels: {{- include "opensampl.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} annotations: - kubernetes.io/ingress.class: {{ .Values.ingress.className }} - {{- with .Values.ingress.annotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} tls: + {{- range .Values.ingress.tls }} - hosts: - - {{ .Values.ingress.host }} - secretName: {{ include "opensampl.fullname" . }}-tls + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} rules: - - host: {{ .Values.ingress.host }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} http: paths: - # Backend API - - path: /api - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-backend - port: - number: {{ .Values.backend.port }} - - # Grafana UI - - path: /grafana - pathType: Prefix + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} backend: service: - name: {{ include "opensampl.fullname" . }}-grafana + name: {{ include "opensampl.fullname" $ }}-{{ .service }} port: - number: {{ .Values.grafana.port }} -{{- end }} + number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml new file mode 100644 index 0000000..3b8aefb --- /dev/null +++ b/helm/values-prod.yaml @@ -0,0 +1,63 @@ +# values-production.yaml + +global: + imagePullPolicy: Always + +# Database configuration +db: + image: + repository: ghcr.io/ornl/opensampl-db + tag: latest + database: opensampl + username: opensampl + password: "admin" + port: 5432 + persistence: + size: 50Gi + storageClass: "longhorn" + +# Backend configuration +backend: + image: + repository: ghcr.io/ornl/opensampl-backend + tag: latest + replicas: 2 + service: + type: ClusterIP + port: 8000 + +# Grafana configuration +grafana: + image: + repository: ghcr.io/ornl/opensampl-grafana + tag: latest + adminUser: admin + adminPassword: "admin" + port: 3000 + service: + type: ClusterIP + +# Migrations (disabled until ready) +migrations: + enabled: false + +# Ingress configuration +ingress: + enabled: true + className: "nginx" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" # If using cert-manager + # nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - host: opensampl-test.ornl.gov + paths: + - path: / + pathType: Prefix + service: backend + - path: /grafana + pathType: Prefix + service: grafana + tls: + - secretName: opensampl-tls + hosts: + - opensampl-test.ornl.gov \ No newline at end of file From b61cbdcde6bad04a8e19615174e7294999d3feef Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 11:56:18 -0400 Subject: [PATCH 18/49] update chart version --- helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2abb772..4d154d5 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.0 +version: 0.1.1 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From fce444defe59d026d3184f17822255bd344b1fd1 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 12:57:00 -0400 Subject: [PATCH 19/49] add grafana subpath to test UI in mariner --- "helm/Accept\357\200\272" | 0 helm/Chart.yaml | 2 +- helm/GET | 0 "helm/Host\357\200\272" | 0 "helm/User-Agent\357\200\272" | 0 helm/templates/grafana/deployment.yaml | 5 +++++ helm/values-prod.yaml | 2 ++ 7 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 "helm/Accept\357\200\272" create mode 100644 helm/GET create mode 100644 "helm/Host\357\200\272" create mode 100644 "helm/User-Agent\357\200\272" diff --git "a/helm/Accept\357\200\272" "b/helm/Accept\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 4d154d5..e01d14e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.1 +version: 0.1.2 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/GET b/helm/GET new file mode 100644 index 0000000..e69de29 diff --git "a/helm/Host\357\200\272" "b/helm/Host\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git "a/helm/User-Agent\357\200\272" "b/helm/User-Agent\357\200\272" new file mode 100644 index 0000000..e69de29 diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 82f0b3a..d297ecd 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -38,6 +38,11 @@ spec: secretKeyRef: name: {{ include "opensampl.fullname" . }}-db-secret key: POSTGRES_PASSWORD + # For testing in Mariner + - name: GF_SERVER_ROOT_URL + value: {{ .Values.grafana.rootUrl | default "http://localhost:3000" }} + - name: GF_SERVER_SERVE_FROM_SUB_PATH + value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} readinessProbe: httpGet: path: /login diff --git a/helm/values-prod.yaml b/helm/values-prod.yaml index 3b8aefb..b7df5f2 100644 --- a/helm/values-prod.yaml +++ b/helm/values-prod.yaml @@ -36,6 +36,8 @@ grafana: port: 3000 service: type: ClusterIP + rootUrl: "https://opensampl-test.ornl.gov/grafana" + serveFromSubPath: true # Migrations (disabled until ready) migrations: From a015828a192d455d1fe3719ac12644c86177fa7d Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 13:19:28 -0400 Subject: [PATCH 20/49] remove subpath for grafana so it hosts at opensampl-test.ornl.gov --- helm/Chart.yaml | 2 +- helm/{values-prod.yaml => values-mariner.yaml} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename helm/{values-prod.yaml => values-mariner.yaml} (92%) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index e01d14e..a0c7f3e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.2 +version: 0.1.3 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-prod.yaml b/helm/values-mariner.yaml similarity index 92% rename from helm/values-prod.yaml rename to helm/values-mariner.yaml index b7df5f2..14825bb 100644 --- a/helm/values-prod.yaml +++ b/helm/values-mariner.yaml @@ -36,8 +36,8 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" - serveFromSubPath: true + rootUrl: "https://opensampl-test.ornl.gov/" + serveFromSubPath: false # Migrations (disabled until ready) migrations: @@ -53,10 +53,10 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: / + - path: /api pathType: Prefix service: backend - - path: /grafana + - path: / pathType: Prefix service: grafana tls: From da4dd431ef6b790f8a3811d94d116c229d6efe7a Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 13:53:48 -0400 Subject: [PATCH 21/49] try and host grafana at opensampl.ornl.gov --- helm/Chart.yaml | 2 +- helm/values-mariner.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index a0c7f3e..2aa74f8 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.3 +version: 0.1.4 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 14825bb..f489fc9 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -53,7 +53,7 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api + - path: /backend pathType: Prefix service: backend - path: / From 450c3ad55ae450192c282f8fd006a82a916c235b Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 15:56:56 -0400 Subject: [PATCH 22/49] move grafana ui to /grafana to avoid API collisions --- helm/Chart.yaml | 2 +- helm/templates/grafana/deployment.yaml | 4 +-- helm/templates/ingress.yaml | 43 +++++++++-------------- helm/values-mariner.yaml | 8 ++--- opensampl/backend/{README.d => README.md} | 0 5 files changed, 23 insertions(+), 34 deletions(-) rename opensampl/backend/{README.d => README.md} (100%) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2aa74f8..92e779e 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.4 +version: 0.1.5 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index d297ecd..b06cf61 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -40,9 +40,9 @@ spec: key: POSTGRES_PASSWORD # For testing in Mariner - name: GF_SERVER_ROOT_URL - value: {{ .Values.grafana.rootUrl | default "http://localhost:3000" }} + value: "https://opensampl-test.ornl.gov/grafana/" - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} + value: "true" readinessProbe: httpGet: path: /login diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index daf2d68..628007a 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,40 +2,29 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.fullname" . }}-ingress - labels: - {{- include "opensampl.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + name: {{ include "opensampl.fullname" . }} annotations: - {{- toYaml . | nindent 4 }} - {{- end }} + {{- toYaml .Values.ingress.annotations | nindent 4 }} spec: - {{- if .Values.ingress.className }} ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} + {{- toYaml .Values.ingress.tls | nindent 4 }} rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} + - host: {{ (index .Values.ingress.hosts 0).host }} http: paths: - {{- range .paths }} - - path: {{ .path }} - pathType: {{ .pathType }} + - path: /(.*) + pathType: Prefix backend: service: - name: {{ include "opensampl.fullname" $ }}-{{ .service }} + name: {{ include "opensampl.fullname" . }}-backend port: - number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} - {{- end }} - {{- end }} -{{- end }} \ No newline at end of file + number: 80 + - path: /grafana(/|$)(.*) + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} +{{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index f489fc9..7eee203 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -36,8 +36,8 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/" - serveFromSubPath: false + rootUrl: "https://opensampl-test.ornl.gov/grafana" + serveFromSubPath: true # Migrations (disabled until ready) migrations: @@ -53,10 +53,10 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /backend + - path: /(.*) pathType: Prefix service: backend - - path: / + - path: /grafana(/|$)(.*) pathType: Prefix service: grafana tls: diff --git a/opensampl/backend/README.d b/opensampl/backend/README.md similarity index 100% rename from opensampl/backend/README.d rename to opensampl/backend/README.md From 1d2d06f21c33af4de4a86b353e24ae1624765ab9 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 16:24:07 -0400 Subject: [PATCH 23/49] fix path to servce /api/docs --- helm/Chart.yaml | 2 +- helm/templates/backend/service.yaml | 4 ++-- helm/templates/ingress.yaml | 13 +++++++++++-- helm/values-mariner.yaml | 6 +++--- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 92e779e..3c0113f 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.5 +version: 0.1.6 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/service.yaml b/helm/templates/backend/service.yaml index 2f091c5..c38c02c 100644 --- a/helm/templates/backend/service.yaml +++ b/helm/templates/backend/service.yaml @@ -10,5 +10,5 @@ spec: app: {{ include "opensampl.name" . }}-backend ports: - name: http - port: {{ .Values.backend.port }} - targetPort: {{ .Values.backend.port }} + port: 80 + targetPort: 8000 \ No newline at end of file diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 628007a..bb17380 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -13,13 +13,14 @@ spec: - host: {{ (index .Values.ingress.hosts 0).host }} http: paths: - - path: /(.*) + - path: /api(/|$)(.*) pathType: Prefix backend: service: name: {{ include "opensampl.fullname" . }}-backend port: - number: 80 + number: {{ .Values.backend.service.port }} + - path: /grafana(/|$)(.*) pathType: Prefix backend: @@ -27,4 +28,12 @@ spec: name: {{ include "opensampl.fullname" . }}-grafana port: number: {{ .Values.grafana.port }} + + - path: / + pathType: Prefix + backend: + service: + name: {{ include "opensampl.fullname" . }}-grafana + port: + number: {{ .Values.grafana.port }} {{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 7eee203..09966dc 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -21,7 +21,7 @@ backend: image: repository: ghcr.io/ornl/opensampl-backend tag: latest - replicas: 2 + replicas: 1 service: type: ClusterIP port: 8000 @@ -36,7 +36,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" + rootUrl: "https://opensampl-test.ornl.gov/grafana/" serveFromSubPath: true # Migrations (disabled until ready) @@ -53,7 +53,7 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /(.*) + - path: /api(/|$)(.*) pathType: Prefix service: backend - path: /grafana(/|$)(.*) From c1bc729b25118831937cb577b343d75c52157d0c Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 16:36:56 -0400 Subject: [PATCH 24/49] fix backend ingress section to point to backend --- helm/templates/ingress.yaml | 4 ++-- helm/values-mariner.yaml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index bb17380..957857b 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -33,7 +33,7 @@ spec: pathType: Prefix backend: service: - name: {{ include "opensampl.fullname" . }}-grafana + name: {{ include "opensampl.fullname" . }}-backend port: - number: {{ .Values.grafana.port }} + number: {{ .Values.backend.service.port }} {{- end }} diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 09966dc..a222296 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -56,6 +56,9 @@ ingress: - path: /api(/|$)(.*) pathType: Prefix service: backend + - path: /docs(/|$)(.*) + pathType: Prefix + service: backend - path: /grafana(/|$)(.*) pathType: Prefix service: grafana From 0604f8d8bb21b83bd9eed198c6d9989dac074155 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:14:01 -0400 Subject: [PATCH 25/49] fix backend 503 error --- helm/Chart.yaml | 2 +- helm/templates/backend/deployment.yaml | 17 ++++++++++++++--- helm/templates/backend/service.yaml | 6 ++++-- helm/values-mariner.yaml | 23 +++++++++++++---------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 3c0113f..2d20e33 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.6 +version: 0.1.7 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index a9f1708..16d48c1 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -35,10 +35,21 @@ spec: image: "{{ .Values.backend.image.repository }}:{{ .Values.backend.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} ports: - - containerPort: {{ .Values.backend.port }} + - containerPort: 8000 env: - name: DATABASE_URL - value: postgresql://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + - name: POSTGRES_USER + value: {{ .Values.db.username }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "opensampl.fullname" . }}-db-secret + key: POSTGRES_PASSWORD + - name: BACKEND_LOG_LEVEL + value: {{ .Values.backend.logLevel | default "INFO" | quote }} + - name: USE_API_KEY + value: {{ .Values.backend.useApiKey | default "false" | quote }} {{- with .Values.backend.env }} {{- toYaml . | nindent 12 }} - {{- end }} + {{- end }} \ No newline at end of file diff --git a/helm/templates/backend/service.yaml b/helm/templates/backend/service.yaml index c38c02c..d2be513 100644 --- a/helm/templates/backend/service.yaml +++ b/helm/templates/backend/service.yaml @@ -7,8 +7,10 @@ metadata: spec: type: ClusterIP selector: - app: {{ include "opensampl.name" . }}-backend + app.kubernetes.io/name: {{ include "opensampl.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + role: backend ports: - name: http - port: 80 + port: {{ .Values.backend.service.port }} targetPort: 8000 \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index a222296..e014271 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -25,6 +25,9 @@ backend: service: type: ClusterIP port: 8000 + logLevel: "INFO" + useApiKey: false + env: [] # Grafana configuration grafana: @@ -36,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana/" + rootUrl: "https://opensampl-test.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) @@ -53,15 +56,15 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api(/|$)(.*) - pathType: Prefix - service: backend - - path: /docs(/|$)(.*) - pathType: Prefix - service: backend - - path: /grafana(/|$)(.*) - pathType: Prefix - service: grafana + - path: /api + pathType: prefix + service: backend + - path: /docs + pathType: Prefix + service: backend + - path: /grafana(/|$)(.*) + pathType: Prefix + service: grafana tls: - secretName: opensampl-tls hosts: From eadb2f7c71c1e6ff58051214bc46878ff8257e11 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:26:32 -0400 Subject: [PATCH 26/49] fix indentation on service.port --- helm/values-mariner.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index e014271..54473bf 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -25,9 +25,9 @@ backend: service: type: ClusterIP port: 8000 - logLevel: "INFO" - useApiKey: false - env: [] + logLevel: "INFO" + useApiKey: false + env: [] # Grafana configuration grafana: From 7d94b670439737393d08e4862d645a55c02b9b28 Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 17:34:41 -0400 Subject: [PATCH 27/49] deploy backend to /api and /docs --- helm/values.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index dbae2f6..753c5cb 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -24,13 +24,12 @@ backend: repository: ghcr.io/ornl/opensampl-backend tag: latest replicas: 1 - port: 8000 - env: - # Additional backend environment variables - - name: LOG_LEVEL - value: INFO - - name: USE_API_KEY - value: "false" + service: + type: ClusterIP + port: 8000 + logLevel: "INFO" + useApiKey: false + env: [] # Grafana service grafana: From 847f2bc62fb3f6c0fc41bba70ea01fe226e2d30f Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 18:33:25 -0400 Subject: [PATCH 28/49] move grafana to the top of hosts --- helm/templates/grafana/deployment.yaml | 2 +- helm/values-mariner.yaml | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index b06cf61..9f132b2 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -40,7 +40,7 @@ spec: key: POSTGRES_PASSWORD # For testing in Mariner - name: GF_SERVER_ROOT_URL - value: "https://opensampl-test.ornl.gov/grafana/" + value: "https://opensampl-test.ornl.gov/grafana" - name: GF_SERVER_SERVE_FROM_SUB_PATH value: "true" readinessProbe: diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 54473bf..9c72706 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -56,15 +56,15 @@ ingress: hosts: - host: opensampl-test.ornl.gov paths: - - path: /api - pathType: prefix - service: backend - - path: /docs - pathType: Prefix - service: backend - - path: /grafana(/|$)(.*) - pathType: Prefix - service: grafana + - path: /grafana + pathType: Prefix + service: grafana + - path: /api + pathType: prefix + service: backend + - path: /docs + pathType: Prefix + service: backend tls: - secretName: opensampl-tls hosts: From 19bec243c7420fe3834e9c1b6d21a92471bf9cee Mon Sep 17 00:00:00 2001 From: 7va Date: Fri, 31 Oct 2025 18:39:10 -0400 Subject: [PATCH 29/49] add subpath value for grafana --- helm/templates/grafana/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/grafana/deployment.yaml b/helm/templates/grafana/deployment.yaml index 9f132b2..cf27599 100644 --- a/helm/templates/grafana/deployment.yaml +++ b/helm/templates/grafana/deployment.yaml @@ -42,7 +42,7 @@ spec: - name: GF_SERVER_ROOT_URL value: "https://opensampl-test.ornl.gov/grafana" - name: GF_SERVER_SERVE_FROM_SUB_PATH - value: "true" + value: {{ .Values.grafana.serveFromSubPath | default "false" | quote }} readinessProbe: httpGet: path: /login From abf5b07676a0ec62ce702ba294997edb74b7d9e3 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 3 Nov 2025 11:17:17 -0500 Subject: [PATCH 30/49] fix /grafana path --- helm/templates/ingress.yaml | 52 +++++++++++++++++++------------------ helm/values-mariner.yaml | 3 --- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 957857b..daf2d68 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -2,38 +2,40 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{ include "opensampl.fullname" . }} + name: {{ include "opensampl.fullname" . }}-ingress + labels: + {{- include "opensampl.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} annotations: - {{- toYaml .Values.ingress.annotations | nindent 4 }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: + {{- if .Values.ingress.className }} ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} tls: - {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} rules: - - host: {{ (index .Values.ingress.hosts 0).host }} + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} http: paths: - - path: /api(/|$)(.*) - pathType: Prefix + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} backend: service: - name: {{ include "opensampl.fullname" . }}-backend + name: {{ include "opensampl.fullname" $ }}-{{ .service }} port: - number: {{ .Values.backend.service.port }} - - - path: /grafana(/|$)(.*) - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-grafana - port: - number: {{ .Values.grafana.port }} - - - path: / - pathType: Prefix - backend: - service: - name: {{ include "opensampl.fullname" . }}-backend - port: - number: {{ .Values.backend.service.port }} -{{- end }} + number: {{ if eq .service "backend" }}{{ $.Values.backend.service.port }}{{ else }}{{ $.Values.grafana.port }}{{ end }} + {{- end }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 9c72706..0d96384 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -59,9 +59,6 @@ ingress: - path: /grafana pathType: Prefix service: grafana - - path: /api - pathType: prefix - service: backend - path: /docs pathType: Prefix service: backend From f97fe259028382eed3f2b9ee732a0fddb667250f Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 3 Nov 2025 11:50:21 -0500 Subject: [PATCH 31/49] remove /docs and server root --- helm/values-mariner.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/values-mariner.yaml b/helm/values-mariner.yaml index 0d96384..915bf7e 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-mariner.yaml @@ -59,7 +59,7 @@ ingress: - path: /grafana pathType: Prefix service: grafana - - path: /docs + - path: / pathType: Prefix service: backend tls: From 8ab54807b81c3ec63c0016e9b1db1eaae6c59c87 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 14:22:57 -0400 Subject: [PATCH 32/49] Clean up helm chart files --- "helm/Accept\357\200\272" | 0 helm/GET | 0 "helm/Host\357\200\272" | 0 "helm/User-Agent\357\200\272" | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "helm/Accept\357\200\272" delete mode 100644 helm/GET delete mode 100644 "helm/Host\357\200\272" delete mode 100644 "helm/User-Agent\357\200\272" diff --git "a/helm/Accept\357\200\272" "b/helm/Accept\357\200\272" deleted file mode 100644 index e69de29..0000000 diff --git a/helm/GET b/helm/GET deleted file mode 100644 index e69de29..0000000 diff --git "a/helm/Host\357\200\272" "b/helm/Host\357\200\272" deleted file mode 100644 index e69de29..0000000 diff --git "a/helm/User-Agent\357\200\272" "b/helm/User-Agent\357\200\272" deleted file mode 100644 index e69de29..0000000 From 829222355c9dc6f2ddfdd194426aef75798f70cb Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 15:33:29 -0400 Subject: [PATCH 33/49] update cluster specific dns opensampl-test.eig3-orc1.ornl.gov --- ...{values-mariner.yaml => values-cluster-overrides.yaml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename helm/{values-mariner.yaml => values-cluster-overrides.yaml} (86%) diff --git a/helm/values-mariner.yaml b/helm/values-cluster-overrides.yaml similarity index 86% rename from helm/values-mariner.yaml rename to helm/values-cluster-overrides.yaml index 915bf7e..f630251 100644 --- a/helm/values-mariner.yaml +++ b/helm/values-cluster-overrides.yaml @@ -1,4 +1,4 @@ -# values-production.yaml +# Kubernetes cluster specific overrides. global: imagePullPolicy: Always @@ -39,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl: "https://opensampl-test.ornl.gov/grafana" + rootUrl:"https://opensampl-test.eig3-orc1.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) @@ -54,7 +54,7 @@ ingress: cert-manager.io/cluster-issuer: "letsencrypt-prod" # If using cert-manager # nginx.ingress.kubernetes.io/ssl-redirect: "true" hosts: - - host: opensampl-test.ornl.gov + - host: opensampl-test.eig3-orc1.ornl.gov paths: - path: /grafana pathType: Prefix @@ -65,4 +65,4 @@ ingress: tls: - secretName: opensampl-tls hosts: - - opensampl-test.ornl.gov \ No newline at end of file + - opensampl-test.eig3-orc1.ornl.gov \ No newline at end of file From fe6907f9b6b3e399fca28dc57048ddc293d7f3fb Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 15:39:31 -0400 Subject: [PATCH 34/49] build chart artifiact v0.1.8 --- helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2d20e33..b0e5438 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.7 +version: 0.1.8 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From 114d050c43466f663c868d86f81340c38679c029 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 16:08:21 -0400 Subject: [PATCH 35/49] add values-override and helm lint values-override to the helm workflow --- .github/workflows/helm-chart.yml | 13 ++++++++++++- helm/Chart.yaml | 2 +- helm/values-cluster-overrides.yaml | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 191faaf..1d7c638 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -80,10 +80,12 @@ jobs: PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') helm install test-opensampl . \ + -f values-cluster-overrides.yaml \ --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ --set migrations.enabled=false \ + --set ingress.enabled=false \ --wait \ --timeout 5m @@ -121,8 +123,17 @@ jobs: version: v3.14.0 - name: Helm Lint - run: helm lint ./helm + run: | + helm lint ./helm + helm lint ./helm -f ./helm/values-cluster-overrides.yaml + - name: Render Helm Chart + run: | + helm template opensampl ./helm \ + -f ./helm/values-cluster-overrides.yaml \ + --namespace opensampl-test \ + > /tmp/opensampl-rendered.yaml + - name: Package Chart run: helm package ./helm diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b0e5438..b471011 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.8 +version: 0.1.9 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/values-cluster-overrides.yaml b/helm/values-cluster-overrides.yaml index f630251..5d36c43 100644 --- a/helm/values-cluster-overrides.yaml +++ b/helm/values-cluster-overrides.yaml @@ -39,7 +39,7 @@ grafana: port: 3000 service: type: ClusterIP - rootUrl:"https://opensampl-test.eig3-orc1.ornl.gov/grafana" + rootUrl: "https://opensampl-test.eig3-orc1.ornl.gov/grafana" serveFromSubPath: true # Migrations (disabled until ready) From 0c9a7151030b008e2573621b3419abf818783cd0 Mon Sep 17 00:00:00 2001 From: 7va Date: Mon, 17 Aug 2026 16:22:05 -0400 Subject: [PATCH 36/49] push chart 0.2.0 --- .github/workflows/helm-chart.old | 76 -------------------------------- .github/workflows/helm-chart.yml | 1 - helm/Chart.yaml | 2 +- 3 files changed, 1 insertion(+), 78 deletions(-) delete mode 100644 .github/workflows/helm-chart.old diff --git a/.github/workflows/helm-chart.old b/.github/workflows/helm-chart.old deleted file mode 100644 index 6e234c0..0000000 --- a/.github/workflows/helm-chart.old +++ /dev/null @@ -1,76 +0,0 @@ -name: Helm Chart CI/CD - -on: - push: - branches: [ helm-deploy ] - pull_request: - branches: [ helm-deploy ] - workflow_dispatch: - -jobs: - build-docker-images: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - strategy: - matrix: - service: - - name: db - context: ./opensampl/db - - name: backend - context: ./opensampl/backend - - name: grafana - context: ./opensampl/server/grafana - #- name: migrations - # context: ./opensampl/migrations - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push ${{ matrix.service.name }} image - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.service.context }} - push: true - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest - cache-from: type=gha,scope=${{ matrix.service.name }} - cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - - helm: - runs-on: ubuntu-latest - needs: build-docker-images - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v3 - with: - version: v3.14.0 - - - name: Helm Lint - run: helm lint ./helm - - - name: Package Chart - run: helm package ./helm - - - name: Push to GHCR - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts - diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 1d7c638..0e9112b 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -80,7 +80,6 @@ jobs: PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') helm install test-opensampl . \ - -f values-cluster-overrides.yaml \ --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ --set grafana.image.tag=pr-${PR_NUMBER} \ diff --git a/helm/Chart.yaml b/helm/Chart.yaml index b471011..f377aed 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.1.9 +version: 0.2.0 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file From e2c1c7db2c0e8de8b21a82161a2d43c7a1adb77b Mon Sep 17 00:00:00 2001 From: 7va Date: Tue, 18 Aug 2026 08:57:21 -0400 Subject: [PATCH 37/49] pin busy box image to eliminate ornl pull limits --- helm/Chart.yaml | 2 +- helm/templates/backend/deployment.yaml | 2 +- helm/values-cluster-overrides.yaml | 2 +- helm/values.yaml | 5 +++++ 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index f377aed..77decd3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.2.0 +version: 0.2.1 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file diff --git a/helm/templates/backend/deployment.yaml b/helm/templates/backend/deployment.yaml index 16d48c1..3976f1d 100644 --- a/helm/templates/backend/deployment.yaml +++ b/helm/templates/backend/deployment.yaml @@ -21,7 +21,7 @@ spec: spec: initContainers: - name: wait-for-db - image: busybox + image: "{{ .Values.busybox.image.repository }}:{{ .Values.busybox.image.tag }}" command: - sh - -c diff --git a/helm/values-cluster-overrides.yaml b/helm/values-cluster-overrides.yaml index 5d36c43..b27719c 100644 --- a/helm/values-cluster-overrides.yaml +++ b/helm/values-cluster-overrides.yaml @@ -1,4 +1,4 @@ -# Kubernetes cluster specific overrides. +# Laboratory and Kubernetes cluster specific overrides. global: imagePullPolicy: Always diff --git a/helm/values.yaml b/helm/values.yaml index 753c5cb..429cfdf 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -2,6 +2,11 @@ global: imagePullPolicy: IfNotPresent +busybox: + image: + repository: ghcr.io/ornl/opensampl-busybox + tag: "1.36.1" + # PostgreSQL Database db: image: From 0a403d88cac24b68bd0bf62646c9a024538b544c Mon Sep 17 00:00:00 2001 From: 7va Date: Thu, 29 Oct 2026 11:24:54 -0400 Subject: [PATCH 38/49] update helm/readme.md with deployment instructions --- helm/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/helm/README.md b/helm/README.md index 4f56c30..3247d61 100644 --- a/helm/README.md +++ b/helm/README.md @@ -1,3 +1,16 @@ +# Deployment +# Update --version + +helm install opensampl-test \ + oci://ghcr.io/ornl/charts/opensampl \ + --version 0.1.7 \ + --namespace opensampl-test \ + --create-namespace \ + --dry-run \ + --debug + + + charts/ └── opensampl/ ├── Chart.yaml From 6b0f35e26d5d6ca866bc49dfa69007364d40c654 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 12:11:28 -0500 Subject: [PATCH 39/49] fixes to address the helm deploly test --- .github/workflows/helm-chart.yml | 32 ++++++++++++++++---------------- helm/values.yaml | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 0e9112b..9468b44 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -15,7 +15,7 @@ jobs: permissions: contents: read packages: write - + strategy: matrix: service: @@ -25,21 +25,21 @@ jobs: context: ./opensampl/backend - name: grafana context: ./opensampl/server/grafana - + steps: - name: Checkout uses: actions/checkout@v4 - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - + - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - + - name: Set image tags id: tags run: | @@ -49,7 +49,7 @@ jobs: else echo "tag=latest" >> $GITHUB_OUTPUT fi - + - name: Build and push ${{ matrix.service.name }} image uses: docker/build-push-action@v5 with: @@ -58,7 +58,7 @@ jobs: tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.tags.outputs.tag }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - + test-helm-chart: name: Test Helm Chart Deployment runs-on: ubuntu-latest @@ -67,18 +67,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - + - name: Start minikube uses: medyagh/setup-minikube@latest - + - name: Set up Helm uses: azure/setup-helm@v4 - + - name: Install Helm Chart working-directory: helm/ run: | PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') - + helm install test-opensampl . \ --set db.image.tag=pr-${PR_NUMBER} \ --set backend.image.tag=pr-${PR_NUMBER} \ @@ -86,14 +86,14 @@ jobs: --set migrations.enabled=false \ --set ingress.enabled=false \ --wait \ - --timeout 5m - + --timeout 15m + - name: Verify Deployment run: | kubectl get pods kubectl get services kubectl get pvc - + - name: Show Pod Logs on Failure if: failure() run: | @@ -132,7 +132,7 @@ jobs: -f ./helm/values-cluster-overrides.yaml \ --namespace opensampl-test \ > /tmp/opensampl-rendered.yaml - + - name: Package Chart run: helm package ./helm @@ -141,4 +141,4 @@ jobs: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts \ No newline at end of file + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/helm/values.yaml b/helm/values.yaml index 429cfdf..16a665e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -4,7 +4,7 @@ global: busybox: image: - repository: ghcr.io/ornl/opensampl-busybox + repository: busybox tag: "1.36.1" # PostgreSQL Database From 732b741993c9da30d36a46434ad4b7a94962e313 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 12:21:11 -0500 Subject: [PATCH 40/49] updates to the tests and to ruff formatting --- .github/workflows/tests.yml | 5 +- opensampl/backend/main.py | 244 +++++++++++++++++++----------------- pyproject.toml | 1 + 3 files changed, 136 insertions(+), 114 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d05ce02..3e6b857 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ on: jobs: uv-pytest: name: Python ${{ matrix.python-version }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -35,7 +35,8 @@ jobs: sudo curl -fsSL \ https://www.postgresql.org/media/keys/ACCC4CF8.asc \ -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc - echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt resolute-pgdg main" \ + codename="$(. /etc/os-release && echo "$VERSION_CODENAME")" + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${codename}-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-16 postgresql-client-16 postgresql-16-postgis-3 postgresql-16-postgis-3-scripts diff --git a/opensampl/backend/main.py b/opensampl/backend/main.py index 26fbb7c..8ea516e 100644 --- a/opensampl/backend/main.py +++ b/opensampl/backend/main.py @@ -1,43 +1,55 @@ +"""FastAPI backend exposing OpenSAMPL database loading and health endpoints.""" + import io import json import os import sys import time -from typing import Any, Dict, Callable, Optional -from datetime import datetime, timedelta +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Any + import pandas as pd -from fastapi import FastAPI, HTTPException, File, UploadFile, Form, Request, Response, Security, Depends -from fastapi.security.api_key import APIKeyHeader +import psycopg2 +from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, Response, Security, UploadFile from fastapi.responses import JSONResponse, RedirectResponse +from fastapi.security.api_key import APIKeyHeader from loguru import logger +from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest from pydantic import BaseModel -from sqlalchemy import create_engine, text, select, or_, and_ -from sqlalchemy.exc import SQLAlchemyError, IntegrityError -from sqlalchemy.orm import sessionmaker, Session -from prometheus_client import Counter, Histogram, generate_latest, CONTENT_TYPE_LATEST -from opensampl.db.access_orm import APIAccessKey +from sqlalchemy import create_engine, or_, select, text +from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from sqlalchemy.orm import Session, sessionmaker + from opensampl import load_data +from opensampl.db.access_orm import APIAccessKey from opensampl.db.orm import ProbeMetadata -from opensampl.vendors.constants import VendorType, ProbeKey from opensampl.metrics import METRICS, MetricType from opensampl.references import REF_TYPES, CompoundReferenceType, ReferenceType -import psycopg2 +from opensampl.vendors.constants import ProbeKey, VendorType + class TimeDataPoint(BaseModel): + """A single time/value sample.""" + time: str value: float class WriteTablePayload(BaseModel): + """Payload for writing a row to a table.""" + table: str - data: Dict[str, Any] - if_exists: load_data.conflict_actions = 'update' + data: dict[str, Any] + if_exists: load_data.conflict_actions = "update" class ProbeMetadataPayload(BaseModel): + """Payload for loading probe metadata.""" + vendor: VendorType probe_key: ProbeKey - data: Dict[str, Any] + data: dict[str, Any] DATABASE_URI = os.getenv("DATABASE_URL") @@ -69,18 +81,19 @@ class ProbeMetadataPayload(BaseModel): api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) def get_keys(): - env_keys = os.getenv('API_KEYS', '').strip() - keys = [k.strip() for k in env_keys.split(',') if k.strip()] + """Return the valid API keys from the environment, falling back to the database.""" + env_keys = os.getenv("API_KEYS", "").strip() + keys = [k.strip() for k in env_keys.split(",") if k.strip()] if keys: logger.debug("api access keys loaded from env") return keys try: - Session = sessionmaker(bind=engine) - with Session() as session: - now = datetime.utcnow() + session_factory = sessionmaker(bind=engine) + with session_factory() as session: + now = datetime.now(tz=UTC).replace(tzinfo=None) stmt = select(APIAccessKey.key).where( or_( - APIAccessKey.expires_at == None, + APIAccessKey.expires_at.is_(None), APIAccessKey.expires_at > now ) ) @@ -93,19 +106,23 @@ def get_keys(): return [] def validate_api_key(api_key: str = Security(api_key_header)): + """Validate the request API key when API key security is enabled.""" if not USE_API_KEY: - return # Security is disabled + return None # Security is disabled if api_key not in get_keys(): raise HTTPException(status_code=403, detail="Invalid or missing API key") return api_key def get_db(): - Session = sessionmaker(bind=engine) + """Yield a database session and close it afterwards.""" + session_factory = sessionmaker(bind=engine) + session = None try: - session = Session() + session = session_factory() yield session finally: - session.close() + if session is not None: + session.close() @app.middleware("http") async def metrics_middleware(request: Request, call_next: Callable) -> Response: @@ -133,14 +150,13 @@ async def metrics_middleware(request: Request, call_next: Callable) -> Response: # add route to docs from / to /docs @app.get("/", include_in_schema=False) async def docs_redirect(): - return RedirectResponse(url='/docs') + """Redirect the root path to the API docs.""" + return RedirectResponse(url="/docs") @app.get("/setloglevel") def set_log_level(newloglevel: str, api_key: str = Depends(validate_api_key)): - """ - change visible log level in backend container - """ + """Change visible log level in backend container""" newloglevel = newloglevel.upper() logger.configure(handlers=[{"sink": sys.stderr, "level": newloglevel}]) return {"loglevel": newloglevel} @@ -148,120 +164,120 @@ def set_log_level(newloglevel: str, api_key: str = Depends(validate_api_key)): @app.get("/checkloglevel") def check_log_level(api_key: str = Depends(validate_api_key)): - """ - This is to check which log levels are visible in backend container - """ - logger.debug('Debug test') - logger.info('Info test') - logger.warning('Warning test') - logger.error('Error test') - current_level = list(logger._core.handlers.values())[0]["level"].name + """Report which log level is currently visible in the backend container.""" + logger.debug("Debug test") + logger.info("Info test") + logger.warning("Warning test") + logger.error("Error test") + current_level = next(iter(logger._core.handlers.values()))["level"].name # noqa: SLF001 return {"loglevel": current_level} @app.post("/write_to_table") -def write_to_table(payload: WriteTablePayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): +def write_to_table( + payload: WriteTablePayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) +): + """Write a row of data to the given table.""" try: load_data.write_to_table(table=payload.table, data=payload.data, if_exists=payload.if_exists, session=session) - logger.debug(f'Successfully wrote to {payload.table} using: {payload.data}') + logger.debug(f"Successfully wrote to {payload.table} using: {payload.data}") return JSONResponse(content={"message": f"Succeeded loading data into {payload.table}"}, status_code=200) except IntegrityError as e: if isinstance(e.orig, psycopg2.errors.UniqueViolation): return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) except SQLAlchemyError as e: - logger.error(f'SQLAlchemy error: {e}') + logger.error(f"SQLAlchemy error: {e}") return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) except json.JSONDecodeError as e: - logger.error(f'JSON decode error: {e}') + logger.error(f"JSON decode error: {e}") return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) except Exception as e: - logger.error(f'Unexpected error: {e}') + logger.error(f"Unexpected error: {e}") return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) -@app.post("/load_time_data") -async def load_time_data(probe_key_str: str = Form(...), - metric_type_str: Optional[str] = Form(None), - reference_type_str: Optional[str] = Form(None), - compound_key_str: Optional[str] = Form(None), - file: UploadFile = File(...), - api_key: str = Depends(validate_api_key), - session: Session = Depends(get_db)): - try: +def _parse_metric_type(metric_type_str: str | None) -> MetricType: + if metric_type_str is None: + return METRICS.UNKNOWN + return MetricType(**json.loads(metric_type_str)) - probe_key = ProbeKey(**json.loads(probe_key_str)) - if metric_type_str is not None: - metric_type_dict = json.loads(metric_type_str) - metric_type = MetricType(**metric_type_dict) - else: - metric_type = METRICS.UNKNOWN - - if reference_type_str is not None: - reference_type_dict = json.loads(reference_type_str) - if 'reference_table' in reference_type_dict: - reference_type = CompoundReferenceType(**reference_type_dict) - else: - reference_type = ReferenceType(**reference_type_dict) - else: - reference_type = REF_TYPES.UNKNOWN +def _parse_reference_type(reference_type_str: str | None) -> ReferenceType | CompoundReferenceType: + if reference_type_str is None: + return REF_TYPES.UNKNOWN + reference_type_dict = json.loads(reference_type_str) + if "reference_table" in reference_type_dict: + return CompoundReferenceType(**reference_type_dict) + return ReferenceType(**reference_type_dict) + + +def _discard_session(session: Session | None) -> None: + if session: + session.rollback() + session.close() + +@app.post("/load_time_data") +async def load_time_data( + probe_key_str: str = Form(...), + metric_type_str: str | None = Form(None), + reference_type_str: str | None = Form(None), + compound_key_str: str | None = Form(None), + file: UploadFile = File(...), + api_key: str = Depends(validate_api_key), + session: Session = Depends(get_db), +): + """Load a CSV of time series data for a probe.""" + try: + probe_key = ProbeKey(**json.loads(probe_key_str)) + metric_type = _parse_metric_type(metric_type_str) + reference_type = _parse_reference_type(reference_type_str) compound_key = None if compound_key_str is None else json.loads(compound_key_str) content = await file.read() df = pd.read_csv(io.BytesIO(content)) logger.info(df.head()) # Convert time strings back to datetime - df['time'] = pd.to_datetime(df['time']) - - # Convert value strings back to float64 - # df['value'] = df['value'].astype('float64') + df["time"] = pd.to_datetime(df["time"]) - # Use the same load_time_data function as before load_data.load_time_data( probe_key=probe_key, metric_type=metric_type, reference_type=reference_type, compound_key=compound_key, data=df, - session=session + session=session, ) - return JSONResponse( - content={"message": f"Successfully loaded {len(df)} data points"}, - status_code=200 - ) + return JSONResponse(content={"message": f"Successfully loaded {len(df)} data points"}, status_code=200) except IntegrityError as e: - if session: - session.rollback() - session.close() + _discard_session(session) if isinstance(e.orig, psycopg2.errors.UniqueViolation): return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) except SQLAlchemyError as e: - logger.error(f'Database error: {e}') - if session: - session.rollback() - session.close() - raise HTTPException(status_code=500, detail=f"Database error: {str(e)}") + logger.error(f"Database error: {e}") + _discard_session(session) + raise HTTPException(status_code=500, detail=f"Database error: {e!s}") from e except Exception as e: - logger.error(f'Unexpected error: {e}') - if session: - session.rollback() - session.close() - raise HTTPException(status_code=500, detail=f"Error processing time series data: {str(e)}") + logger.error(f"Unexpected error: {e}") + _discard_session(session) + raise HTTPException(status_code=500, detail=f"Error processing time series data: {e!s}") from e @app.post("/load_probe_metadata") -def load_probe_metadata(payload: ProbeMetadataPayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): +def load_probe_metadata( + payload: ProbeMetadataPayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) +): + """Load metadata for a probe.""" logger.debug(f"Received payload: {payload.model_dump()}") try: load_data.load_probe_metadata(vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, session=session) logger.debug( - f'Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}') + f"Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}") return JSONResponse(content={"message": f"Succeeded loaded metadata for {payload.probe_key}"}, status_code=200) except IntegrityError as e: session.rollback() @@ -269,69 +285,76 @@ def load_probe_metadata(payload: ProbeMetadataPayload, api_key: str = Depends(va return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) except SQLAlchemyError as e: - logger.error(f'SQLAlchemy error: {e}') + logger.error(f"SQLAlchemy error: {e}") return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) except json.JSONDecodeError as e: - logger.error(f'JSON decode error: {e}') + logger.error(f"JSON decode error: {e}") return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) except Exception as e: - logger.exception(f'Unexpected error: {e}') + logger.exception(f"Unexpected error: {e}") return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) @app.get("/create_new_tables") -def create_new_tables(create_schema: bool = True, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db)): +def create_new_tables( + create_schema: bool = True, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) +): + """Create any tables that do not yet exist.""" try: load_data.create_new_tables(create_schema=create_schema, session=session) - return JSONResponse(content={"message": f"Succeeded in creating any new tables"}, status_code=200) + return JSONResponse(content={"message": "Succeeded in creating any new tables"}, status_code=200) except SQLAlchemyError as e: - logger.error(f'SQLAlchemy error: {e}') + logger.error(f"SQLAlchemy error: {e}") return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) except json.JSONDecodeError as e: - logger.error(f'JSON decode error: {e}') + logger.error(f"JSON decode error: {e}") return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) except Exception as e: - logger.error(f'Unexpected error: {e}') + logger.error(f"Unexpected error: {e}") return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) @app.get("/gen_api_key") -def generate_api_key(expire_after: Optional[int] = None, session: Session = Depends(get_db)): +def generate_api_key(expire_after: int | None = None, session: Session = Depends(get_db)): + """Generate a new API access key, optionally expiring after the given number of days.""" try: - new_key = APIAccessKey() new_key.generate_key() if expire_after: - new_key.expires_at = datetime.utcnow() + timedelta(days=expire_after) + new_key.expires_at = datetime.now(tz=UTC).replace(tzinfo=None) + timedelta(days=expire_after) session.add(new_key) session.commit() - return JSONResponse(content={"message": f"Succeeded in creating new access key"}, status_code=200) + return JSONResponse(content={"message": "Succeeded in creating new access key"}, status_code=200) except SQLAlchemyError as e: - logger.error(f'SQLAlchemy error: {e}') + logger.error(f"SQLAlchemy error: {e}") return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) except Exception as e: - logger.error(f'Unexpected error: {e}') + logger.error(f"Unexpected error: {e}") return JSONResponse(content={"message": f"Failed to create new access key: {e}"}, status_code=500) @app.get("/healthcheck") def healthcheck(): + """Report that the service is up.""" return {"status": "OK"} @app.get("/healthcheck_database") def healthcheck_db(): + """Report whether the database is reachable.""" try: with engine.connect() as connection: connection.execute(text("SELECT 1")) + except SQLAlchemyError as e: + raise HTTPException(status_code=503, detail="Database connection error") from e + else: return {"status": "OK"} - except SQLAlchemyError: - raise HTTPException(status_code=503, detail="Database connection error") @app.get("/healthcheck_metadata") def healthcheck_metadata(): + """Report whether the castdb schema exists.""" try: with engine.connect() as connection: result = connection.execute( @@ -339,15 +362,12 @@ def healthcheck_metadata(): schema_exists = result.fetchone() is not None if schema_exists: return {"status": "OK"} - else: - raise HTTPException(status_code=500, detail="Schema 'castdb' does not exist") + raise HTTPException(status_code=500, detail="Schema 'castdb' does not exist") except SQLAlchemyError as e: - raise HTTPException(status_code=503, detail=f"Database connection error: {str(e)}") + raise HTTPException(status_code=503, detail=f"Database connection error: {e!s}") from e @app.get("/metrics", include_in_schema=False) def metrics(): - """ - Expose Prometheus metrics. - """ + """Expose Prometheus metrics.""" return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) diff --git a/pyproject.toml b/pyproject.toml index 67d8612..c115921 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,6 +165,7 @@ ignore = ["D203", "D212", "D400", "D415", "ANN401", "S101", "PLR2004", "COM812", [tool.ruff.lint.per-file-ignores] "opensampl/vendors/**/*.py" = ['S311'] # we want to ignore the errors about random "opensampl/server/backend/main.py" = ['B008', 'ARG001'] #ignore complaints about calling functions in args +"opensampl/backend/main.py" = ['B008', 'ARG001', 'INP001'] # FastAPI Depends/Form args; backend is a script, not a package "opensampl/mixins/random_data.py" = ['S311'] [tool.ruff.lint.pylint] From bbd3b56ece77c8fb06baaa28a811e7aa380dc21b Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 13:44:09 -0500 Subject: [PATCH 41/49] updates to the tests and to ruff formatting --- .github/workflows/tests.yml | 6 ++--- opensampl/backend/main.py | 44 ++++++++++++++----------------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e6b857..3bbf7bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,14 +32,14 @@ jobs: run: | sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -fsSL \ + sudo curl -fsSL --retry 5 --retry-all-errors --retry-delay 3 --connect-timeout 20 \ https://www.postgresql.org/media/keys/ACCC4CF8.asc \ -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc codename="$(. /etc/os-release && echo "$VERSION_CODENAME")" echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${codename}-pgdg main" \ | sudo tee /etc/apt/sources.list.d/pgdg.list - sudo apt-get update - sudo apt-get install -y postgresql-16 postgresql-client-16 postgresql-16-postgis-3 postgresql-16-postgis-3-scripts + sudo apt-get -o Acquire::Retries=5 update + sudo apt-get -o Acquire::Retries=5 install -y postgresql-16 postgresql-client-16 postgresql-16-postgis-3 postgresql-16-postgis-3-scripts echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH" - name: Install the project diff --git a/opensampl/backend/main.py b/opensampl/backend/main.py index 8ea516e..fdd0180 100644 --- a/opensampl/backend/main.py +++ b/opensampl/backend/main.py @@ -59,16 +59,10 @@ class ProbeMetadataPayload(BaseModel): app = FastAPI() -REQUEST_COUNT = Counter( - "http_requests_total", - "Total number of HTTP requests", - ["method", "endpoint", "http_status"] -) +REQUEST_COUNT = Counter("http_requests_total", "Total number of HTTP requests", ["method", "endpoint", "http_status"]) REQUEST_LATENCY = Histogram( - "http_request_duration_seconds", - "Duration of HTTP requests in seconds", - ["method", "endpoint"] + "http_request_duration_seconds", "Duration of HTTP requests in seconds", ["method", "endpoint"] ) EXCLUDED_PATHS = {"/metrics", "/healthcheck", "/healthcheck_database", "/healthcheck_metadata"} @@ -80,6 +74,7 @@ class ProbeMetadataPayload(BaseModel): api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) + def get_keys(): """Return the valid API keys from the environment, falling back to the database.""" env_keys = os.getenv("API_KEYS", "").strip() @@ -91,12 +86,7 @@ def get_keys(): session_factory = sessionmaker(bind=engine) with session_factory() as session: now = datetime.now(tz=UTC).replace(tzinfo=None) - stmt = select(APIAccessKey.key).where( - or_( - APIAccessKey.expires_at.is_(None), - APIAccessKey.expires_at > now - ) - ) + stmt = select(APIAccessKey.key).where(or_(APIAccessKey.expires_at.is_(None), APIAccessKey.expires_at > now)) result = session.execute(stmt) keys = [row[0] for row in result.all()] logger.debug("api access keys loaded from db") @@ -105,6 +95,7 @@ def get_keys(): logger.debug(f"exception attempting to load api access keys from db: {e}") return [] + def validate_api_key(api_key: str = Security(api_key_header)): """Validate the request API key when API key security is enabled.""" if not USE_API_KEY: @@ -113,6 +104,7 @@ def validate_api_key(api_key: str = Security(api_key_header)): raise HTTPException(status_code=403, detail="Invalid or missing API key") return api_key + def get_db(): """Yield a database session and close it afterwards.""" session_factory = sessionmaker(bind=engine) @@ -124,6 +116,7 @@ def get_db(): if session is not None: session.close() + @app.middleware("http") async def metrics_middleware(request: Request, call_next: Callable) -> Response: """Middleware to track request metrics.""" @@ -133,16 +126,9 @@ async def metrics_middleware(request: Request, call_next: Callable) -> Response: response: Response = await call_next(request) duration = time.time() - start_time - REQUEST_COUNT.labels( - method=request.method, - endpoint=request.url.path, - http_status=response.status_code - ).inc() + REQUEST_COUNT.labels(method=request.method, endpoint=request.url.path, http_status=response.status_code).inc() - REQUEST_LATENCY.labels( - method=request.method, - endpoint=request.url.path - ).observe(duration) + REQUEST_LATENCY.labels(method=request.method, endpoint=request.url.path).observe(duration) return response @@ -274,10 +260,12 @@ def load_probe_metadata( logger.debug(f"Received payload: {payload.model_dump()}") try: - load_data.load_probe_metadata(vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, - session=session) + load_data.load_probe_metadata( + vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, session=session + ) logger.debug( - f"Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}") + f"Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}" + ) return JSONResponse(content={"message": f"Succeeded loaded metadata for {payload.probe_key}"}, status_code=200) except IntegrityError as e: session.rollback() @@ -340,6 +328,7 @@ def healthcheck(): """Report that the service is up.""" return {"status": "OK"} + @app.get("/healthcheck_database") def healthcheck_db(): """Report whether the database is reachable.""" @@ -358,7 +347,8 @@ def healthcheck_metadata(): try: with engine.connect() as connection: result = connection.execute( - text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'castdb';")) + text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'castdb';") + ) schema_exists = result.fetchone() is not None if schema_exists: return {"status": "OK"} From 7b4537574390a7bffad1e1a7518252df1f0a788f Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 14:31:33 -0500 Subject: [PATCH 42/49] updates to the documentation process --- docs/api/backend/main.md | 7 +++++++ docs/api/index.md | 2 ++ mkdocs.yaml | 2 ++ opensampl/backend/__init__.py | 1 + pyproject.toml | 2 +- 5 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/api/backend/main.md create mode 100644 opensampl/backend/__init__.py diff --git a/docs/api/backend/main.md b/docs/api/backend/main.md new file mode 100644 index 0000000..b6c9074 --- /dev/null +++ b/docs/api/backend/main.md @@ -0,0 +1,7 @@ +# `opensampl.backend.main` + +::: opensampl.backend.main + options: + show_root_heading: false + show_submodules: true + show_source: true diff --git a/docs/api/index.md b/docs/api/index.md index 1d0f052..a9e8a78 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4,6 +4,8 @@ Welcome to the OpenSAMPL API documentation. Browse the modules and packages below: +- Backend + - [Main](backend/main.md) - [Cli](cli.md) - Collect - [Cli](collect/cli.md) diff --git a/mkdocs.yaml b/mkdocs.yaml index 4af6484..5a3326d 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -20,6 +20,8 @@ nav: - Random Data: guides/random-data-generation.md - API: - index: api/index.md + - backend: + - main: api/backend/main.md - cli: api/cli.md - collect: - cli: api/collect/cli.md diff --git a/opensampl/backend/__init__.py b/opensampl/backend/__init__.py new file mode 100644 index 0000000..0a903d1 --- /dev/null +++ b/opensampl/backend/__init__.py @@ -0,0 +1 @@ +"""FastAPI backend service for OpenSAMPL.""" diff --git a/pyproject.toml b/pyproject.toml index c115921..26ed91b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,7 +165,7 @@ ignore = ["D203", "D212", "D400", "D415", "ANN401", "S101", "PLR2004", "COM812", [tool.ruff.lint.per-file-ignores] "opensampl/vendors/**/*.py" = ['S311'] # we want to ignore the errors about random "opensampl/server/backend/main.py" = ['B008', 'ARG001'] #ignore complaints about calling functions in args -"opensampl/backend/main.py" = ['B008', 'ARG001', 'INP001'] # FastAPI Depends/Form args; backend is a script, not a package +"opensampl/backend/main.py" = ['B008', 'ARG001'] # FastAPI Depends/Form args "opensampl/mixins/random_data.py" = ['S311'] [tool.ruff.lint.pylint] From 425bf19f247d91afebe03ca50e6d4a8c6cdc38a0 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 15:05:53 -0500 Subject: [PATCH 43/49] chore: update helm chart version and workflows --- .github/workflows/helm-chart.yml | 134 ++++++++++++++++++++++--------- .github/workflows/helm-readme.md | 31 +++---- .github/workflows/publish.yml | 2 +- .github/workflows/static.yml | 2 +- helm/Chart.yaml | 4 +- 5 files changed, 112 insertions(+), 61 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 9468b44..031ce52 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -1,12 +1,12 @@ name: Helm Chart CI/CD +# Pull requests only validate (build images locally, lint the chart, test-install it). +# Images and the Helm chart are published to GHCR only after a merge to main. on: push: - branches: [ helm-deploy ] + branches: [ main ] pull_request: - branches: - - helm-deploy - - main + branches: [ main ] workflow_dispatch: jobs: @@ -26,6 +26,9 @@ jobs: - name: grafana context: ./opensampl/server/grafana + env: + PUBLISH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} + steps: - name: Checkout uses: actions/checkout@v4 @@ -33,32 +36,87 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Get package version + id: version + run: echo "version=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2)" >> "$GITHUB_OUTPUT" + - name: Log in to GitHub Container Registry + if: env.PUBLISH == 'true' uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set image tags - id: tags - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - PR_NUMBER=$(echo ${{ github.ref }} | awk -F'/' '{print $3}') - echo "tag=pr-${PR_NUMBER}" >> $GITHUB_OUTPUT - else - echo "tag=latest" >> $GITHUB_OUTPUT - fi - - name: Build and push ${{ matrix.service.name }} image + if: env.PUBLISH == 'true' uses: docker/build-push-action@v5 with: context: ${{ matrix.service.context }} push: true - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.tags.outputs.tag }} + tags: | + ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.version.outputs.version }} cache-from: type=gha,scope=${{ matrix.service.name }} cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} + - name: Build ${{ matrix.service.name }} image (not pushed) + if: env.PUBLISH != 'true' + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.service.context }} + push: false + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:pr-local + outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.service.name }}.tar + cache-from: type=gha,scope=${{ matrix.service.name }} + + - name: Upload image for chart test + if: env.PUBLISH != 'true' + uses: actions/upload-artifact@v4 + with: + name: image-${{ matrix.service.name }} + path: ${{ runner.temp }}/${{ matrix.service.name }}.tar + retention-days: 1 + + helm-validate: + name: Lint Helm Chart + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.14.0 + + - name: Helm Lint + run: | + helm lint ./helm + helm lint ./helm -f ./helm/values-cluster-overrides.yaml + + - name: Render Helm Chart + run: | + helm template opensampl ./helm \ + -f ./helm/values-cluster-overrides.yaml \ + --namespace opensampl-test \ + > "$RUNNER_TEMP/opensampl-rendered.yaml" + + - name: Require chart version bump when chart changes + if: github.event_name == 'pull_request' + run: | + git fetch origin main --depth=1 + if ! git diff --quiet origin/main -- helm; then + base="$(git show origin/main:helm/Chart.yaml | grep -m1 '^version:' | awk '{print $2}')" + head="$(grep -m1 '^version:' helm/Chart.yaml | awk '{print $2}')" + if [ "$base" = "$head" ]; then + echo "::error file=helm/Chart.yaml::helm/ changed but chart version is still $head; bump it so the push to GHCR does not collide." + exit 1 + fi + fi + test-helm-chart: name: Test Helm Chart Deployment runs-on: ubuntu-latest @@ -71,18 +129,31 @@ jobs: - name: Start minikube uses: medyagh/setup-minikube@latest + - name: Download built images + uses: actions/download-artifact@v4 + with: + pattern: image-* + path: ${{ runner.temp }}/images + merge-multiple: true + + - name: Load images into minikube + run: | + for tar in "$RUNNER_TEMP"/images/*.tar; do + minikube image load "$tar" + done + minikube image ls + - name: Set up Helm uses: azure/setup-helm@v4 - name: Install Helm Chart working-directory: helm/ run: | - PR_NUMBER=$(echo ${GITHUB_REF} | awk -F'/' '{print $3}') - helm install test-opensampl . \ - --set db.image.tag=pr-${PR_NUMBER} \ - --set backend.image.tag=pr-${PR_NUMBER} \ - --set grafana.image.tag=pr-${PR_NUMBER} \ + --set global.imagePullPolicy=Never \ + --set db.image.tag=pr-local \ + --set backend.image.tag=pr-local \ + --set grafana.image.tag=pr-local \ --set migrations.enabled=false \ --set ingress.enabled=false \ --wait \ @@ -104,14 +175,17 @@ jobs: echo "=== Pod Logs ===" for pod in $(kubectl get pods -o name); do echo "Logs for $pod:" - kubectl logs $pod --all-containers=true || true + kubectl logs "$pod" --all-containers=true || true done helm-package: name: Package and Push Helm Chart runs-on: ubuntu-latest - needs: build-docker-images - if: github.event_name != 'pull_request' + needs: [build-docker-images, helm-validate] + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write steps: - name: Checkout uses: actions/checkout@v4 @@ -121,18 +195,6 @@ jobs: with: version: v3.14.0 - - name: Helm Lint - run: | - helm lint ./helm - helm lint ./helm -f ./helm/values-cluster-overrides.yaml - - - name: Render Helm Chart - run: | - helm template opensampl ./helm \ - -f ./helm/values-cluster-overrides.yaml \ - --namespace opensampl-test \ - > /tmp/opensampl-rendered.yaml - - name: Package Chart run: helm package ./helm @@ -140,5 +202,5 @@ jobs: env: CR_PAT: ${{ secrets.GITHUB_TOKEN }} run: | - helm registry login ghcr.io -u $GITHUB_ACTOR -p $CR_PAT + helm registry login ghcr.io -u "$GITHUB_ACTOR" -p "$CR_PAT" helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/.github/workflows/helm-readme.md b/.github/workflows/helm-readme.md index 4580128..f74660d 100644 --- a/.github/workflows/helm-readme.md +++ b/.github/workflows/helm-readme.md @@ -1,27 +1,16 @@ -### 1. **Smart Image Tagging** -Images now get tagged with: -- `pr-123` for pull requests -- `helm-deploy-abc123` for branch commits -- `latest` for main branch +# Helm chart workflow (`helm-chart.yml`) -### 2. **New Test Job** (`test-helm-chart`) -- **Only runs on PRs** (not regular pushes) -- Spins up local Kubernetes (minikube) -- Installs your Helm chart with PR-tagged images -- Verifies pods start successfully -- Shows detailed logs if anything fails +Artifacts are published **only after a merge to `main`**. Pull requests validate but never push. -### 3. **Separated Helm Packaging** -- `helm-package` job only runs on **direct pushes** (not PRs) -- Lints, packages, and pushes the chart - -## Workflow Flow - -**On Pull Request:** +## On pull requests to `main` ``` -Build Images (with pr-123 tags) → Test in Kubernetes → ✓ Pass/Fail +Build images (not pushed) -> load into minikube -> helm install test +Helm lint + template, and a check that helm/Chart.yaml version was bumped if helm/ changed ``` +- Images are built with the `pr-local` tag, shared as short-lived workflow artifacts, and loaded into minikube with `imagePullPolicy=Never`, so nothing is pulled from or pushed to GHCR. +- If anything under `helm/` changes, bump `version` in `helm/Chart.yaml` (otherwise `helm push` would collide with an existing version). -**On Push to helm-deploy:** +## On push to `main` +``` +Build + push images (latest and the pyproject version) -> package + push chart to oci://ghcr.io/ornl/charts ``` -Build Images (with latest tag) → Package & Push Helm Chart \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 052c719..e9fbb44 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,7 +4,7 @@ on: push: branches: - main - workflow_dispatch: + workflow_dispatch: permissions: contents: read pages: write diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index fff4697..09d5afa 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -4,7 +4,7 @@ on: push: branches: - main - workflow_dispatch: + workflow_dispatch: permissions: contents: read pages: write diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 77decd3..bf397e0 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.2.1 +version: 0.2.2 appVersion: "1.0.0" -icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png \ No newline at end of file +icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png From d6dc27683fd5245805b3e65a73f308440a6f3582 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 15:28:33 -0500 Subject: [PATCH 44/49] updates to the remove or refactor the backend to server appropriately; also run tests, cause they are useful --- .github/dependabot.yml | 41 +- .github/workflows/build-images.yml | 117 ++++++ .github/workflows/dependency-security.yml | 5 +- .github/workflows/helm-chart.yml | 206 ---------- .github/workflows/helm-checks.yml | 120 ++++++ .github/workflows/helm-readme.md | 30 +- .github/workflows/lint.yml | 2 - .github/workflows/publish-artifacts.yml | 53 +++ .github/workflows/tests.yml | 2 - docs/api/backend/main.md | 7 - docs/api/index.md | 2 - helm/Chart.yaml | 2 +- helm/templates/db/statefulset.yaml | 6 + helm/templates/migrations/migrations-job.yaml | 26 +- mkdocs.yaml | 2 - opensampl/backend/Dockerfile | 9 - opensampl/backend/README.md | 2 - opensampl/backend/__init__.py | 1 - opensampl/backend/main.py | 363 ------------------ opensampl/backend/requirements.txt | 8 - opensampl/server/backend/Dockerfile | 17 +- opensampl/{ => server}/db/Dockerfile | 0 opensampl/{ => server}/db/create-grafana.sh | 0 opensampl/server/docker-bake.hcl | 28 +- opensampl/server/docker-compose.dev.yaml | 2 + opensampl/server/docker-compose.yaml | 2 + opensampl/server/migrations/Dockerfile | 17 +- pyproject.toml | 3 +- 28 files changed, 426 insertions(+), 647 deletions(-) create mode 100644 .github/workflows/build-images.yml delete mode 100644 .github/workflows/helm-chart.yml create mode 100644 .github/workflows/helm-checks.yml create mode 100644 .github/workflows/publish-artifacts.yml delete mode 100644 docs/api/backend/main.md delete mode 100644 opensampl/backend/Dockerfile delete mode 100644 opensampl/backend/README.md delete mode 100644 opensampl/backend/__init__.py delete mode 100644 opensampl/backend/main.py delete mode 100644 opensampl/backend/requirements.txt rename opensampl/{ => server}/db/Dockerfile (100%) rename opensampl/{ => server}/db/create-grafana.sh (100%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6c33cba..e8f90ee 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,54 +1,53 @@ version: 2 +# Security-only configuration. +# +# `open-pull-requests-limit: 0` turns off routine version-update PRs for an ecosystem while +# Dependabot *security* updates (PRs for dependencies with known advisories/CVEs) keep working. +# The `schedule` block is still required by the schema, but it only affects version updates. +# Prerequisite: "Dependabot security updates" must be enabled in the repository's +# Settings > Code security. +# +# The `security-updates` groups below batch advisory fixes for an ecosystem into a single PR. + updates: - package-ecosystem: uv directory: / target-branch: bugfix/dependabot + open-pull-requests-limit: 0 schedule: interval: weekly - day: monday - time: "09:00" - timezone: America/New_York groups: - python-minor-and-patch: + python-security: + applies-to: security-updates patterns: - "*" - update-types: - - minor - - patch - package-ecosystem: github-actions directory: / target-branch: bugfix/dependabot + open-pull-requests-limit: 0 schedule: interval: weekly - day: monday - time: "09:15" - timezone: America/New_York groups: - actions-minor-and-patch: + actions-security: + applies-to: security-updates patterns: - "*" - update-types: - - minor - - patch - package-ecosystem: docker directories: - / - /opensampl/server/backend + - /opensampl/server/db - /opensampl/server/grafana - /opensampl/server/migrations target-branch: bugfix/dependabot + open-pull-requests-limit: 0 schedule: interval: weekly - day: monday - time: "09:30" - timezone: America/New_York groups: - container-minor-and-patch: + container-security: + applies-to: security-updates patterns: - "*" - update-types: - - minor - - patch diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 0000000..0ec9ed6 --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,117 @@ +name: Build images + +# Reusable workflow: builds the opensampl wheel once, then builds every container image from it. +# push=false -> images are built but NOT published (exported as artifacts for the chart test). +# push=true -> images are pushed to GHCR (caller must grant packages: write). +on: + workflow_call: + inputs: + push: + description: Push images to GHCR instead of exporting them as artifacts + type: boolean + default: false + +jobs: + build-wheel: + name: Build wheel + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.12.21" + + - name: Build wheel + run: uv build --wheel --out-dir dist + + - name: Get package version + id: version + run: echo "version=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2)" >> "$GITHUB_OUTPUT" + + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: opensampl-wheel + path: dist/*.whl + if-no-files-found: error + retention-days: 1 + + build-images: + name: Build ${{ matrix.service.name }} + runs-on: ubuntu-latest + needs: build-wheel + strategy: + fail-fast: false + matrix: + service: + - name: db + context: ./opensampl/server/db + - name: backend + context: ./opensampl/server/backend + target: prod + wheel: true + - name: grafana + context: ./opensampl/server/grafana + - name: migrations + context: ./opensampl/server/migrations + wheel: true + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Download wheel + if: matrix.service.wheel + uses: actions/download-artifact@v4 + with: + name: opensampl-wheel + path: ${{ runner.temp }}/dist + + - name: Log in to GitHub Container Registry + if: inputs.push + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ matrix.service.name }} image + if: inputs.push + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.service.context }} + target: ${{ matrix.service.target }} + build-contexts: ${{ matrix.service.wheel && format('wheel={0}/dist', runner.temp) || '' }} + build-args: OPENSAMPL_VERSION=${{ needs.build-wheel.outputs.version }} + push: true + tags: | + ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest + ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ needs.build-wheel.outputs.version }} + cache-from: type=gha,scope=${{ matrix.service.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} + + - name: Build ${{ matrix.service.name }} image (not pushed) + if: ${{ !inputs.push }} + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.service.context }} + target: ${{ matrix.service.target }} + build-contexts: ${{ matrix.service.wheel && format('wheel={0}/dist', runner.temp) || '' }} + build-args: OPENSAMPL_VERSION=${{ needs.build-wheel.outputs.version }} + push: false + tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:pr-local + outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.service.name }}.tar + cache-from: type=gha,scope=${{ matrix.service.name }} + + - name: Upload image for chart test + if: ${{ !inputs.push }} + uses: actions/upload-artifact@v4 + with: + name: image-${{ matrix.service.name }} + path: ${{ runner.temp }}/${{ matrix.service.name }}.tar + retention-days: 1 diff --git a/.github/workflows/dependency-security.yml b/.github/workflows/dependency-security.yml index 8b5fd6a..aa481e1 100644 --- a/.github/workflows/dependency-security.yml +++ b/.github/workflows/dependency-security.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] schedule: - cron: "17 9 * * 1" workflow_dispatch: @@ -15,6 +14,8 @@ permissions: jobs: ensure-branch: + # writes a branch, so never run it for pull requests (fork PRs only get a read-only token) + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: write @@ -33,6 +34,8 @@ jobs: audit: needs: ensure-branch + # run even when ensure-branch is skipped (pull requests) + if: ${{ !cancelled() && (needs.ensure-branch.result == 'success' || needs.ensure-branch.result == 'skipped') }} name: Audit locked dependencies runs-on: ubuntu-latest steps: diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml deleted file mode 100644 index 031ce52..0000000 --- a/.github/workflows/helm-chart.yml +++ /dev/null @@ -1,206 +0,0 @@ -name: Helm Chart CI/CD - -# Pull requests only validate (build images locally, lint the chart, test-install it). -# Images and the Helm chart are published to GHCR only after a merge to main. -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: - -jobs: - build-docker-images: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - strategy: - matrix: - service: - - name: db - context: ./opensampl/db - - name: backend - context: ./opensampl/backend - - name: grafana - context: ./opensampl/server/grafana - - env: - PUBLISH: ${{ github.event_name != 'pull_request' && github.ref == 'refs/heads/main' }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Get package version - id: version - run: echo "version=$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2)" >> "$GITHUB_OUTPUT" - - - name: Log in to GitHub Container Registry - if: env.PUBLISH == 'true' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push ${{ matrix.service.name }} image - if: env.PUBLISH == 'true' - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.service.context }} - push: true - tags: | - ghcr.io/ornl/opensampl-${{ matrix.service.name }}:latest - ghcr.io/ornl/opensampl-${{ matrix.service.name }}:${{ steps.version.outputs.version }} - cache-from: type=gha,scope=${{ matrix.service.name }} - cache-to: type=gha,mode=max,scope=${{ matrix.service.name }} - - - name: Build ${{ matrix.service.name }} image (not pushed) - if: env.PUBLISH != 'true' - uses: docker/build-push-action@v5 - with: - context: ${{ matrix.service.context }} - push: false - tags: ghcr.io/ornl/opensampl-${{ matrix.service.name }}:pr-local - outputs: type=docker,dest=${{ runner.temp }}/${{ matrix.service.name }}.tar - cache-from: type=gha,scope=${{ matrix.service.name }} - - - name: Upload image for chart test - if: env.PUBLISH != 'true' - uses: actions/upload-artifact@v4 - with: - name: image-${{ matrix.service.name }} - path: ${{ runner.temp }}/${{ matrix.service.name }}.tar - retention-days: 1 - - helm-validate: - name: Lint Helm Chart - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Helm - uses: azure/setup-helm@v4 - with: - version: v3.14.0 - - - name: Helm Lint - run: | - helm lint ./helm - helm lint ./helm -f ./helm/values-cluster-overrides.yaml - - - name: Render Helm Chart - run: | - helm template opensampl ./helm \ - -f ./helm/values-cluster-overrides.yaml \ - --namespace opensampl-test \ - > "$RUNNER_TEMP/opensampl-rendered.yaml" - - - name: Require chart version bump when chart changes - if: github.event_name == 'pull_request' - run: | - git fetch origin main --depth=1 - if ! git diff --quiet origin/main -- helm; then - base="$(git show origin/main:helm/Chart.yaml | grep -m1 '^version:' | awk '{print $2}')" - head="$(grep -m1 '^version:' helm/Chart.yaml | awk '{print $2}')" - if [ "$base" = "$head" ]; then - echo "::error file=helm/Chart.yaml::helm/ changed but chart version is still $head; bump it so the push to GHCR does not collide." - exit 1 - fi - fi - - test-helm-chart: - name: Test Helm Chart Deployment - runs-on: ubuntu-latest - needs: build-docker-images - if: github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Start minikube - uses: medyagh/setup-minikube@latest - - - name: Download built images - uses: actions/download-artifact@v4 - with: - pattern: image-* - path: ${{ runner.temp }}/images - merge-multiple: true - - - name: Load images into minikube - run: | - for tar in "$RUNNER_TEMP"/images/*.tar; do - minikube image load "$tar" - done - minikube image ls - - - name: Set up Helm - uses: azure/setup-helm@v4 - - - name: Install Helm Chart - working-directory: helm/ - run: | - helm install test-opensampl . \ - --set global.imagePullPolicy=Never \ - --set db.image.tag=pr-local \ - --set backend.image.tag=pr-local \ - --set grafana.image.tag=pr-local \ - --set migrations.enabled=false \ - --set ingress.enabled=false \ - --wait \ - --timeout 15m - - - name: Verify Deployment - run: | - kubectl get pods - kubectl get services - kubectl get pvc - - - name: Show Pod Logs on Failure - if: failure() - run: | - echo "=== Pod Status ===" - kubectl get pods - echo "=== Pod Descriptions ===" - kubectl describe pods - echo "=== Pod Logs ===" - for pod in $(kubectl get pods -o name); do - echo "Logs for $pod:" - kubectl logs "$pod" --all-containers=true || true - done - - helm-package: - name: Package and Push Helm Chart - runs-on: ubuntu-latest - needs: [build-docker-images, helm-validate] - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Helm - uses: azure/setup-helm@v4 - with: - version: v3.14.0 - - - name: Package Chart - run: helm package ./helm - - - name: Push to GHCR - env: - CR_PAT: ${{ secrets.GITHUB_TOKEN }} - run: | - helm registry login ghcr.io -u "$GITHUB_ACTOR" -p "$CR_PAT" - helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/.github/workflows/helm-checks.yml b/.github/workflows/helm-checks.yml new file mode 100644 index 0000000..3e08300 --- /dev/null +++ b/.github/workflows/helm-checks.yml @@ -0,0 +1,120 @@ +name: Helm chart checks + +# Runs on every pull request. Builds images without pushing, lints the chart and test-installs it. +# Nothing is published from this workflow; see publish-artifacts.yml for that. +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: helm-checks-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-images: + uses: ./.github/workflows/build-images.yml + with: + push: false + + helm-validate: + name: Lint Helm chart + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.14.0 + + - name: Helm lint + run: | + helm lint ./helm + helm lint ./helm -f ./helm/values-cluster-overrides.yaml + + - name: Render Helm chart + run: | + helm template opensampl ./helm \ + -f ./helm/values-cluster-overrides.yaml \ + --namespace opensampl-test \ + > "$RUNNER_TEMP/opensampl-rendered.yaml" + + - name: Require chart version bump when chart changes + if: github.event_name == 'pull_request' + run: | + git fetch origin main --depth=1 + if ! git diff --quiet origin/main -- helm; then + base="$(git show origin/main:helm/Chart.yaml | grep -m1 '^version:' | awk '{print $2}')" + head="$(grep -m1 '^version:' helm/Chart.yaml | awk '{print $2}')" + if [ "$base" = "$head" ]; then + echo "::error file=helm/Chart.yaml::helm/ changed but chart version is still $head; bump it so the push to GHCR does not collide." + exit 1 + fi + fi + + test-helm-chart: + name: Test Helm chart deployment + runs-on: ubuntu-latest + needs: build-images + steps: + - uses: actions/checkout@v4 + + - name: Start minikube + uses: medyagh/setup-minikube@latest + + - name: Download built images + uses: actions/download-artifact@v4 + with: + pattern: image-* + path: ${{ runner.temp }}/images + merge-multiple: true + + - name: Load images into minikube + run: | + for tar in "$RUNNER_TEMP"/images/*.tar; do + minikube image load "$tar" + done + minikube image ls + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Install Helm chart + working-directory: helm/ + run: | + helm install test-opensampl . \ + --set global.imagePullPolicy=Never \ + --set db.image.tag=pr-local \ + --set backend.image.tag=pr-local \ + --set grafana.image.tag=pr-local \ + --set migrations.image.tag=pr-local \ + --set migrations.enabled=true \ + --set ingress.enabled=false \ + --wait \ + --wait-for-jobs \ + --timeout 15m + + - name: Verify deployment + run: | + kubectl get pods + kubectl get services + kubectl get pvc + kubectl get jobs + + - name: Show pod logs on failure + if: failure() + run: | + echo "=== Pod Status ===" + kubectl get pods + echo "=== Pod Descriptions ===" + kubectl describe pods + echo "=== Pod Logs ===" + for pod in $(kubectl get pods -o name); do + echo "Logs for $pod:" + kubectl logs "$pod" --all-containers=true || true + done diff --git a/.github/workflows/helm-readme.md b/.github/workflows/helm-readme.md index f74660d..b1be320 100644 --- a/.github/workflows/helm-readme.md +++ b/.github/workflows/helm-readme.md @@ -1,16 +1,28 @@ -# Helm chart workflow (`helm-chart.yml`) +# CI/CD overview -Artifacts are published **only after a merge to `main`**. Pull requests validate but never push. +Checks run on **every pull request**. Images and charts are published **only after a merge to `main`**. -## On pull requests to `main` +| Workflow | Trigger | Publishes? | +|---|---|---| +| `lint.yml`, `tests.yml`, `dependency-security.yml` | every PR, push to `main` | no | +| `helm-checks.yml` | every PR | no | +| `publish-artifacts.yml` | push to `main` | images + Helm chart to GHCR | +| `publish.yml` (PyPI), `static.yml` (Pages) | push to `main` | yes | +| `build-images.yml` | reusable (called by the two above) | only when `push: true` | + +## Images are built from a wheel +`build-images.yml` runs `uv build --wheel` once and passes the wheel to the `backend` and `migrations` +Docker builds as the named context `wheel`. Their Dockerfiles install that wheel when present and fall back +to `opensampl==$OPENSAMPL_VERSION` from PyPI for local builds, so images always contain the code being built. + +## Pull requests (`helm-checks.yml`) ``` -Build images (not pushed) -> load into minikube -> helm install test -Helm lint + template, and a check that helm/Chart.yaml version was bumped if helm/ changed +build wheel -> build 4 images (not pushed) -> load into minikube -> helm install (migrations enabled) +helm lint + template, and a check that helm/Chart.yaml `version` was bumped if helm/ changed ``` -- Images are built with the `pr-local` tag, shared as short-lived workflow artifacts, and loaded into minikube with `imagePullPolicy=Never`, so nothing is pulled from or pushed to GHCR. -- If anything under `helm/` changes, bump `version` in `helm/Chart.yaml` (otherwise `helm push` would collide with an existing version). +Images are tagged `pr-local` and installed with `imagePullPolicy=Never`; nothing touches GHCR. -## On push to `main` +## Push to `main` (`publish-artifacts.yml`) ``` -Build + push images (latest and the pyproject version) -> package + push chart to oci://ghcr.io/ornl/charts +build wheel -> build + push db/backend/grafana/migrations (latest and the pyproject version) -> package + push chart ``` diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1ba914a..7b1ddeb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,8 +5,6 @@ on: branches: - main pull_request: - branches: - - main workflow_dispatch: permissions: diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml new file mode 100644 index 0000000..d706547 --- /dev/null +++ b/.github/workflows/publish-artifacts.yml @@ -0,0 +1,53 @@ +name: Publish images and Helm chart + +# Publishes container images and the Helm chart to GHCR. Only runs for main. +# Pull request validation lives in helm-checks.yml. +on: + push: + branches: [ main ] + workflow_dispatch: + +concurrency: + group: publish-artifacts + cancel-in-progress: false + +jobs: + build-images: + if: github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-images.yml + with: + push: true + + helm-package: + name: Package and push Helm chart + runs-on: ubuntu-latest + needs: build-images + if: github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: v3.14.0 + + - name: Helm lint + run: | + helm lint ./helm + helm lint ./helm -f ./helm/values-cluster-overrides.yaml + + - name: Package chart + run: helm package ./helm + + - name: Push to GHCR + env: + CR_PAT: ${{ secrets.GITHUB_TOKEN }} + run: | + helm registry login ghcr.io -u "$GITHUB_ACTOR" -p "$CR_PAT" + helm push opensampl-*.tgz oci://ghcr.io/ornl/charts diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3bbf7bb..f40b46e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,8 +4,6 @@ on: branches: - main pull_request: - branches: - - main workflow_dispatch: jobs: diff --git a/docs/api/backend/main.md b/docs/api/backend/main.md deleted file mode 100644 index b6c9074..0000000 --- a/docs/api/backend/main.md +++ /dev/null @@ -1,7 +0,0 @@ -# `opensampl.backend.main` - -::: opensampl.backend.main - options: - show_root_heading: false - show_submodules: true - show_source: true diff --git a/docs/api/index.md b/docs/api/index.md index a9e8a78..1d0f052 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -4,8 +4,6 @@ Welcome to the OpenSAMPL API documentation. Browse the modules and packages below: -- Backend - - [Main](backend/main.md) - [Cli](cli.md) - Collect - [Cli](collect/cli.md) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index bf397e0..08f3ee3 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: opensampl description: Helm chart for OpenSAMPL (Postgres + Backend + Grafana + Migrations) type: application -version: 0.2.2 +version: 0.2.3 appVersion: "1.0.0" icon: https://raw.githubusercontent.com/ORNL/OpenSAMPL/main/docs/logo.png diff --git a/helm/templates/db/statefulset.yaml b/helm/templates/db/statefulset.yaml index 1724be1..715355e 100644 --- a/helm/templates/db/statefulset.yaml +++ b/helm/templates/db/statefulset.yaml @@ -26,6 +26,12 @@ spec: - name: db image: "{{ .Values.db.image.repository }}:{{ .Values.db.image.tag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + - postgres + - -c + - shared_preload_libraries=timescaledb,pg_cron + - -c + - cron.database_name=$(POSTGRES_DB) ports: - containerPort: {{ .Values.db.port }} name: postgres diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 8edd519..8d5ca1e 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -3,19 +3,39 @@ apiVersion: batch/v1 kind: Job metadata: name: {{ include "opensampl.fullname" . }}-migrations + labels: + {{- include "opensampl.labels" . | nindent 4 }} + role: migrations spec: template: spec: + initContainers: + - name: wait-for-db + image: "{{ .Values.busybox.image.repository }}:{{ .Values.busybox.image.tag }}" + command: + - sh + - -c + - > + until nc -z {{ include "opensampl.fullname" . }}-db {{ .Values.db.port }}; + do echo "Waiting for DB..."; + sleep 3; + done; containers: - name: migrations image: "{{ .Values.migrations.image.repository }}:{{ .Values.migrations.image.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} env: - - name: DATABASE_URL - value: postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + # the migrations image (alembic env.py) reads DB_URI + - name: DB_URI + value: postgresql+psycopg2://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@{{ include "opensampl.fullname" . }}-db:{{ .Values.db.port }}/{{ .Values.db.database }} + {{- range $key, $value := .Values.migrations.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} envFrom: - secretRef: name: {{ include "opensampl.fullname" . }}-db-secret - configMapRef: name: {{ include "opensampl.fullname" . }}-db-config restartPolicy: OnFailure -{{- end }} \ No newline at end of file +{{- end }} diff --git a/mkdocs.yaml b/mkdocs.yaml index 5a3326d..4af6484 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -20,8 +20,6 @@ nav: - Random Data: guides/random-data-generation.md - API: - index: api/index.md - - backend: - - main: api/backend/main.md - cli: api/cli.md - collect: - cli: api/collect/cli.md diff --git a/opensampl/backend/Dockerfile b/opensampl/backend/Dockerfile deleted file mode 100644 index d1ab545..0000000 --- a/opensampl/backend/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.11 -WORKDIR /tmp -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt && \ - rm requirements.txt -ENV ROUTE_TO_BACKEND=false -COPY main.py . -CMD ["uvicorn", "main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"] -# CMD ["tail", "-f", "/dev/null"] \ No newline at end of file diff --git a/opensampl/backend/README.md b/opensampl/backend/README.md deleted file mode 100644 index f46eed8..0000000 --- a/opensampl/backend/README.md +++ /dev/null @@ -1,2 +0,0 @@ -CAST Backend -http://localhost:8000/docs - swagger endpoint \ No newline at end of file diff --git a/opensampl/backend/__init__.py b/opensampl/backend/__init__.py deleted file mode 100644 index 0a903d1..0000000 --- a/opensampl/backend/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""FastAPI backend service for OpenSAMPL.""" diff --git a/opensampl/backend/main.py b/opensampl/backend/main.py deleted file mode 100644 index fdd0180..0000000 --- a/opensampl/backend/main.py +++ /dev/null @@ -1,363 +0,0 @@ -"""FastAPI backend exposing OpenSAMPL database loading and health endpoints.""" - -import io -import json -import os -import sys -import time -from collections.abc import Callable -from datetime import UTC, datetime, timedelta -from typing import Any - -import pandas as pd -import psycopg2 -from fastapi import Depends, FastAPI, File, Form, HTTPException, Request, Response, Security, UploadFile -from fastapi.responses import JSONResponse, RedirectResponse -from fastapi.security.api_key import APIKeyHeader -from loguru import logger -from prometheus_client import CONTENT_TYPE_LATEST, Counter, Histogram, generate_latest -from pydantic import BaseModel -from sqlalchemy import create_engine, or_, select, text -from sqlalchemy.exc import IntegrityError, SQLAlchemyError -from sqlalchemy.orm import Session, sessionmaker - -from opensampl import load_data -from opensampl.db.access_orm import APIAccessKey -from opensampl.db.orm import ProbeMetadata -from opensampl.metrics import METRICS, MetricType -from opensampl.references import REF_TYPES, CompoundReferenceType, ReferenceType -from opensampl.vendors.constants import ProbeKey, VendorType - - -class TimeDataPoint(BaseModel): - """A single time/value sample.""" - - time: str - value: float - - -class WriteTablePayload(BaseModel): - """Payload for writing a row to a table.""" - - table: str - data: dict[str, Any] - if_exists: load_data.conflict_actions = "update" - - -class ProbeMetadataPayload(BaseModel): - """Payload for loading probe metadata.""" - - vendor: VendorType - probe_key: ProbeKey - data: dict[str, Any] - - -DATABASE_URI = os.getenv("DATABASE_URL") -engine = create_engine(DATABASE_URI) - -loglevel = os.getenv("BACKEND_LOG_LEVEL", "INFO") -app = FastAPI() - - -REQUEST_COUNT = Counter("http_requests_total", "Total number of HTTP requests", ["method", "endpoint", "http_status"]) - -REQUEST_LATENCY = Histogram( - "http_request_duration_seconds", "Duration of HTTP requests in seconds", ["method", "endpoint"] -) - -EXCLUDED_PATHS = {"/metrics", "/healthcheck", "/healthcheck_database", "/healthcheck_metadata"} - -logger.configure(handlers=[{"sink": sys.stderr, "level": loglevel}]) - -USE_API_KEY = os.getenv("USE_API_KEY", "false").lower() == "true" -API_KEY_NAME = "access-key" - -api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False) - - -def get_keys(): - """Return the valid API keys from the environment, falling back to the database.""" - env_keys = os.getenv("API_KEYS", "").strip() - keys = [k.strip() for k in env_keys.split(",") if k.strip()] - if keys: - logger.debug("api access keys loaded from env") - return keys - try: - session_factory = sessionmaker(bind=engine) - with session_factory() as session: - now = datetime.now(tz=UTC).replace(tzinfo=None) - stmt = select(APIAccessKey.key).where(or_(APIAccessKey.expires_at.is_(None), APIAccessKey.expires_at > now)) - result = session.execute(stmt) - keys = [row[0] for row in result.all()] - logger.debug("api access keys loaded from db") - return keys - except Exception as e: - logger.debug(f"exception attempting to load api access keys from db: {e}") - return [] - - -def validate_api_key(api_key: str = Security(api_key_header)): - """Validate the request API key when API key security is enabled.""" - if not USE_API_KEY: - return None # Security is disabled - if api_key not in get_keys(): - raise HTTPException(status_code=403, detail="Invalid or missing API key") - return api_key - - -def get_db(): - """Yield a database session and close it afterwards.""" - session_factory = sessionmaker(bind=engine) - session = None - try: - session = session_factory() - yield session - finally: - if session is not None: - session.close() - - -@app.middleware("http") -async def metrics_middleware(request: Request, call_next: Callable) -> Response: - """Middleware to track request metrics.""" - if request.url.path in EXCLUDED_PATHS: - return await call_next(request) - start_time = time.time() - response: Response = await call_next(request) - duration = time.time() - start_time - - REQUEST_COUNT.labels(method=request.method, endpoint=request.url.path, http_status=response.status_code).inc() - - REQUEST_LATENCY.labels(method=request.method, endpoint=request.url.path).observe(duration) - - return response - - -# add route to docs from / to /docs -@app.get("/", include_in_schema=False) -async def docs_redirect(): - """Redirect the root path to the API docs.""" - return RedirectResponse(url="/docs") - - -@app.get("/setloglevel") -def set_log_level(newloglevel: str, api_key: str = Depends(validate_api_key)): - """Change visible log level in backend container""" - newloglevel = newloglevel.upper() - logger.configure(handlers=[{"sink": sys.stderr, "level": newloglevel}]) - return {"loglevel": newloglevel} - - -@app.get("/checkloglevel") -def check_log_level(api_key: str = Depends(validate_api_key)): - """Report which log level is currently visible in the backend container.""" - logger.debug("Debug test") - logger.info("Info test") - logger.warning("Warning test") - logger.error("Error test") - current_level = next(iter(logger._core.handlers.values()))["level"].name # noqa: SLF001 - return {"loglevel": current_level} - - -@app.post("/write_to_table") -def write_to_table( - payload: WriteTablePayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) -): - """Write a row of data to the given table.""" - try: - load_data.write_to_table(table=payload.table, data=payload.data, if_exists=payload.if_exists, session=session) - logger.debug(f"Successfully wrote to {payload.table} using: {payload.data}") - return JSONResponse(content={"message": f"Succeeded loading data into {payload.table}"}, status_code=200) - except IntegrityError as e: - if isinstance(e.orig, psycopg2.errors.UniqueViolation): - return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) - return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) - except SQLAlchemyError as e: - logger.error(f"SQLAlchemy error: {e}") - return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) - except json.JSONDecodeError as e: - logger.error(f"JSON decode error: {e}") - return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) - except Exception as e: - logger.error(f"Unexpected error: {e}") - return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) - - -def _parse_metric_type(metric_type_str: str | None) -> MetricType: - if metric_type_str is None: - return METRICS.UNKNOWN - return MetricType(**json.loads(metric_type_str)) - - -def _parse_reference_type(reference_type_str: str | None) -> ReferenceType | CompoundReferenceType: - if reference_type_str is None: - return REF_TYPES.UNKNOWN - reference_type_dict = json.loads(reference_type_str) - if "reference_table" in reference_type_dict: - return CompoundReferenceType(**reference_type_dict) - return ReferenceType(**reference_type_dict) - - -def _discard_session(session: Session | None) -> None: - if session: - session.rollback() - session.close() - - -@app.post("/load_time_data") -async def load_time_data( - probe_key_str: str = Form(...), - metric_type_str: str | None = Form(None), - reference_type_str: str | None = Form(None), - compound_key_str: str | None = Form(None), - file: UploadFile = File(...), - api_key: str = Depends(validate_api_key), - session: Session = Depends(get_db), -): - """Load a CSV of time series data for a probe.""" - try: - probe_key = ProbeKey(**json.loads(probe_key_str)) - metric_type = _parse_metric_type(metric_type_str) - reference_type = _parse_reference_type(reference_type_str) - compound_key = None if compound_key_str is None else json.loads(compound_key_str) - - content = await file.read() - df = pd.read_csv(io.BytesIO(content)) - logger.info(df.head()) - # Convert time strings back to datetime - df["time"] = pd.to_datetime(df["time"]) - - load_data.load_time_data( - probe_key=probe_key, - metric_type=metric_type, - reference_type=reference_type, - compound_key=compound_key, - data=df, - session=session, - ) - - return JSONResponse(content={"message": f"Successfully loaded {len(df)} data points"}, status_code=200) - except IntegrityError as e: - _discard_session(session) - if isinstance(e.orig, psycopg2.errors.UniqueViolation): - return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) - return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) - except SQLAlchemyError as e: - logger.error(f"Database error: {e}") - _discard_session(session) - raise HTTPException(status_code=500, detail=f"Database error: {e!s}") from e - except Exception as e: - logger.error(f"Unexpected error: {e}") - _discard_session(session) - raise HTTPException(status_code=500, detail=f"Error processing time series data: {e!s}") from e - - -@app.post("/load_probe_metadata") -def load_probe_metadata( - payload: ProbeMetadataPayload, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) -): - """Load metadata for a probe.""" - logger.debug(f"Received payload: {payload.model_dump()}") - - try: - load_data.load_probe_metadata( - vendor=payload.vendor, probe_key=payload.probe_key, data=payload.data, session=session - ) - logger.debug( - f"Successfully wrote to {ProbeMetadata.__tablename__} and {payload.vendor.metadata_table}: {payload.data}" - ) - return JSONResponse(content={"message": f"Succeeded loaded metadata for {payload.probe_key}"}, status_code=200) - except IntegrityError as e: - session.rollback() - if isinstance(e.orig, psycopg2.errors.UniqueViolation): - return JSONResponse(content={"message": f"Unique violation error: {e}"}, status_code=409) - return JSONResponse(content={"message": f"Integrity error: {e}"}, status_code=500) - except SQLAlchemyError as e: - logger.error(f"SQLAlchemy error: {e}") - return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) - except json.JSONDecodeError as e: - logger.error(f"JSON decode error: {e}") - return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) - except Exception as e: - logger.exception(f"Unexpected error: {e}") - return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) - - -@app.get("/create_new_tables") -def create_new_tables( - create_schema: bool = True, api_key: str = Depends(validate_api_key), session: Session = Depends(get_db) -): - """Create any tables that do not yet exist.""" - try: - load_data.create_new_tables(create_schema=create_schema, session=session) - return JSONResponse(content={"message": "Succeeded in creating any new tables"}, status_code=200) - except SQLAlchemyError as e: - logger.error(f"SQLAlchemy error: {e}") - return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) - except json.JSONDecodeError as e: - logger.error(f"JSON decode error: {e}") - return JSONResponse(content={"message": f"Invalid JSON data: {e}"}, status_code=400) - except Exception as e: - logger.error(f"Unexpected error: {e}") - return JSONResponse(content={"message": f"Failed to load JSON into database: {e}"}, status_code=500) - - -@app.get("/gen_api_key") -def generate_api_key(expire_after: int | None = None, session: Session = Depends(get_db)): - """Generate a new API access key, optionally expiring after the given number of days.""" - try: - new_key = APIAccessKey() - new_key.generate_key() - if expire_after: - new_key.expires_at = datetime.now(tz=UTC).replace(tzinfo=None) + timedelta(days=expire_after) - - session.add(new_key) - - session.commit() - return JSONResponse(content={"message": "Succeeded in creating new access key"}, status_code=200) - except SQLAlchemyError as e: - logger.error(f"SQLAlchemy error: {e}") - return JSONResponse(content={"message": f"Database error: {e}"}, status_code=500) - except Exception as e: - logger.error(f"Unexpected error: {e}") - return JSONResponse(content={"message": f"Failed to create new access key: {e}"}, status_code=500) - - -@app.get("/healthcheck") -def healthcheck(): - """Report that the service is up.""" - return {"status": "OK"} - - -@app.get("/healthcheck_database") -def healthcheck_db(): - """Report whether the database is reachable.""" - try: - with engine.connect() as connection: - connection.execute(text("SELECT 1")) - except SQLAlchemyError as e: - raise HTTPException(status_code=503, detail="Database connection error") from e - else: - return {"status": "OK"} - - -@app.get("/healthcheck_metadata") -def healthcheck_metadata(): - """Report whether the castdb schema exists.""" - try: - with engine.connect() as connection: - result = connection.execute( - text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'castdb';") - ) - schema_exists = result.fetchone() is not None - if schema_exists: - return {"status": "OK"} - raise HTTPException(status_code=500, detail="Schema 'castdb' does not exist") - except SQLAlchemyError as e: - raise HTTPException(status_code=503, detail=f"Database connection error: {e!s}") from e - - -@app.get("/metrics", include_in_schema=False) -def metrics(): - """Expose Prometheus metrics.""" - return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) diff --git a/opensampl/backend/requirements.txt b/opensampl/backend/requirements.txt deleted file mode 100644 index ca91da0..0000000 --- a/opensampl/backend/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -psycopg2-binary -fastapi -uvicorn -sqlalchemy -loguru -python-multipart -prometheus-client -opensampl==1.1.5 diff --git a/opensampl/server/backend/Dockerfile b/opensampl/server/backend/Dockerfile index b0bedc6..10e877f 100644 --- a/opensampl/server/backend/Dockerfile +++ b/opensampl/server/backend/Dockerfile @@ -1,13 +1,24 @@ +# Optional build context holding a locally built opensampl wheel. +# CI overrides this stage with `--build-context wheel=`; +# without it the stage is empty and the released version is installed from PyPI. +FROM scratch AS wheel + FROM python:3.14 AS base -ARG OPENSAMPL_VERSION=1.2.6 +ARG OPENSAMPL_VERSION=1.2.7 WORKDIR /tmp ENV ROUTE_TO_BACKEND=false FROM base AS prod -RUN pip install --no-cache-dir \ - "opensampl[backend]==${OPENSAMPL_VERSION}" \ +RUN --mount=type=bind,from=wheel,target=/wheels \ + if ls /wheels/opensampl-*.whl >/dev/null 2>&1; then \ + spec="$(ls /wheels/opensampl-*.whl)[backend]"; \ + else \ + spec="opensampl[backend]==${OPENSAMPL_VERSION}"; \ + fi && \ + pip install --no-cache-dir \ + "${spec}" \ "anyio>=4.14.2,<5" \ "urllib3>=2.8.0,<3" diff --git a/opensampl/db/Dockerfile b/opensampl/server/db/Dockerfile similarity index 100% rename from opensampl/db/Dockerfile rename to opensampl/server/db/Dockerfile diff --git a/opensampl/db/create-grafana.sh b/opensampl/server/db/create-grafana.sh similarity index 100% rename from opensampl/db/create-grafana.sh rename to opensampl/server/db/create-grafana.sh diff --git a/opensampl/server/docker-bake.hcl b/opensampl/server/docker-bake.hcl index a7d858c..3188396 100644 --- a/opensampl/server/docker-bake.hcl +++ b/opensampl/server/docker-bake.hcl @@ -23,14 +23,39 @@ variable "PLATFORMS" { default = ["linux/amd64", "linux/arm64"] } +// Directory containing a locally built opensampl wheel (`uv build --wheel`). +// If it holds no wheel the images install OPENSAMPL_VERSION from PyPI. +variable "WHEEL_DIR" { + default = "../../dist" +} + group "default" { - targets = ["backend", "migrations"] + targets = ["db", "backend", "grafana", "migrations"] +} + +target "db" { + context = "./db" + platforms = PLATFORMS + tags = [ + "${REGISTRY}/db:latest", + "${REGISTRY}/db:${OPENSAMPL_VERSION}", + ] +} + +target "grafana" { + context = "./grafana" + platforms = PLATFORMS + tags = [ + "${REGISTRY}/grafana:latest", + "${REGISTRY}/grafana:${OPENSAMPL_VERSION}", + ] } target "backend" { context = "./backend" dockerfile = "Dockerfile" target = "prod" + contexts = { wheel = WHEEL_DIR } platforms = PLATFORMS args = { OPENSAMPL_VERSION = OPENSAMPL_VERSION @@ -44,6 +69,7 @@ target "backend" { target "migrations" { context = "./migrations" dockerfile = "Dockerfile" + contexts = { wheel = WHEEL_DIR } platforms = PLATFORMS args = { OPENSAMPL_VERSION = OPENSAMPL_VERSION diff --git a/opensampl/server/docker-compose.dev.yaml b/opensampl/server/docker-compose.dev.yaml index 90cfa7b..418df6a 100644 --- a/opensampl/server/docker-compose.dev.yaml +++ b/opensampl/server/docker-compose.dev.yaml @@ -1,6 +1,8 @@ services: db: image: savannah.ornl.gov/opensampl/db:latest + build: + context: ./db ports: - "5415:5432" volumes: diff --git a/opensampl/server/docker-compose.yaml b/opensampl/server/docker-compose.yaml index 5a7184b..9bbb867 100644 --- a/opensampl/server/docker-compose.yaml +++ b/opensampl/server/docker-compose.yaml @@ -1,6 +1,8 @@ services: db: image: savannah.ornl.gov/opensampl/db:latest + build: + context: ./db ports: - "5415:5432" volumes: diff --git a/opensampl/server/migrations/Dockerfile b/opensampl/server/migrations/Dockerfile index 93bb28c..2b37cf6 100644 --- a/opensampl/server/migrations/Dockerfile +++ b/opensampl/server/migrations/Dockerfile @@ -1,10 +1,21 @@ +# Optional build context holding a locally built opensampl wheel. +# CI overrides this stage with `--build-context wheel=`; +# without it the stage is empty and the released version is installed from PyPI. +FROM scratch AS wheel + FROM python:3.14 -ARG OPENSAMPL_VERSION=1.2.6 +ARG OPENSAMPL_VERSION=1.2.7 WORKDIR / -RUN pip install --no-cache-dir \ - "opensampl[migrations]==${OPENSAMPL_VERSION}" \ +RUN --mount=type=bind,from=wheel,target=/wheels \ + if ls /wheels/opensampl-*.whl >/dev/null 2>&1; then \ + spec="$(ls /wheels/opensampl-*.whl)[migrations]"; \ + else \ + spec="opensampl[migrations]==${OPENSAMPL_VERSION}"; \ + fi && \ + pip install --no-cache-dir \ + "${spec}" \ "mako>=1.4.2,<2" \ "urllib3>=2.8.0,<3" diff --git a/pyproject.toml b/pyproject.toml index 26ed91b..fb42ec2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ include = [ "opensampl/server/migrations/*", "opensampl/server/grafana/*", "opensampl/server/backend/*", + "opensampl/server/db/*", "opensampl/create/templates/*", "opensampl/**/*.py", ] @@ -139,6 +140,7 @@ include = [ "opensampl/server/migrations/*", "opensampl/server/grafana/*", "opensampl/server/backend/*", + "opensampl/server/db/*", "opensampl/create/templates/*", "opensampl/**/*.py", ] @@ -165,7 +167,6 @@ ignore = ["D203", "D212", "D400", "D415", "ANN401", "S101", "PLR2004", "COM812", [tool.ruff.lint.per-file-ignores] "opensampl/vendors/**/*.py" = ['S311'] # we want to ignore the errors about random "opensampl/server/backend/main.py" = ['B008', 'ARG001'] #ignore complaints about calling functions in args -"opensampl/backend/main.py" = ['B008', 'ARG001'] # FastAPI Depends/Form args "opensampl/mixins/random_data.py" = ['S311'] [tool.ruff.lint.pylint] From c234175dab58a9a1ee9eefa7442550b44864105a Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 15:39:48 -0500 Subject: [PATCH 45/49] back to the current opensampl version --- opensampl/server/backend/Dockerfile | 2 +- opensampl/server/migrations/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensampl/server/backend/Dockerfile b/opensampl/server/backend/Dockerfile index 10e877f..a541a26 100644 --- a/opensampl/server/backend/Dockerfile +++ b/opensampl/server/backend/Dockerfile @@ -4,7 +4,7 @@ FROM scratch AS wheel FROM python:3.14 AS base -ARG OPENSAMPL_VERSION=1.2.7 +ARG OPENSAMPL_VERSION=1.2.6 WORKDIR /tmp ENV ROUTE_TO_BACKEND=false diff --git a/opensampl/server/migrations/Dockerfile b/opensampl/server/migrations/Dockerfile index 2b37cf6..90dff4a 100644 --- a/opensampl/server/migrations/Dockerfile +++ b/opensampl/server/migrations/Dockerfile @@ -5,7 +5,7 @@ FROM scratch AS wheel FROM python:3.14 -ARG OPENSAMPL_VERSION=1.2.7 +ARG OPENSAMPL_VERSION=1.2.6 WORKDIR / RUN --mount=type=bind,from=wheel,target=/wheels \ From 4b7f775c842897ebb52c7a8d253eea91331a31ba Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 15:46:35 -0500 Subject: [PATCH 46/49] update all the things --- opensampl/server/backend/Dockerfile | 2 +- opensampl/server/migrations/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opensampl/server/backend/Dockerfile b/opensampl/server/backend/Dockerfile index a541a26..10e877f 100644 --- a/opensampl/server/backend/Dockerfile +++ b/opensampl/server/backend/Dockerfile @@ -4,7 +4,7 @@ FROM scratch AS wheel FROM python:3.14 AS base -ARG OPENSAMPL_VERSION=1.2.6 +ARG OPENSAMPL_VERSION=1.2.7 WORKDIR /tmp ENV ROUTE_TO_BACKEND=false diff --git a/opensampl/server/migrations/Dockerfile b/opensampl/server/migrations/Dockerfile index 90dff4a..2b37cf6 100644 --- a/opensampl/server/migrations/Dockerfile +++ b/opensampl/server/migrations/Dockerfile @@ -5,7 +5,7 @@ FROM scratch AS wheel FROM python:3.14 -ARG OPENSAMPL_VERSION=1.2.6 +ARG OPENSAMPL_VERSION=1.2.7 WORKDIR / RUN --mount=type=bind,from=wheel,target=/wheels \ From 80087d99ebcdc12c2ead43ac7da58c29be12880a Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 15:50:23 -0500 Subject: [PATCH 47/49] update helm to include migrations --- helm/templates/migrations/migrations-job.yaml | 6 ++++++ helm/values-cluster-overrides.yaml | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 8d5ca1e..0dae126 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -6,6 +6,12 @@ metadata: labels: {{- include "opensampl.labels" . | nindent 4 }} role: migrations + annotations: + # Run as a Helm hook so migrations execute after the db exists on install and are re-run on every + # upgrade (a plain Job is immutable and would block or skip `helm upgrade`). A failed Job is kept + # for log inspection; a successful one is removed. + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: template: spec: diff --git a/helm/values-cluster-overrides.yaml b/helm/values-cluster-overrides.yaml index b27719c..73d1aad 100644 --- a/helm/values-cluster-overrides.yaml +++ b/helm/values-cluster-overrides.yaml @@ -42,9 +42,9 @@ grafana: rootUrl: "https://opensampl-test.eig3-orc1.ornl.gov/grafana" serveFromSubPath: true -# Migrations (disabled until ready) +# Migrations (run as a post-install/post-upgrade Helm hook) migrations: - enabled: false + enabled: true # Ingress configuration ingress: From a4aa883b0670fa5f53b0fd788532227ed21d12c9 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 16:03:03 -0500 Subject: [PATCH 48/49] docs update, also include workflows to only run when relevant changes have been made --- .github/workflows/dependency-security.yml | 8 +++ .github/workflows/helm-checks.yml | 7 +++ .github/workflows/lint.yml | 16 ++++++ .github/workflows/tests.yml | 12 +++++ docs/guides/index.md | 1 + docs/guides/opensampl-server.md | 2 + helm/README.md | 64 +++++++++++++---------- helm/templates/NOTES.txt | 2 +- mkdocs.yaml | 4 ++ 9 files changed, 87 insertions(+), 29 deletions(-) diff --git a/.github/workflows/dependency-security.yml b/.github/workflows/dependency-security.yml index aa481e1..2cfd900 100644 --- a/.github/workflows/dependency-security.yml +++ b/.github/workflows/dependency-security.yml @@ -3,7 +3,15 @@ name: Dependency security on: push: branches: [main] + paths: + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/dependency-security.yml" pull_request: + paths: + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/dependency-security.yml" schedule: - cron: "17 9 * * 1" workflow_dispatch: diff --git a/.github/workflows/helm-checks.yml b/.github/workflows/helm-checks.yml index 3e08300..9eff1ff 100644 --- a/.github/workflows/helm-checks.yml +++ b/.github/workflows/helm-checks.yml @@ -4,6 +4,13 @@ name: Helm chart checks # Nothing is published from this workflow; see publish-artifacts.yml for that. on: pull_request: + paths: + - "helm/**" + - "opensampl/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/helm-checks.yml" + - ".github/workflows/build-images.yml" workflow_dispatch: permissions: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7b1ddeb..d02eb02 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,7 +4,23 @@ on: push: branches: - main + paths: + - "opensampl/**" + - "docs/**" + - "scripts/**" + - "mkdocs.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/lint.yml" pull_request: + paths: + - "opensampl/**" + - "docs/**" + - "scripts/**" + - "mkdocs.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/lint.yml" workflow_dispatch: permissions: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f40b46e..0da1ad9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,19 @@ on: push: branches: - main + paths: + - "opensampl/**" + - "tests/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/tests.yml" pull_request: + paths: + - "opensampl/**" + - "tests/**" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/tests.yml" workflow_dispatch: jobs: diff --git a/docs/guides/index.md b/docs/guides/index.md index 7265702..aaddecd 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -3,6 +3,7 @@ * [Configuration](configuration.md) * [Using the `opensampl` CLI](opensampl-cli.md) * [Using the `opensampl-server` CLI](opensampl-server.md) +* [Deploying on Kubernetes with Helm](kubernetes-helm.md) * [Value Backfill](value-backfill.md) * [Collection Guide](collection.md) * [Random Data Guide](random-data-generation.md) diff --git a/docs/guides/opensampl-server.md b/docs/guides/opensampl-server.md index 1906028..ab64c54 100644 --- a/docs/guides/opensampl-server.md +++ b/docs/guides/opensampl-server.md @@ -4,6 +4,8 @@ The `opensampl-server` CLI is a thin wrapper around the packaged Docker Compose It is useful when you want a local database, backend API, migrations container, and Grafana instance without managing the compose arguments yourself. +For cluster deployments, see [Deploying on Kubernetes with Helm](kubernetes-helm.md). + ## Prerequisites - Docker with either `docker compose` or `docker-compose` diff --git a/helm/README.md b/helm/README.md index 3247d61..e5059d4 100644 --- a/helm/README.md +++ b/helm/README.md @@ -1,30 +1,38 @@ -# Deployment -# Update --version +# openSAMPL Helm chart -helm install opensampl-test \ +Deploys openSAMPL (TimescaleDB, backend API, Grafana, and a migrations hook job) on Kubernetes. + +Full instructions: [Deploying on Kubernetes with Helm](../docs/guides/kubernetes-helm.md). + +## Quick start + +```bash +# Find the latest version in Chart.yaml or on the GHCR packages page +helm install opensampl \ oci://ghcr.io/ornl/charts/opensampl \ - --version 0.1.7 \ - --namespace opensampl-test \ - --create-namespace \ - --dry-run \ - --debug - - - -charts/ -└── opensampl/ - ├── Chart.yaml - ├── values.yaml - ├── templates/ - │ ├── _helpers.tpl - │ ├── db-statefulset.yaml - │ ├── db-service.yaml - │ ├── grafana-deployment.yaml - │ ├── grafana-service.yaml - │ ├── backend-deployment.yaml - │ ├── backend-service.yaml - │ ├── migrations-job.yaml - │ ├── pvc.yaml - │ ├── ingress.yaml (optional) - │ └── NOTES.txt - └── .helmignore \ No newline at end of file + --version \ + --namespace opensampl \ + --create-namespace +``` + +Preview first with `--dry-run --debug`; override settings with `-f my-values.yaml` +(see `values.yaml` and the cluster example in `values-cluster-overrides.yaml`). + +## Layout + +``` +helm/ +├── Chart.yaml +├── values.yaml # defaults +├── values-cluster-overrides.yaml # example overrides for the ORNL cluster +└── templates/ + ├── _helpers.tpl + ├── NOTES.txt + ├── ingress.yaml # optional + ├── backend/ # deployment, service + ├── db/ # statefulset, service, pvc, secret, configmaps + ├── grafana/ # deployment, service + └── migrations/ # post-install/post-upgrade hook job +``` + +Charts and images are published to GHCR from `main`; see `docs/maintainers/ci-cd.md`. diff --git a/helm/templates/NOTES.txt b/helm/templates/NOTES.txt index 44279c2..82d6494 100644 --- a/helm/templates/NOTES.txt +++ b/helm/templates/NOTES.txt @@ -1,7 +1,7 @@ Thank you for installing OpenSAMPL! To access your services: -- Backend: ClusterIP service "{{ include "opensampl.name" . }}-backend" on port {{ .Values.backend.port }} +- Backend: ClusterIP service "{{ include "opensampl.name" . }}-backend" on port {{ .Values.backend.service.port }} - Grafana: ClusterIP service "{{ include "opensampl.name" . }}-grafana" on port {{ .Values.grafana.port }} - Database: StatefulSet "{{ include "opensampl.name" . }}-db" with {{ .Values.db.storage }} storage diff --git a/mkdocs.yaml b/mkdocs.yaml index 4af6484..e60450c 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -13,11 +13,15 @@ nav: - Expected Table Format: guides/expected_table_format.md - Create: guides/create_probe_type.md - Server: guides/opensampl-server.md + - Kubernetes (Helm): guides/kubernetes-helm.md - Value Backfill: guides/value-backfill.md - Collect: guides/collection.md - Automatic Ingest: guides/automatic_ingest.md - NTP Extension: guides/ntp-extension.md - Random Data: guides/random-data-generation.md +- Maintainers: + - CI/CD and Releases: maintainers/ci-cd.md + - Security Remediation: maintainers/security-remediation.md - API: - index: api/index.md - cli: api/cli.md From faae363a095bdfae59c05c5a842c4cdcfc29d0d6 Mon Sep 17 00:00:00 2001 From: "Grant, Josh" Date: Wed, 4 Nov 2026 16:51:59 -0500 Subject: [PATCH 49/49] fix the migrations issue due to the grafana role not existing --- helm/templates/db/db-init-configmap.yaml | 10 +++++----- helm/templates/migrations/migrations-job.yaml | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/helm/templates/db/db-init-configmap.yaml b/helm/templates/db/db-init-configmap.yaml index 9dd7f5e..8af5303 100644 --- a/helm/templates/db/db-init-configmap.yaml +++ b/helm/templates/db/db-init-configmap.yaml @@ -8,9 +8,9 @@ data: 020_create_grafana_user.sh: | #!/bin/bash set -e - - # This script creates additional database setup for Grafana + + # Create the read-only role Grafana connects as. The migrations grant privileges to this role, + # so it must exist before they run. Mirrors opensampl/server/db/create-grafana.sh. psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL - -- Grant necessary privileges - GRANT ALL PRIVILEGES ON DATABASE ${POSTGRES_DB} TO ${POSTGRES_USER}; - EOSQL \ No newline at end of file + CREATE ROLE grafana WITH LOGIN PASSWORD '${GF_SECURITY_ADMIN_PASSWORD}'; + EOSQL diff --git a/helm/templates/migrations/migrations-job.yaml b/helm/templates/migrations/migrations-job.yaml index 0dae126..ddd1aec 100644 --- a/helm/templates/migrations/migrations-job.yaml +++ b/helm/templates/migrations/migrations-job.yaml @@ -13,6 +13,9 @@ metadata: "helm.sh/hook": post-install,post-upgrade "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: + # fail reasonably fast (the default of 6 retries with backoff takes ~8 minutes) so a broken + # migration is reported promptly + backoffLimit: 2 template: spec: initContainers: