Adding garage (S3 like) and zot (private docker registry)

This commit is contained in:
Adrien
2026-04-06 10:53:58 +00:00
parent d2e050f1f1
commit e05f1c0de6
22 changed files with 1446 additions and 1 deletions
+91
View File
@@ -0,0 +1,91 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "garage.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "garage.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create the name of the rpc secret
*/}}
{{- define "garage.rpcSecretName" -}}
{{- .Values.garage.existingRpcSecret | default (printf "%s-rpc-secret" (include "garage.fullname" .)) -}}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "garage.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "garage.labels" -}}
helm.sh/chart: {{ include "garage.chart" . }}
{{ include "garage.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- with .Values.commonLabels }}
{{- toYaml . | nindent 0 }}
{{- end }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "garage.selectorLabels" -}}
app.kubernetes.io/name: {{ include "garage.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "garage.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "garage.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
Returns given number of random Hex characters.
In practice, it generates up to 100 randAlphaNum strings
that are filtered from non-hex characters and augmented
to the resulting string that is finally trimmed down.
*/}}
{{- define "jupyterhub.randHex" -}}
{{- $result := "" }}
{{- range $i := until 100 }}
{{- if lt (len $result) . }}
{{- $rand_list := randAlphaNum . | splitList "" -}}
{{- $reduced_list := without $rand_list "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" }}
{{- $rand_string := join "" $reduced_list }}
{{- $result = print $result $rand_string -}}
{{- end }}
{{- end }}
{{- $result | trunc . }}
{{- end }}
+28
View File
@@ -0,0 +1,28 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
labels:
{{- include "garage.labels" . | nindent 4 }}
rules:
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "list", "watch", "create", "patch"]
- apiGroups: ["deuxfleurs.fr"]
resources: ["garagenodes"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: allow-crds-for-{{ .Release.Namespace }}-{{ .Release.Name }}
labels:
{{- include "garage.labels" . | nindent 4 }}
subjects:
- kind: ServiceAccount
name: {{ include "garage.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
apiGroup: rbac.authorization.k8s.io
+62
View File
@@ -0,0 +1,62 @@
{{- if not .Values.garage.existingConfigMap }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "garage.fullname" . }}-config
data:
garage.toml: |-
{{- if .Values.garage.garageTomlString }}
{{- tpl (index (index .Values.garage) "garageTomlString") $ | nindent 4 }}
{{- else }}
metadata_dir = "/mnt/meta"
data_dir = "/mnt/data"
db_engine = "{{ .Values.garage.dbEngine }}"
block_size = "{{ .Values.garage.blockSize }}"
replication_factor = {{ .Values.garage.replicationFactor }}
consistency_mode = "{{ .Values.garage.consistencyMode }}"
compression_level = {{ .Values.garage.compressionLevel }}
{{- if .Values.garage.metadataAutoSnapshotInterval }}
metadata_auto_snapshot_interval = {{ .Values.garage.metadataAutoSnapshotInterval | quote }}
{{- end }}
rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
# rpc_secret will be populated by the init container from a k8s secret object
rpc_secret = "__RPC_SECRET_REPLACE__"
bootstrap_peers = [
{{- range $index, $peer := .Values.garage.bootstrapPeers }}
{{- if $index}}, {{ end }}{{ $peer | quote }}
{{ end }}
]
{{- if .Values.garage.additionalTopLevelConfig }}
{{ .Values.garage.additionalTopLevelConfig | nindent 4 }}
{{- end }}
[kubernetes_discovery]
namespace = "{{ .Release.Namespace }}"
service_name = "{{ include "garage.fullname" . }}"
skip_crd = {{ .Values.garage.kubernetesSkipCrd }}
[s3_api]
s3_region = "{{ .Values.garage.s3.api.region }}"
api_bind_addr = "[::]:3900"
root_domain = "{{ .Values.garage.s3.api.rootDomain }}"
[s3_web]
bind_addr = "[::]:3902"
root_domain = "{{ .Values.garage.s3.web.rootDomain }}"
index = "{{ .Values.garage.s3.web.index }}"
[admin]
api_bind_addr = "[::]:3903"
{{- if .Values.monitoring.tracing.sink }}
trace_sink = "{{ .Values.monitoring.tracing.sink }}"
{{- end }}
{{- end }}
{{- end }}
+129
View File
@@ -0,0 +1,129 @@
{{- if .Values.ingress.s3.api.enabled -}}
{{- $fullName := include "garage.fullname" . -}}
{{- $svcPort := .Values.service.s3.api.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.api.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}-s3-api
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.ingress.s3.api.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.ingress.s3.api.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.s3.api.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.s3.api.className }}
{{- end }}
{{- if .Values.ingress.s3.api.tls }}
tls:
{{- range .Values.ingress.s3.api.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.s3.api.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
---
{{- if .Values.ingress.s3.web.enabled -}}
{{- $fullName := include "garage.fullname" . -}}
{{- $svcPort := .Values.service.s3.web.port -}}
{{- if and .Values.ingress.s3.web.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.web.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}-s3-web
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.ingress.s3.web.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- with .Values.ingress.s3.web.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.s3.web.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.s3.web.className }}
{{- end }}
{{- if .Values.ingress.s3.web.tls }}
tls:
{{- range .Values.ingress.s3.web.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.s3.web.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
+16
View File
@@ -0,0 +1,16 @@
{{- if not .Values.garage.existingRpcSecret }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "garage.rpcSecretName" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
type: Opaque
data:
{{/* retrieve the secret data using lookup function and when not exists, return an empty dictionary / map as result */}}
{{- $prevSecret := (lookup "v1" "Secret" .Release.Namespace (include "garage.rpcSecretName" .)) | default dict }}
{{- $prevSecretData := $prevSecret.data | default dict }}
{{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
{{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}
{{- end }}
+22
View File
@@ -0,0 +1,22 @@
{{- if eq .Values.deployment.kind "StatefulSet" -}}
apiVersion: v1
kind: Service
metadata:
name: {{ include "garage.fullname" . }}-headless
labels:
{{- include "garage.labels" . | nindent 4 }}
spec:
type: ClusterIP
clusterIP: None
ports:
- port: {{ .Values.service.s3.api.port }}
targetPort: 3900
protocol: TCP
name: s3-api
- port: {{ .Values.service.s3.web.port }}
targetPort: 3902
protocol: TCP
name: s3-web
selector:
{{- include "garage.selectorLabels" . | nindent 4 }}
{{- end }}
+44
View File
@@ -0,0 +1,44 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "garage.fullname" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.service.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.s3.api.port }}
targetPort: 3900
protocol: TCP
name: s3-api
- port: {{ .Values.service.s3.web.port }}
targetPort: 3902
protocol: TCP
name: s3-web
selector:
{{- include "garage.selectorLabels" . | nindent 4 }}
{{- if .Values.monitoring.metrics.enabled }}
---
apiVersion: v1
kind: Service
metadata:
name: {{ include "garage.fullname" . }}-metrics
labels:
{{- include "garage.labels" . | nindent 4 }}
annotations:
prometheus.io/scrape: "true"
spec:
type: ClusterIP
clusterIP: None
ports:
- port: 3903
targetPort: 3903
protocol: TCP
name: metrics
selector:
{{- include "garage.selectorLabels" . | nindent 4 }}
{{- end }}
+12
View File
@@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "garage.serviceAccountName" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
+44
View File
@@ -0,0 +1,44 @@
{{- if .Values.monitoring.metrics.serviceMonitor.enabled }}
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ include "garage.fullname" . }}
{{- if .Values.monitoring.metrics.serviceMonitor.namespace }}
namespace: {{ tpl .Values.monitoring.metrics.serviceMonitor.namespace . }}
{{- else }}
namespace: {{ .Release.Namespace }}
{{- end }}
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.monitoring.metrics.serviceMonitor.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
endpoints:
- port: metrics
{{- with .Values.monitoring.metrics.serviceMonitor.interval }}
interval: {{ . }}
{{- end }}
{{- with .Values.monitoring.metrics.serviceMonitor.scrapeTimeout }}
scrapeTimeout: {{ . }}
{{- end }}
honorLabels: true
path: {{ .Values.monitoring.metrics.serviceMonitor.path }}
scheme: {{ .Values.monitoring.metrics.serviceMonitor.scheme }}
{{- with .Values.monitoring.metrics.serviceMonitor.tlsConfig }}
tlsConfig:
{{- toYaml . | nindent 6 }}
{{- end }}
{{- with .Values.monitoring.metrics.serviceMonitor.relabelings }}
relabelings:
{{- toYaml . | nindent 6 }}
{{- end }}
jobLabel: "{{ .Release.Name }}"
selector:
matchLabels:
{{- include "garage.selectorLabels" . | nindent 6 }}
namespaceSelector:
matchNames:
- {{ .Release.Namespace }}
{{- end }}
+154
View File
@@ -0,0 +1,154 @@
apiVersion: apps/v1
kind: {{ .Values.deployment.kind }}
metadata:
name: {{ include "garage.fullname" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
{{- include "garage.selectorLabels" . | nindent 6 }}
{{- if eq .Values.deployment.kind "StatefulSet" }}
replicas: {{ .Values.deployment.replicaCount }}
serviceName: {{ include "garage.fullname" . }}-headless
podManagementPolicy: {{ .Values.deployment.podManagementPolicy }}
{{- end }}
template:
metadata:
annotations:
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
{{- with .Values.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "garage.labels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "garage.serviceAccountName" . }}
{{- with .Values.priorityClassName }}
priorityClassName: {{ . }}
{{- end }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
initContainers:
# Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder
- name: {{ .Chart.Name }}-init
image: "{{ .Values.initImage.repository }}:{{ .Values.initImage.tag }}"
imagePullPolicy: {{ .Values.initImage.pullPolicy }}
command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"]
env:
- name: RPC_SECRET
valueFrom:
secretKeyRef:
name: {{ include "garage.rpcSecretName" . }}
key: rpcSecret
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
volumeMounts:
- name: configmap
mountPath: /mnt/garage.toml
subPath: garage.toml
- name: etc
mountPath: /mnt/etc
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- containerPort: 3900
name: s3-api
- containerPort: 3902
name: web-api
- containerPort: 3903
name: admin
{{- with .Values.environment }}
env:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
- name: meta
mountPath: /mnt/meta
- name: data
mountPath: /mnt/data
- name: etc
mountPath: /etc/garage.toml
subPath: garage.toml
{{- with .Values.extraVolumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.livenessProbe }}
livenessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.readinessProbe }}
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumes:
- name: configmap
configMap:
name: {{ include "garage.fullname" . }}-config
- name: etc
emptyDir: {}
{{- if .Values.persistence.enabled }}
{{- if eq .Values.deployment.kind "DaemonSet" }}
- name: meta
hostPath:
path: {{ .Values.persistence.meta.hostPath }}
type: DirectoryOrCreate
- name: data
hostPath:
path: {{ .Values.persistence.data.hostPath }}
type: DirectoryOrCreate
{{- end }}
{{- else }}
- name: meta
emptyDir: {}
- name: data
emptyDir: {}
{{- end }}
{{- with .Values.extraVolumes }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if and .Values.persistence.enabled (eq .Values.deployment.kind "StatefulSet") }}
volumeClaimTemplates:
- metadata:
name: meta
spec:
accessModes: [ "ReadWriteOnce" ]
{{- if hasKey .Values.persistence.meta "storageClass" }}
storageClassName: {{ .Values.persistence.meta.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.meta.size | quote }}
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
{{- if hasKey .Values.persistence.data "storageClass" }}
storageClassName: {{ .Values.persistence.data.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.data.size | quote }}
{{- end }}