SMFI-15: helm chart media finder (#124)

* Startup probe path configurable * Support new configuration files in media-finder * Support new configuration files in media-finder * Add option for init containers * Final fixes --------- Co-authored-by: Stefan Morgenstern <[email protected]>
subshell · Nov 5, 2024 · 5948933 · 5948933
1 parent 8b69428
commit 5948933
Show file tree

Hide file tree

Showing 5 changed files with 202 additions and 12 deletions.
diff --git a/charts/sophora-media-finder/Chart.yaml b/charts/sophora-media-finder/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: sophora-media-finder
 description: A Helm chart for Sophora Media Finder
 type: application
-version: 0.1.1
+version: 0.2.0
 appVersion: "5.x"
diff --git a/charts/sophora-media-finder/templates/_helpers.tpl b/charts/sophora-media-finder/templates/_helpers.tpl
@@ -49,3 +49,8 @@ Selector labels
 app.kubernetes.io/name: {{ include "sophora-media-finder.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
+
+{{- define "sophora-media-finder.utils.joinListWithSpace" -}}
+{{- $local := dict "first" true -}}
+{{- range $k, $v := . -}}{{- if not $local.first -}}{{- " " -}}{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}}
+{{- end -}}
diff --git a/charts/sophora-media-finder/templates/configmap.yaml b/charts/sophora-media-finder/templates/configmap.yaml
@@ -5,4 +5,7 @@ metadata:
   labels:
   {{- include "sophora-media-finder.labels" . | nindent 4 }}
 data:
-  application.yaml: |- {{ toYaml (required "A valid application.yaml config is required" .Values.configuration) | nindent 4 }}
+  application.yaml: |- {{ toYaml (required "A valid application.yaml config is required" .Values.mediaFinder.configuration) | nindent 4 }}
+  loader.properties: |- {{ toString (required "A valid loader.properties config is required" .Values.mediaFinder.loaderProperties) | nindent 4 }}
+  logback-spring.xml: |- {{ toString (required "A valid logback-spring.xml config is required" .Values.mediaFinder.logbackSpring) | nindent 4 }}
+  media-finder-executable.conf: |- {{ toString .Values.mediaFinder.mediaFinderExecutable | nindent 4 }}
diff --git a/charts/sophora-media-finder/templates/deployment.yaml b/charts/sophora-media-finder/templates/deployment.yaml
@@ -18,6 +18,62 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      initContainers:
+        {{ if eq .Values.mediaFinder.plugins.source "s3" }}
+        - name: plugins-downloader-via-s3
+          image: {{ .Values.downloadViaS3Image.repository }}:{{ .Values.downloadViaS3Image.tag }}
+          imagePullPolicy: {{ .Values.downloadViaS3Image.pullPolicy }}
+          volumeMounts:
+            - mountPath: /plugins
+              name: plugins
+          {{ with .Values.mediaFinder.plugins.downloadFromS3 }}
+          env:
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  key: {{ .accessKeyIdKey }}
+                  name: {{ .secretName }}
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  key: {{ .secretAccessKeyKey }}
+                  name: {{ .secretName }}
+            - name: S3_ENDPOINT
+              value: {{ .s3Endpoint }}
+            - name: S3_NAME
+              value: {{ .bucketName }}
+            - name: S3_FILE_PATHS
+              value: {{ include "sophora-media-finder.utils.joinListWithSpace" .jarPaths | quote }}
+          command: ["/bin/sh"]
+          args:
+            - "-c"
+            - |-
+              mkdir dl && cd dl || exit;
+              if [ -n "$S3_FILE_PATHS" ]; then
+                for S3_FILE_PATH in $S3_FILE_PATHS; do
+                  aws --endpoint="$S3_ENDPOINT" s3 cp "s3://$S3_NAME$S3_FILE_PATH" ./
+                  echo "Downloaded $S3_FILE_PATH"
+                done
+              fi
+              mv *.jar /plugins
+          {{- end }}
+        {{- else if eq .Values.mediaFinder.plugins.source "sidecar" }}
+        - name: plugins-downloader-via-sidecar
+          image: {{ .Values.mediaFinder.plugins.copyFromSidecarImage.repository }}:{{ .Values.mediaFinder.plugins.copyFromSidecarImage.tag }}
+          imagePullPolicy: {{ .Values.mediaFinder.plugins.copyFromSidecarImage.pullPolicy }}
+          volumeMounts:
+            - mountPath: /plugins
+              name: plugins
+          env:
+            - name: JAR_FOLDER
+              value: {{ .Values.mediaFinder.plugins.copyFromSidecarImage.jarFolder }}
+          command: ["/bin/sh"]
+          args:
+            - "-c"
+            - |-
+              cd $JAR_FOLDER
+              mv *.jar /plugins
+        {{- end }}
       containers:
         - name: {{ .Chart.Name }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -26,13 +82,34 @@ spec:
             - name: API_KEY
               valueFrom:
                 secretKeyRef:
-                  name: {{ .Values.apiKey.secretName }}
+                  name: {{ required "A valid secret name must be provided in .Values.apiKey.secretName" .Values.apiKey.secretName | quote }}
                   key: {{ .Values.apiKey.secretFieldName }}
                   optional: false
+            - name: SOPHORA_CLIENT_SERVERCONNECTION_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.sophora.authentication.secret.name | quote }}
+                  key: {{ .Values.sophora.authentication.secret.usernameKey | quote }}
+            - name: SOPHORA_CLIENT_SERVERCONNECTION_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.sophora.authentication.secret.name | quote }}
+                  key: {{ .Values.sophora.authentication.secret.passwordKey | quote }}
           volumeMounts:
             - name: config
-              mountPath: /config
-              readOnly: true
+              subPath: application.yaml
+              mountPath: /sophora/application.yaml
+            - name: config
+              subPath: loader.properties
+              mountPath: /sophora/loader.properties
+            - name: config
+              subPath: logback-spring.xml
+              mountPath: /sophora/logback-spring.xml
+            - name: config
+              subPath: media-finder-executable.conf
+              mountPath: /sophora/media-finder-executable.conf
+            - name: plugins
+              mountPath: /sophora/plugins
           ports:
             - name: http
               containerPort: 8080
@@ -70,4 +147,6 @@ spec:
       volumes:
         - name: config
           configMap:
-            name: {{ include "sophora-media-finder.fullname" . }}
+            name: {{ include "sophora-media-finder.fullname" . }}
+        - name: plugins
+          emptyDir: {}
diff --git a/charts/sophora-media-finder/values.yaml b/charts/sophora-media-finder/values.yaml
@@ -6,6 +6,11 @@ image:
   # Overrides the image tag whose default is the chart appVersion.
   tag: "5.x"
 
+downloadViaS3Image:
+  repository: "docker.subshell.com/misc/aws-cli-tools"
+  tag: "0.0.2"
+  pullPolicy: IfNotPresent
+
 imagePullSecrets: []
 
 nameOverride: ""
@@ -47,9 +52,107 @@ apiKey:
   secretName: ""
   secretFieldName: api-key
 
-configuration:
-  proxy:
-    api.base.url: ""
-    image.service.url: ""
-    auth.basic.token: ${API_KEY}
-    maxInMemorySizeInMB: 10
+sophora:
+  authentication:
+    secret:
+      name: ""
+      usernameKey: username
+      passwordKey: password
+
+mediaFinder:
+  configuration:
+    proxy:
+      api.base.url: ""
+      image.service.url: ""
+      auth.basic.token: ${API_KEY}
+      maxInMemorySizeInMB: 10
+
+    spring:
+      application:
+        spring-additional-base-packages: ""
+
+    # Connection to the Sophora server.
+    sophora:
+      client:
+        server-connection:
+          urls: []
+          username: #in secret
+          password: #in secret
+          retries: 10
+          retry-interval: 10
+  plugins:
+    source: # "s3" or "sidecar". leave blank for none.
+    downloadFromS3:
+      secretName:
+      s3Endpoint: "https://storage.googleapis.com"
+      bucketName:
+      jarPaths: []
+      secretAccessKeyKey: "secretAccessKey"
+      accessKeyIdKey: "accessKeyId"
+    copyFromSidecarImage:
+      repository:
+      tag:
+      pullPolicy: IfNotPresent
+      jarFolder: # the folder containing the jar file(s)
+
+  mediaFinderExecutable:
+    JAVA_OPTS="-Xmx1G"
+
+  loaderProperties:
+    loader.path=plugins
+
+  logbackSpring: |
+    <?xml version="1.0" encoding="UTF-8"?>
+    <configuration scan="true" scanPeriod="60 seconds">
+      <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+      
+      <!-- Logger -->
+      <logger name="com.subshell.sophora" level="INFO"/>
+      <logger name="com.subshell.media.finder" level="INFO"/>
+      <logger name="org.springframework.boot" level="INFO"/>
+      
+      <!-- Default logging configuration -->
+      <springProfile name="!json-logs">
+        <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+          <encoder>
+            <pattern>[%d{ISO8601}, %-5level] [%-35.35thread] [%-36logger{36}:%4line] %msg%n</pattern>
+          </encoder>
+        </appender>
+        <appender name="logfile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+          <File>logs/media-finder.log</File>
+          <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>logs/media-finder.%d{yyyy-MM-dd}.log.gz</fileNamePattern>
+            <maxHistory>90</maxHistory>
+          </rollingPolicy>
+          <encoder>
+            <charset>UTF-8</charset>
+            <pattern>[%d{ISO8601}, %-5p] [%-35t] [%-40logger{40}:%-3L] %m%n</pattern>
+          </encoder>
+        </appender>
+        
+        <root level="WARN">
+          <appender-ref ref="console"/>
+          <appender-ref ref="logfile"/>
+        </root>
+      </springProfile>
+        
+      <!-- JSON logging configuration -->
+      <springProfile name="json-logs">
+        <appender name="jsonConsole" class="ch.qos.logback.core.ConsoleAppender">
+          <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
+        </appender>
+        <appender name="jsonLogfile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+          <File>logs/media-finder.log</File>
+          <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>logs/media-finder.%d{yyyy-MM-dd}.log.gz</fileNamePattern>
+            <maxHistory>90</maxHistory>
+          </rollingPolicy>
+          <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
+        </appender>
+        <root level="WARN">
+          <appender-ref ref="jsonConsole"/>
+          <appender-ref ref="jsonLogfile"/>
+        </root>
+      </springProfile>
+  
+    </configuration>