From 70e40c08d69050f499d9de641a94946aed9b4b0e Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 30 Oct 2024 15:42:25 +0530 Subject: [PATCH 01/18] add trimmed down dockerfile --- base/Docker-refresh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 base/Docker-refresh diff --git a/base/Docker-refresh b/base/Docker-refresh new file mode 100644 index 0000000000..ae953d7545 --- /dev/null +++ b/base/Docker-refresh @@ -0,0 +1,32 @@ +ARG MICRODIR=/microdir +ARG PACKAGES_TO_INSTALL="temurin-17-jre shadow-utils" + +FROM registry.access.redhat.com/ubi8 AS BUILD +ARG MICRODIR +ARG PACKAGES_TO_INSTALL +RUN mkdir ${MICRODIR} + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo + +RUN yum --nodocs install -y --setopt=install_weak_deps=False ${PACKAGES_TO_INSTALL} + +RUN yum clean all + +FROM registry.access.redhat.com/ubi8-micro AS REFRESH +ARG MICRODIR +COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ ${MICRODIR}/jre/ + +ENV PATH="${MICRODIR}/jre/bin:${PATH}" + +RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ + mkdir -p /home/appuser && \ + chown 1001:1001 /home/appuser + +USER appuser +WORKDIR /home/appuser From 2541a910ba8756f9fd54d3b098119f830d63f701 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 12:11:12 +0530 Subject: [PATCH 02/18] add docker refresh module --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index ae82f30b12..f568f634bb 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,7 @@ utility-belt docker-utils + refresh base base-lite jmxterm @@ -34,6 +35,7 @@ 7.9.0-0 8.10-1086 + 8.10-13 3.0.9 From 409bb4c7846f73e2cf24929a56633aa4f69ec4d1 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 12:11:36 +0530 Subject: [PATCH 03/18] copy binaries from cp-base-new --- .../include/etc/confluent/docker/bash-config | 23 +++++++++++++++ .../etc/confluent/docker/mesos-setup.sh | 27 ++++++++++++++++++ .../include/etc/cp-base-new/log4j.properties | 28 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 refresh/include/etc/confluent/docker/bash-config create mode 100644 refresh/include/etc/confluent/docker/mesos-setup.sh create mode 100644 refresh/include/etc/cp-base-new/log4j.properties diff --git a/refresh/include/etc/confluent/docker/bash-config b/refresh/include/etc/confluent/docker/bash-config new file mode 100644 index 0000000000..43c08d9b50 --- /dev/null +++ b/refresh/include/etc/confluent/docker/bash-config @@ -0,0 +1,23 @@ +# +# Copyright 2018 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o nounset \ + -o errexit + +# Trace may expose passwords/credentials by printing them to stdout, so turn on with care. +if [ "${TRACE:-}" == "true" ]; then + set -o verbose \ + -o xtrace +fi diff --git a/refresh/include/etc/confluent/docker/mesos-setup.sh b/refresh/include/etc/confluent/docker/mesos-setup.sh new file mode 100644 index 0000000000..b3874daf6d --- /dev/null +++ b/refresh/include/etc/confluent/docker/mesos-setup.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set +o nounset + +if [ -z $SKIP_MESOS_AUTO_SETUP ]; then + if [ -n $MESOS_SANDBOX ] && [ -e $MESOS_SANDBOX/.ssl/scheduler.crt ] && [ -e $MESOS_SANDBOX/.ssl/scheduler.key ]; then + echo "Entering Mesos auto setup for Java SSL truststore. You should not see this if you are not on mesos ..." + + openssl pkcs12 -export -in $MESOS_SANDBOX/.ssl/scheduler.crt -inkey $MESOS_SANDBOX/.ssl/scheduler.key \ + -out /tmp/keypair.p12 -name keypair \ + -CAfile $MESOS_SANDBOX/.ssl/ca-bundle.crt -caname root -passout pass:export + + keytool -importkeystore \ + -deststorepass changeit -destkeypass changeit -destkeystore /tmp/kafka-keystore.jks \ + -srckeystore /tmp/keypair.p12 -srcstoretype PKCS12 -srcstorepass export \ + -alias keypair + + keytool -import \ + -trustcacerts \ + -alias root \ + -file $MESOS_SANDBOX/.ssl/ca-bundle.crt \ + -storepass changeit \ + -keystore /tmp/kafka-truststore.jks -noprompt + fi +fi + +set -o nounset diff --git a/refresh/include/etc/cp-base-new/log4j.properties b/refresh/include/etc/cp-base-new/log4j.properties new file mode 100644 index 0000000000..40fd0b7329 --- /dev/null +++ b/refresh/include/etc/cp-base-new/log4j.properties @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=OFF + +# Only log errors from Kafka and ZKClient +log4j.logger.org.apache.kafka=ERROR +log4j.logger.org.I0Itec.zkclient.ZkClient=ERROR + +# Log informational messages from the CLI and Zookeeper +log4j.logger.io.confluent.admin.utils=INFO, stderr +log4j.logger.org.apache.zookeeper=INFO, stderr +# STDERR Appender +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.Target=System.err +log4j.appender.stderr.layout.ConversionPattern=%m%n From 169acaec1a539da75c5ca5ce49900c978712f84d Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 12:12:13 +0530 Subject: [PATCH 04/18] create dockerfile with jre and ubi-micro --- refresh/Dockerfile.ubi8 | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 refresh/Dockerfile.ubi8 diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 new file mode 100644 index 0000000000..dfecc6862a --- /dev/null +++ b/refresh/Dockerfile.ubi8 @@ -0,0 +1,53 @@ +ARG MICRODIR=/microdir +ARG UBI_MICRO_VERSION=8.10-13 +ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" +FROM registry.access.redhat.com/ubi8 AS BUILD + +RUN printf "[temurin-jre] \n\ +name=temurin-jre \n\ +baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch \n\ +enabled=1 \n\ +gpgcheck=1 \n\ +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public \n\ +" > /etc/yum.repos.d/adoptium.repo +RUN echo "installing temurin-17-jre:${TEMURIN_JDK_VERSION}" +RUN yum --nodocs install -y --setopt=install_weak_deps=False temurin-17-jre + +RUN yum clean all + +FROM registry.access.redhat.com/ubi8-micro:${UBI_MICRO_VERSION} AS REFRESH +ARG PROJECT_VERSION +ARG ARTIFACT_ID + +# Remember where we came from +LABEL io.confluent.docker.git.repo="confluentinc/common-docker" + +ARG GIT_COMMIT +LABEL io.confluent.docker.git.id=$GIT_COMMIT + +ARG BUILD_NUMBER=-1 +LABEL io.confluent.docker.build.number=$BUILD_NUMBER + +LABEL maintainer="tools@confluent.io" +LABEL vendor="Confluent" +LABEL version=$GIT_COMMIT +LABEL release=$PROJECT_VERSION +LABEL name=$ARTIFACT_ID +LABEL summary="Common base image for new Confluent lightweight Docker images." +LABEL description="Common base image for Confluent lightweight Docker images." +LABEL io.confluent.docker=true +# This affects how strings in Java class files are interpreted. We want UTF-8 and this is the only locale in the +# base image that supports it +ENV LANG="C.UTF-8" + +#ARG MICRODIR +COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ . + +ENV PATH="jre/bin:${PATH}" + +RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ + mkdir -p /home/appuser && \ + chown 1001:1001 /home/appuser + +USER appuser +WORKDIR /home/appuser From f3c7fbf1160586692bb1df0704e172274bd5a658 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 12:12:30 +0530 Subject: [PATCH 05/18] add mvn build --- refresh/pom.xml | 120 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 refresh/pom.xml diff --git a/refresh/pom.xml b/refresh/pom.xml new file mode 100644 index 0000000000..958459fb17 --- /dev/null +++ b/refresh/pom.xml @@ -0,0 +1,120 @@ + + + + + 4.0.0 + + + io.confluent + common-docker + 7.9.0-0 + + + pom + + cp-base-refresh + + Refreshed Base for Confluent Docker images + ${project.artifactId} + + + false + false + true + + + + + io.confluent + utility-belt + ${io.confluent.common-docker.version} + + + junit + junit + ${junit.version} + test + + + + + + + maven-resources-plugin + ${maven-resources-plugin.version} + + + copy-resources + compile + + copy-resources + + + true + target + + + ${basedir}/../docker-utils/target + + docker-utils-${CONFLUENT_VERSION}-jar-with-dependencies.jar + + + + + + + + + com.spotify + dockerfile-maven-plugin + + + ${ubi.image.version} + ${ubi.micro.image.version} + -${ubi.temurin.jdk.version} + ${docker.skip-security-update-check} + ${golang.version} + + + + + io.fabric8 + docker-maven-plugin + 0.43.4 + + + + + + ${ubi.image.version} + ${ubi.micro.image.version} + -${ubi.temurin.jdk.version} + + ${docker.skip-security-update-check} + + ${golang.version} + + + + + + + + + From 529ac038faca92326408b18e7c7c633046517584 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 15:24:40 +0530 Subject: [PATCH 06/18] minor fix to env var --- refresh/Dockerfile.ubi8 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index dfecc6862a..218275714c 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -41,9 +41,9 @@ LABEL io.confluent.docker=true ENV LANG="C.UTF-8" #ARG MICRODIR -COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ . +COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ usr/lib -ENV PATH="jre/bin:${PATH}" +ENV PATH="/usr/lib/bin:${PATH}" RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ mkdir -p /home/appuser && \ From b104644b8c455182a32912a5c37b8d1a3ed1cdb4 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 15:41:54 +0530 Subject: [PATCH 07/18] add remaining files --- refresh/requirements.txt | 1 + refresh/setup.py | 23 ++++++++++++++++++++++ refresh/tox.ini | 41 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 refresh/requirements.txt create mode 100644 refresh/setup.py create mode 100644 refresh/tox.ini diff --git a/refresh/requirements.txt b/refresh/requirements.txt new file mode 100644 index 0000000000..879b1e1d13 --- /dev/null +++ b/refresh/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/confluentinc/confluent-docker-utils@v0.0.96 diff --git a/refresh/setup.py b/refresh/setup.py new file mode 100644 index 0000000000..9c69fc03cc --- /dev/null +++ b/refresh/setup.py @@ -0,0 +1,23 @@ +from setuptools import setup + + +setup( + name='common-tests', + version='4.1.0', + + author="Confluent, Inc.", + + description='Docker image tests', + + url="https://github.com/confluentinc/common", + + dependency_links=open('requirements.txt').read().split("\n"), + + packages=['test'], + + include_package_data=True, + + python_requires='>=2.7', + setup_requires=['setuptools-git'], + +) diff --git a/refresh/tox.ini b/refresh/tox.ini new file mode 100644 index 0000000000..af5b6fe10e --- /dev/null +++ b/refresh/tox.ini @@ -0,0 +1,41 @@ +[tox] +envlist = test +toxworkdir = /var/tmp + +[testenv] +deps = + -rrequirements.txt + flake8 + pytest == 4.6.4 + pytest-xdist == 1.29.0 + pytest-cov == 2.7.1 + sphinx!=1.2b2,<2.0.0 +install_command = pip install -U {packages} +recreate = True +skipsdist = True +usedevelop = True +setenv = + PIP_PROCESS_DEPENDENCY_LINKS=1 + PIP_DEFAULT_TIMEOUT=60 + ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future +basepython = python +envdir = {toxworkdir}/confluent + +[testenv:test] +commands = + py.test --color=no {env:PYTESTARGS:} test + +[testenv:style] +commands = + flake8 --config tox.ini + +[testenv:cover] +commands = + py.test {env:PYTESTARGS:} --cov . --cov-report=xml --cov-report=html --cov-report=term test + +[flake8] +ignore = E111,E121,W292,E123,E226 +max-line-length = 160 + +[pytest] +addopts = -n 1 From d1014b9f4ddee310bffb436f3eb20d7a2e0d8a44 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 6 Nov 2024 16:00:31 +0530 Subject: [PATCH 08/18] add tests --- refresh/test/test_refresh_image.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 refresh/test/test_refresh_image.py diff --git a/refresh/test/test_refresh_image.py b/refresh/test/test_refresh_image.py new file mode 100644 index 0000000000..55653f3252 --- /dev/null +++ b/refresh/test/test_refresh_image.py @@ -0,0 +1,21 @@ +import os +import unittest + +import confluent.docker_utils as utils + + +class BaseRefreshImageTest(unittest.TestCase): + + def setUp(self): + self.image = "{0}confluentinc/cp-base-refresh:{1}".format(os.environ["DOCKER_REGISTRY"], os.environ["DOCKER_TAG"]) + + def test_image_build(self): + self.assertTrue(utils.image_exists(self.image)) + + def test_jre_17_installed(self): + jre_cmd = "java --version" + result = utils.run_docker_command(image=self.image, command=jre_cmd) + self.assertTrue(b'17' in result) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 241633b55418ab739a128c94e595fd5702bad9d0 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Tue, 12 Nov 2024 15:35:41 +0530 Subject: [PATCH 09/18] add tests for ub --- refresh/test/test_refresh_image.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/refresh/test/test_refresh_image.py b/refresh/test/test_refresh_image.py index 55653f3252..94ae271f7d 100644 --- a/refresh/test/test_refresh_image.py +++ b/refresh/test/test_refresh_image.py @@ -17,5 +17,12 @@ def test_jre_17_installed(self): result = utils.run_docker_command(image=self.image, command=jre_cmd) self.assertTrue(b'17' in result) + def test_ub_exists(self): + self.assertTrue(utils.path_exists_in_image(self.image, "/usr/bin/ub")) + + def test_ub_runnable(self): + ub_cmd = "bash -c '/usr/bin/ub -h'" + self.assertTrue(b"utility commands" in utils.run_docker_command(image=self.image, command=ub_cmd)) + if __name__ == '__main__': unittest.main() \ No newline at end of file From da30d3feedfd9af51e147a131fd9c0979bb7f7e0 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Tue, 12 Nov 2024 15:38:48 +0530 Subject: [PATCH 10/18] add ub tests --- refresh/Dockerfile.ubi8 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index 218275714c..b350a2f068 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -1,6 +1,9 @@ ARG MICRODIR=/microdir ARG UBI_MICRO_VERSION=8.10-13 ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" +ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" +ARG DOCKER_UPSTREAM_TAG="7.9.x-latest-ubi8" + FROM registry.access.redhat.com/ubi8 AS BUILD RUN printf "[temurin-jre] \n\ @@ -15,6 +18,8 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False temurin-17-jre RUN yum clean all +FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-lite:${DOCKER_UPSTREAM_TAG} AS BASE-LITE + FROM registry.access.redhat.com/ubi8-micro:${UBI_MICRO_VERSION} AS REFRESH ARG PROJECT_VERSION ARG ARTIFACT_ID @@ -42,6 +47,7 @@ ENV LANG="C.UTF-8" #ARG MICRODIR COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ usr/lib +COPY --from=BASE-LITE /usr/bin /usr/bin ENV PATH="/usr/lib/bin:${PATH}" From ea65844471c18580222ba5998e86d9f382739122 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Tue, 12 Nov 2024 15:52:28 +0530 Subject: [PATCH 11/18] add cp-base-refresh to deploy pipeline --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 7d4149c8a6..d809e2f240 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -72,7 +72,7 @@ global_job_config: - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY - export LATEST_TAG=$BRANCH_TAG-latest - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - - export DOCKER_REPOS="confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" + - export DOCKER_REPOS="confluentinc/cp-base-refresh confluentinc/cp-base-new confluentinc/cp-base-lite confluentinc/cp-jmxterm" - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 From 42421f462b410d843c5e1f32e2e45702d3411390 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 13 Nov 2024 11:10:15 +0530 Subject: [PATCH 12/18] remove tag --- refresh/Dockerfile.ubi8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index b350a2f068..33c534fe63 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -2,7 +2,7 @@ ARG MICRODIR=/microdir ARG UBI_MICRO_VERSION=8.10-13 ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" -ARG DOCKER_UPSTREAM_TAG="7.9.x-latest-ubi8" +ARG DOCKER_UPSTREAM_TAG FROM registry.access.redhat.com/ubi8 AS BUILD From 5b32376889b706f324e7f00f3af958acb5917ef0 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 13 Nov 2024 11:48:05 +0530 Subject: [PATCH 13/18] remove tag --- refresh/Dockerfile.ubi8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index 33c534fe63..e98592cf48 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -18,7 +18,7 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False temurin-17-jre RUN yum clean all -FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-lite:${DOCKER_UPSTREAM_TAG} AS BASE-LITE +FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-lite AS BASE-LITE FROM registry.access.redhat.com/ubi8-micro:${UBI_MICRO_VERSION} AS REFRESH ARG PROJECT_VERSION From bbaff1bdbd4afdd72b5ed44cda1f29beaaef111a Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 13 Nov 2024 14:37:38 +0530 Subject: [PATCH 14/18] use cp-base-lite --- refresh/Dockerfile.ubi8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index e98592cf48..6a0a7758e7 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -18,7 +18,7 @@ RUN yum --nodocs install -y --setopt=install_weak_deps=False temurin-17-jre RUN yum clean all -FROM ${DOCKER_UPSTREAM_REGISTRY}confluentinc/cp-base-lite AS BASE-LITE +FROM 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/confluentinc/cp-base-lite:7.9.x-latest-ubi8 AS BASE-LITE FROM registry.access.redhat.com/ubi8-micro:${UBI_MICRO_VERSION} AS REFRESH ARG PROJECT_VERSION From 664c82c874dd87f8beda267ebb4a2f2af67cebb4 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 20 Nov 2024 19:16:12 +0530 Subject: [PATCH 15/18] add dedupe scripts --- refresh/package_dedupe/package_dedupe.go | 68 ++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 refresh/package_dedupe/package_dedupe.go diff --git a/refresh/package_dedupe/package_dedupe.go b/refresh/package_dedupe/package_dedupe.go new file mode 100644 index 0000000000..7af27b59d4 --- /dev/null +++ b/refresh/package_dedupe/package_dedupe.go @@ -0,0 +1,68 @@ +package main + +import ( + "crypto/sha1" + + "fmt" + + "io" + "log" + "os" + "path/filepath" +) + +func dedupe_packages(rootPath string) { + sha2path := make(map[string]string) + err := filepath.Walk(rootPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + sha, err := shaSum(path) + if err != nil { + return err + } + if orig, exists := sha2path[sha]; exists { + relPath, err := filepath.Rel(filepath.Dir(path), orig) + if err != nil { + return err + } + os.Remove(path) + err = os.Symlink(relPath, path) + if err != nil { + return err + } + log.Printf("DEDUP: ln -sf %s %s\n", orig, path) + } else { + sha2path[sha] = path + } + return nil + }) + if err != nil { + log.Fatal(err) + } +} + +func shaSum(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha1.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: dedupe_packages ") + os.Exit(1) + } + basePath := os.Args[1] + dedupe_packages(basePath) +} \ No newline at end of file From c50952453eb534b38b9bf2d09eba4a0b5ec29d2c Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 20 Nov 2024 19:16:39 +0530 Subject: [PATCH 16/18] add dedupe scripts and jars --- refresh/Dockerfile.ubi8 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index 6a0a7758e7..1931a7d79e 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -3,6 +3,7 @@ ARG UBI_MICRO_VERSION=8.10-13 ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" ARG DOCKER_UPSTREAM_TAG +ARG GOLANG_VERSION FROM registry.access.redhat.com/ubi8 AS BUILD @@ -54,6 +55,16 @@ ENV PATH="/usr/lib/bin:${PATH}" RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ mkdir -p /home/appuser && \ chown 1001:1001 /home/appuser + +COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +RUN cd /usr/share/java \ + && package_dedupe $(pwd) + +FROM docker.io/golang:${GOLANG_VERSION} AS build_package_dedupe +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser package_dedupe/package_dedupe.go ./ +RUN go build -ldflags="-w -s" ./package_dedupe.go USER appuser WORKDIR /home/appuser From ce04ac75c132d0e6c7b340cfd2ba78f2a30b909b Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Wed, 20 Nov 2024 20:30:45 +0530 Subject: [PATCH 17/18] minor fixes to user group --- refresh/Dockerfile.ubi8 | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index 1931a7d79e..90d107dea7 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -5,6 +5,12 @@ ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docke ARG DOCKER_UPSTREAM_TAG ARG GOLANG_VERSION +FROM docker.io/golang:${GOLANG_VERSION} AS build_package_dedupe +WORKDIR /build +RUN useradd --no-log-init --create-home --shell /bin/bash appuser +COPY --chown=appuser:appuser package_dedupe/package_dedupe.go ./ +RUN go build -ldflags="-w -s" ./package_dedupe.go + FROM registry.access.redhat.com/ubi8 AS BUILD RUN printf "[temurin-jre] \n\ @@ -49,22 +55,19 @@ ENV LANG="C.UTF-8" #ARG MICRODIR COPY --from=BUILD /usr/lib/jvm/temurin-17-jre/ usr/lib COPY --from=BASE-LITE /usr/bin /usr/bin +COPY --from=build_package_dedupe /build/package_dedupe /usr/lib/bin/package_dedupe ENV PATH="/usr/lib/bin:${PATH}" RUN echo "appuser:x:1001:1001::/home/appuser:/bin/sh" >> /etc/passwd && \ mkdir -p /home/appuser && \ chown 1001:1001 /home/appuser + +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ +COPY target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/java/${ARTIFACT_ID}/* /usr/share/java/${ARTIFACT_ID}/ -COPY --chown=appuser:appuser target/${ARTIFACT_ID}-${PROJECT_VERSION}-package/share/doc/* /usr/share/doc/${ARTIFACT_ID}/ RUN cd /usr/share/java \ && package_dedupe $(pwd) -FROM docker.io/golang:${GOLANG_VERSION} AS build_package_dedupe -WORKDIR /build -RUN useradd --no-log-init --create-home --shell /bin/bash appuser -COPY --chown=appuser:appuser package_dedupe/package_dedupe.go ./ -RUN go build -ldflags="-w -s" ./package_dedupe.go - USER appuser WORKDIR /home/appuser From cabebb21c33f76749d71e516b3e2cdbbac4de970 Mon Sep 17 00:00:00 2001 From: Hrithik Kulkarni Date: Fri, 20 Dec 2024 14:25:20 +0530 Subject: [PATCH 18/18] replace ubi8-micro with ubi8-minimal --- refresh/Dockerfile.ubi8 | 3 ++- refresh/pom.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/refresh/Dockerfile.ubi8 b/refresh/Dockerfile.ubi8 index 90d107dea7..3a849067b8 100644 --- a/refresh/Dockerfile.ubi8 +++ b/refresh/Dockerfile.ubi8 @@ -4,6 +4,7 @@ ARG TEMURIN_JDK_VERSION="17.0.13.0.0.11-2" ARG DOCKER_UPSTREAM_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" ARG DOCKER_UPSTREAM_TAG ARG GOLANG_VERSION +ARG UBI_MINIMAL_VERSION="latest" FROM docker.io/golang:${GOLANG_VERSION} AS build_package_dedupe WORKDIR /build @@ -27,7 +28,7 @@ RUN yum clean all FROM 519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/confluentinc/cp-base-lite:7.9.x-latest-ubi8 AS BASE-LITE -FROM registry.access.redhat.com/ubi8-micro:${UBI_MICRO_VERSION} AS REFRESH +FROM registry.access.redhat.com/ubi8-minimal:${UBI_MINIMAL_VERSION} AS REFRESH ARG PROJECT_VERSION ARG ARTIFACT_ID diff --git a/refresh/pom.xml b/refresh/pom.xml index 958459fb17..9b6fb87707 100644 --- a/refresh/pom.xml +++ b/refresh/pom.xml @@ -28,7 +28,7 @@ pom - cp-base-refresh + cp-base-jre17 Refreshed Base for Confluent Docker images ${project.artifactId}