Release from staging #97
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Release from staging | |
| # This is only expected to be invoked on-demand by a specific user. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| type: string | |
| description: The version we want to release from staging, ensure this is numeric without the v prefix for the tag. | |
| required: true | |
| docker-image: | |
| type: string | |
| description: Optionally override the image name to push to on Docker Hub. | |
| default: fluent/fluent-bit | |
| required: false | |
| github-image: | |
| type: string | |
| description: Optionally override the image name to push to on Github Container Registry. | |
| default: fluent/fluent-bit | |
| required: false | |
| # We do not want a new staging build to run whilst we are releasing the current staging build. | |
| # We also do not want multiples to run for the same version. | |
| concurrency: staging-build-release | |
| env: | |
| STAGING_IMAGE_NAME: ghcr.io/${{ github.repository }}/staging | |
| jobs: | |
| staging-release-version-check: | |
| name: Check staging release matches | |
| environment: release # required to get bucket name | |
| runs-on: ubuntu-latest | |
| outputs: | |
| major-version: ${{ steps.get_major_version.outputs.value }} | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Get the version on staging | |
| run: | | |
| curl --fail -LO "$AWS_URL/latest-version.txt" | |
| cat latest-version.txt | |
| STAGING_VERSION=$(cat latest-version.txt) | |
| [[ "$STAGING_VERSION" != "$RELEASE_VERSION" ]] && echo "Latest version mismatch: $STAGING_VERSION != $RELEASE_VERSION" && exit 1 | |
| # Must end in something that exits 0 | |
| echo "Successfully confirmed version is as expected: $STAGING_VERSION" | |
| shell: bash | |
| env: | |
| AWS_URL: https://${{ secrets.AWS_S3_BUCKET_STAGING }}.s3.amazonaws.com | |
| RELEASE_VERSION: ${{ github.event.inputs.version }} | |
| # Get the major version, i.e. 1.9.3 --> 1.9, or just return the passed in version. | |
| - name: Convert to major version format | |
| id: get_major_version | |
| run: | | |
| MAJOR_VERSION="$RELEASE_VERSION" | |
| if [[ $RELEASE_VERSION =~ ^[0-9]+\.[0-9]+ ]]; then | |
| MAJOR_VERSION="${BASH_REMATCH[0]}" | |
| fi | |
| echo "value=$MAJOR_VERSION" >> $GITHUB_OUTPUT | |
| shell: bash | |
| env: | |
| RELEASE_VERSION: ${{ github.event.inputs.version }} | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| staging-release-generate-package-matrix: | |
| name: Get package matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| deb-build-matrix: ${{ steps.get-matrix.outputs.deb-build-matrix }} | |
| rpm-build-matrix: ${{ steps.get-matrix.outputs.rpm-build-matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup runner | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y jq | |
| shell: bash | |
| # Cope with 1.9 as well as 2.0 | |
| - uses: ./.github/actions/generate-package-build-matrix | |
| id: get-matrix | |
| with: | |
| ref: v${{ inputs.version }} | |
| # Now annotate with whether it is Yum or Apt based | |
| # 1. Take packages from the staging bucket | |
| # 2. Sign them with the release GPG key | |
| # 3. Also take existing release packages from the release bucket. | |
| # 4. Create a full repo configuration using the existing releases as well. | |
| # 5. Upload to release bucket. | |
| # Note we could resign all packages as well potentially if we wanted to update the key. | |
| staging-release-yum-packages: | |
| name: S3 - update YUM packages bucket | |
| runs-on: ubuntu-22.04 # no createrepo on Ubuntu 20.04 | |
| environment: release | |
| needs: | |
| - staging-release-version-check | |
| - staging-release-generate-package-matrix | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.rpm-build-matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup runner | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y createrepo-c rpm | |
| shell: bash | |
| - name: Import GPG key for signing | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@v6 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} | |
| # Download the current release bucket | |
| # Add everything from staging | |
| # Sign and set up metadata for it all | |
| # Upload to release bucket | |
| - name: Sync packages from buckets on S3 | |
| run: | | |
| mkdir -p "packaging/releases/$DISTRO" | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| DISTRO: ${{ matrix.distro }} | |
| shell: bash | |
| - name: GPG set up keys for signing | |
| run: | | |
| gpg --export -a "${{ steps.import_gpg.outputs.name }}" > /tmp/fluentbit.key | |
| rpm --import /tmp/fluentbit.key | |
| shell: bash | |
| - name: Update repo info and remove any staging details | |
| run: | | |
| packaging/update-yum-repo.sh | |
| env: | |
| GPG_KEY: ${{ steps.import_gpg.outputs.name }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} | |
| VERSION: ${{ github.event.inputs.version }} | |
| BASE_PATH: "packaging/releases" | |
| RPM_REPO: ${{ matrix.distro }} | |
| shell: bash | |
| - name: Sync to release bucket on S3 | |
| run: | | |
| aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| DISTRO: ${{ matrix.distro }} | |
| shell: bash | |
| staging-release-apt-packages: | |
| name: S3 - update APT packages bucket | |
| runs-on: ubuntu-latest | |
| environment: release | |
| needs: | |
| - staging-release-version-check | |
| - staging-release-generate-package-matrix | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: ${{ fromJSON(needs.staging-release-generate-package-matrix.outputs.deb-build-matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup runner | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y aptly debsigs distro-info rsync | |
| shell: bash | |
| - name: Convert version to codename | |
| id: get_codename | |
| run: | | |
| CODENAME="$DISTRO" | |
| if [[ "$DISTRO" == ubuntu* ]]; then | |
| echo "Converting Ubuntu version to codename" | |
| UBUNTU_NAME=$(grep "${DISTRO##*/} LTS" /usr/share/distro-info/ubuntu.csv|cut -d ',' -f3) | |
| echo "Got Ubuntu codename: $UBUNTU_NAME" | |
| CODENAME="ubuntu/$UBUNTU_NAME" | |
| fi | |
| echo "Using codename: $CODENAME" | |
| echo "CODENAME=$CODENAME" >> $GITHUB_OUTPUT | |
| shell: bash | |
| env: | |
| DISTRO: ${{ matrix.distro }} | |
| - name: Import GPG key for signing | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@v6 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} | |
| - name: Sync packages from buckets on S3 | |
| run: | | |
| mkdir -p "packaging/releases/$CODENAME" | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$CODENAME" "packaging/releases/$CODENAME" --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| CODENAME: ${{ steps.get_codename.outputs.CODENAME }} | |
| shell: bash | |
| - name: Update repo info and remove any staging details | |
| run: | | |
| packaging/update-apt-repo.sh | |
| env: | |
| GPG_KEY: ${{ steps.import_gpg.outputs.name }} | |
| AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} | |
| VERSION: ${{ github.event.inputs.version }} | |
| BASE_PATH: "packaging/releases" | |
| DEB_REPO: ${{ steps.get_codename.outputs.CODENAME }} | |
| shell: bash | |
| - name: Sync to release bucket on S3 | |
| run: | | |
| aws s3 sync "packaging/releases/$CODENAME" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$CODENAME" --delete --follow-symlinks --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| CODENAME: ${{ steps.get_codename.outputs.CODENAME }} | |
| shell: bash | |
| staging-release-update-non-linux-s3: | |
| name: Update Windows and macOS packages | |
| runs-on: ubuntu-22.04 | |
| environment: release | |
| needs: | |
| - staging-release-version-check | |
| permissions: | |
| contents: none | |
| strategy: | |
| matrix: | |
| distro: | |
| - macos | |
| - windows | |
| fail-fast: false | |
| steps: | |
| - name: Sync packages from buckets on S3 | |
| run: | | |
| mkdir -p "packaging/releases/$DISTRO" | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/$DISTRO" "packaging/releases/$DISTRO" --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| DISTRO: ${{ matrix.distro }} | |
| shell: bash | |
| - name: Sync to release bucket on S3 | |
| run: | | |
| aws s3 sync "packaging/releases/$DISTRO" "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/$DISTRO" --delete --follow-symlinks --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| DISTRO: ${{ matrix.distro }} | |
| shell: bash | |
| staging-release-update-base-s3: | |
| name: Update top-level bucket info | |
| runs-on: ubuntu-22.04 | |
| environment: release | |
| needs: | |
| - staging-release-apt-packages | |
| - staging-release-yum-packages | |
| permissions: | |
| contents: none | |
| steps: | |
| - name: Import GPG key for signing | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@v6 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| passphrase: ${{ secrets.GPG_PRIVATE_KEY_PASSPHRASE }} | |
| - name: GPG public key | |
| run: | | |
| gpg --export -a "${{ steps.import_gpg.outputs.name }}" > ./fluentbit.key | |
| aws s3 cp ./fluentbit.key s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/fluentbit.key --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| shell: bash | |
| - name: JSON schema | |
| continue-on-error: true | |
| run: | | |
| aws s3 sync "s3://${AWS_STAGING_S3_BUCKET}/${VERSION}" "s3://${AWS_RELEASE_S3_BUCKET}/${VERSION}" --no-progress | |
| env: | |
| VERSION: ${{ github.event.inputs.version }} | |
| AWS_REGION: "us-east-1" | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_STAGING_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_STAGING }} | |
| AWS_RELEASE_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET_RELEASE }} | |
| shell: bash | |
| staging-release-source-s3: | |
| name: S3 - update source bucket | |
| runs-on: ubuntu-latest | |
| environment: release | |
| needs: | |
| - staging-release-version-check | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Sync packages from buckets on S3 | |
| run: | | |
| mkdir -p release staging | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" release/ --no-progress | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_STAGING }}/source/" staging/ --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| shell: bash | |
| - name: Move components from staging and setup | |
| run: | | |
| ./packaging/update-source-packages.sh | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| SOURCE_DIR: staging | |
| WINDOWS_SOURCE_DIR: appveyor | |
| TARGET_DIR: release | |
| VERSION: ${{ github.event.inputs.version }} | |
| MAJOR_VERSION: ${{ needs.staging-release-version-check.outputs.major-version }} | |
| shell: bash | |
| - name: Sync to bucket on S3 | |
| run: | | |
| aws s3 sync release/ "s3://${{ secrets.AWS_S3_BUCKET_RELEASE_SOURCES }}" --delete --follow-symlinks --no-progress | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| shell: bash | |
| # Simple skopeo copy jobs to transfer image from staging to release registry with optional GPG key signing. | |
| # Unfortunately skopeo currently does not support Cosign: https://github.com/containers/skopeo/issues/1533 | |
| staging-release-images: | |
| name: Release ${{ matrix.tag }} Linux container images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - staging-release-version-check | |
| environment: release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # All the explicit tags we want to release | |
| tag: [ | |
| "${{ github.event.inputs.version }}", | |
| "${{ needs.staging-release-version-check.outputs.major-version }}", | |
| "${{ github.event.inputs.version }}-debug", | |
| "${{ needs.staging-release-version-check.outputs.major-version }}-debug", | |
| ] | |
| permissions: | |
| packages: write | |
| steps: | |
| # Primarily because the skopeo errors are hard to parse and non-obvious | |
| - name: Check the image exists | |
| run: | | |
| docker pull "$STAGING_IMAGE_NAME:$TAG" | |
| env: | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it. | |
| - name: Promote container images from staging to Dockerhub | |
| run: | | |
| docker run --rm \ | |
| quay.io/skopeo/stable:latest \ | |
| copy \ | |
| --all \ | |
| --retry-times 10 \ | |
| --src-no-creds \ | |
| --dest-creds "$RELEASE_CREDS" \ | |
| "docker://$STAGING_IMAGE_NAME:$TAG" \ | |
| "docker://$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} | |
| RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| - name: Promote container images from staging to GHCR.io | |
| if: ${{ startsWith(github.event.inputs.version, '2.') || startsWith(github.event.inputs.version, '3.') || ! startsWith(matrix.tag, 'latest') }} | |
| run: | | |
| docker run --rm \ | |
| quay.io/skopeo/stable:latest \ | |
| copy \ | |
| --all \ | |
| --retry-times 10 \ | |
| --src-no-creds \ | |
| --dest-creds "$RELEASE_CREDS" \ | |
| "docker://$STAGING_IMAGE_NAME:$TAG" \ | |
| "docker://$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} | |
| RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| # Part of resolution for: https://github.com/fluent/fluent-bit/issues/7748 | |
| # More recent build-push-actions may mean legacy format is not preserved so we provide arch-specific tags just in case | |
| staging-release-images-arch-specific-legacy-tags: | |
| name: Release ${{ matrix.arch }} legacy format Linux container images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - staging-release-images | |
| environment: release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - amd64 | |
| - arm64 | |
| - arm/v7 | |
| permissions: | |
| packages: write | |
| env: | |
| RELEASE_IMAGE_NAME: ${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} | |
| RELEASE_TAG: ${{ github.event.inputs.version }} | |
| steps: | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Convert arch to tag | |
| id: get-tag | |
| run: | | |
| TAG="${RELEASE_TAG}-${{ matrix.arch }}" | |
| echo "Input value: $TAG" | |
| TAG=${TAG/\//-} | |
| echo "Using tag: $TAG" | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - name: Pull release image | |
| run: docker pull --platform='linux/${{ matrix.arch }}' "$RELEASE_IMAGE_NAME:$RELEASE_TAG" | |
| shell: bash | |
| - name: Tag and push legacy format image to DockerHub | |
| run: | | |
| docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" docker.io/"$RELEASE_IMAGE_NAME:$TAG" | |
| docker push docker.io/"$RELEASE_IMAGE_NAME:$TAG" | |
| shell: bash | |
| env: | |
| TAG: ${{ steps.get-tag.outputs.tag }} | |
| - name: Tag and push legacy format image to Github Container Registry | |
| run: | | |
| docker tag "$RELEASE_IMAGE_NAME:$RELEASE_TAG" ghcr.io/"$RELEASE_IMAGE_NAME:$TAG" | |
| docker push ghcr.io/"$RELEASE_IMAGE_NAME:$TAG" | |
| shell: bash | |
| env: | |
| TAG: ${{ steps.get-tag.outputs.tag }} | |
| staging-release-images-latest-tags: | |
| # Only update latest tags for 3.1 releases | |
| if: startsWith(github.event.inputs.version, '3.1') | |
| # if: startsWith(github.event.inputs.version, '4.0') | |
| name: Release latest Linux container images | |
| runs-on: ubuntu-latest | |
| needs: | |
| - staging-release-images | |
| environment: release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: [ | |
| "latest", | |
| "latest-debug" | |
| ] | |
| permissions: | |
| packages: write | |
| steps: | |
| # Primarily because the skopeo errors are hard to parse and non-obvious | |
| - name: Check the image exists | |
| run: | | |
| docker pull "$STAGING_IMAGE_NAME:$TAG" | |
| env: | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| # Use the container to prevent any rootless issues and we do not need to use GPG signing as DockerHub does not support it. | |
| - name: Promote container images from staging to Dockerhub | |
| run: | | |
| docker run --rm \ | |
| quay.io/skopeo/stable:latest \ | |
| copy \ | |
| --all \ | |
| --retry-times 10 \ | |
| --src-no-creds \ | |
| --dest-creds "$RELEASE_CREDS" \ | |
| "docker://$STAGING_IMAGE_NAME:$TAG" \ | |
| "docker://$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} | |
| RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| - name: Promote container images from staging to GHCR.io | |
| run: | | |
| docker run --rm \ | |
| quay.io/skopeo/stable:latest \ | |
| copy \ | |
| --all \ | |
| --retry-times 10 \ | |
| --src-no-creds \ | |
| --dest-creds "$RELEASE_CREDS" \ | |
| "docker://$STAGING_IMAGE_NAME:$TAG" \ | |
| "docker://$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} | |
| RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| staging-release-images-windows: | |
| name: Release Windows images | |
| # Cannot be done by Skopeo on a Linux runner unfortunately | |
| runs-on: windows-latest | |
| needs: | |
| - staging-release-version-check | |
| environment: release | |
| permissions: | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tag: [ | |
| "windows-2019-${{ github.event.inputs.version }}", | |
| "windows-2022-${{ github.event.inputs.version }}" | |
| ] | |
| steps: | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check the image exists | |
| run: | | |
| docker pull "$STAGING_IMAGE_NAME:$TAG" | |
| env: | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| - name: Promote container images from staging to GHCR.io | |
| run: | | |
| docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG" | |
| docker push "$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} | |
| RELEASE_CREDS: ${{ github.actor }}:${{ secrets.GITHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Promote container images from staging to Dockerhub | |
| run: | | |
| docker tag "$STAGING_IMAGE_NAME:$TAG" "$RELEASE_IMAGE_NAME:$TAG" | |
| docker push "$RELEASE_IMAGE_NAME:$TAG" | |
| env: | |
| RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} | |
| RELEASE_CREDS: ${{ secrets.DOCKERHUB_USERNAME }}:${{ secrets.DOCKERHUB_TOKEN }} | |
| TAG: ${{ matrix.tag }} | |
| shell: bash | |
| staging-release-images-sign: | |
| name: Sign container image manifests | |
| permissions: write-all | |
| runs-on: ubuntu-latest | |
| environment: release | |
| needs: | |
| - staging-release-images | |
| env: | |
| DH_RELEASE_IMAGE_NAME: docker.io/${{ github.event.inputs.docker-image || secrets.DOCKERHUB_ORGANIZATION }} | |
| GHCR_RELEASE_IMAGE_NAME: ghcr.io/${{ github.event.inputs.github-image || github.repository }} | |
| steps: | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@v2 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cosign with a key | |
| # Only run if we have a key defined | |
| if: ${{ env.COSIGN_PRIVATE_KEY }} | |
| # The key needs to cope with newlines | |
| run: | | |
| echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key | |
| cosign sign --key /tmp/my_cosign.key --recursive \ | |
| -a "repo=${{ github.repository }}" \ | |
| -a "workflow=${{ github.workflow }}" \ | |
| -a "release=${{ github.event.inputs.version }}" \ | |
| "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ | |
| "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \ | |
| "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ | |
| "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" | |
| rm -f /tmp/my_cosign.key | |
| shell: bash | |
| env: | |
| COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional | |
| - name: Cosign keyless signing using Rektor public transparency log | |
| # This step uses the identity token to provision an ephemeral certificate | |
| # against the sigstore community Fulcio instance, and records it to the | |
| # sigstore community Rekor transparency log. | |
| # | |
| # We use recursive signing on the manifest to cover all the images. | |
| run: | | |
| cosign sign --yes --recursive \ | |
| -a "repo=${{ github.repository }}" \ | |
| -a "workflow=${{ github.workflow }}" \ | |
| -a "release=${{ github.event.inputs.version }}" \ | |
| "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ | |
| "$GHCR_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" \ | |
| "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}" \ | |
| "$DH_RELEASE_IMAGE_NAME:${{ github.event.inputs.version }}-debug" | |
| shell: bash | |
| env: | |
| COSIGN_EXPERIMENTAL: true | |
| staging-release-upload-cosign-key: | |
| name: Upload Cosign public key for verification | |
| needs: | |
| - staging-release-images-sign | |
| permissions: | |
| contents: none | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Install cosign | |
| uses: sigstore/cosign-installer@v2 | |
| - name: Get public key and add to S3 bucket | |
| # Only run if we have a key defined | |
| if: ${{ env.COSIGN_PRIVATE_KEY }} | |
| # The key needs to cope with newlines | |
| run: | | |
| echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key | |
| cosign public-key --key /tmp/my_cosign.key > ./cosign.pub | |
| rm -f /tmp/my_cosign.key | |
| cat ./cosign.pub | |
| aws s3 cp ./cosign.pub "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/cosign.pub" --no-progress | |
| shell: bash | |
| env: | |
| COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} | |
| COSIGN_PASSWORD: ${{ secrets.COSIGN_PRIVATE_KEY_PASSWORD }} # optional | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| staging-release-smoke-test-packages: | |
| name: Run package smoke tests | |
| permissions: | |
| contents: read | |
| runs-on: ubuntu-latest | |
| environment: release | |
| needs: | |
| - staging-release-apt-packages | |
| - staging-release-yum-packages | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Test release packages | |
| run: | | |
| ./packaging/test-release-packages.sh | |
| shell: bash | |
| env: | |
| VERSION_TO_CHECK_FOR: ${{ github.event.inputs.version }} | |
| FLUENT_BIT_PACKAGES_URL: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com | |
| FLUENT_BIT_PACKAGES_KEY: http://${{ secrets.AWS_S3_BUCKET_RELEASE }}.s3.amazonaws.com/fluentbit.key | |
| staging-release-smoke-test-containers: | |
| name: Run container smoke tests | |
| permissions: | |
| contents: read | |
| packages: read | |
| runs-on: ubuntu-latest | |
| environment: release | |
| needs: | |
| - staging-release-images | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Test containers | |
| run: | | |
| ./packaging/testing/smoke/container/container-smoke-test.sh | |
| shell: bash | |
| env: | |
| IMAGE_TAG: ${{ github.event.inputs.version }} | |
| staging-release-create-release: | |
| name: Create the Github Release once packages and containers are up | |
| needs: | |
| - staging-release-images | |
| - staging-release-apt-packages | |
| - staging-release-yum-packages | |
| permissions: | |
| contents: write | |
| environment: release | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Release 2.0 - not latest | |
| uses: softprops/action-gh-release@v2 | |
| if: startsWith(inputs.version, '2.0') | |
| with: | |
| body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" | |
| draft: false | |
| generate_release_notes: true | |
| name: "Fluent Bit ${{ inputs.version }}" | |
| tag_name: v${{ inputs.version }} | |
| target_commitish: '2.0' | |
| make_latest: false | |
| - name: Release 2.1 - not latest | |
| uses: softprops/action-gh-release@v2 | |
| if: startsWith(inputs.version, '2.1') | |
| with: | |
| body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" | |
| draft: false | |
| generate_release_notes: true | |
| name: "Fluent Bit ${{ inputs.version }}" | |
| tag_name: v${{ inputs.version }} | |
| target_commitish: '2.1' | |
| make_latest: false | |
| - name: Release 3.0 - not latest | |
| uses: softprops/action-gh-release@v2 | |
| if: startsWith(inputs.version, '3.0') | |
| with: | |
| body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" | |
| draft: false | |
| generate_release_notes: true | |
| name: "Fluent Bit ${{ inputs.version }}" | |
| tag_name: v${{ inputs.version }} | |
| target_commitish: '3.0' | |
| make_latest: false | |
| - name: Release 3.1 and latest | |
| # TODO: change to 3.1 branch once 4.0 series is ready | |
| uses: softprops/action-gh-release@v2 | |
| if: startsWith(inputs.version, '3.1') | |
| with: | |
| body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" | |
| draft: false | |
| generate_release_notes: true | |
| name: "Fluent Bit ${{ inputs.version }}" | |
| tag_name: v${{ inputs.version }} | |
| make_latest: true | |
| # - name: Release 4.0 and latest | |
| # uses: softprops/action-gh-release@v2 | |
| # if: startsWith(inputs.version, '4.0') | |
| # with: | |
| # body: "https://fluentbit.io/announcements/v${{ inputs.version }}/" | |
| # draft: false | |
| # generate_release_notes: true | |
| # name: "Fluent Bit ${{ inputs.version }}" | |
| # tag_name: v${{ inputs.version }} | |
| # make_latest: true | |
| staging-release-windows-checksums: | |
| name: Get Windows checksums for new release | |
| runs-on: ubuntu-22.04 | |
| environment: release | |
| needs: | |
| - staging-release-update-non-linux-s3 | |
| permissions: | |
| contents: none | |
| outputs: | |
| windows-exe32-hash: ${{ steps.hashes.outputs.WIN_32_EXE_HASH }} | |
| windows-zip32-hash: ${{ steps.hashes.outputs.WIN_32_ZIP_HASH }} | |
| windows-exe64-hash: ${{ steps.hashes.outputs.WIN_64_EXE_HASH }} | |
| windows-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ZIP_HASH }} | |
| windows-arm-exe64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_EXE_HASH }} | |
| windows-arm-zip64-hash: ${{ steps.hashes.outputs.WIN_64_ARM_ZIP_HASH }} | |
| steps: | |
| - name: Sync release Windows directory to get checksums | |
| run: | |
| aws s3 sync "s3://${{ secrets.AWS_S3_BUCKET_RELEASE }}/windows" ./ --exclude "*" --include "*.sha256" | |
| shell: bash | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "us-east-1" | |
| - name: Provide output for documentation PR | |
| id: hashes | |
| # do not fail the build for this | |
| continue-on-error: true | |
| run: | | |
| ls -l | |
| export WIN_32_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.exe.sha256"|awk '{print $1}') | |
| export WIN_32_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win32.zip.sha256"|awk '{print $1}') | |
| export WIN_64_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.exe.sha256"|awk '{print $1}') | |
| export WIN_64_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-win64.zip.sha256"|awk '{print $1}') | |
| if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256" ]]; then | |
| export WIN_64_ARM_EXE_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.exe.sha256"|awk '{print $1}') | |
| fi | |
| if [[ -f "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256" ]]; then | |
| export WIN_64_ARM_ZIP_HASH=$(cat "./fluent-bit-${{ inputs.version }}-winarm64.zip.sha256"|awk '{print $1}') | |
| fi | |
| set | grep WIN_ | |
| echo WIN_32_EXE_HASH="$WIN_32_EXE_HASH" >> $GITHUB_OUTPUT | |
| echo WIN_32_ZIP_HASH="$WIN_32_ZIP_HASH" >> $GITHUB_OUTPUT | |
| echo WIN_64_EXE_HASH="$WIN_64_EXE_HASH" >> $GITHUB_OUTPUT | |
| echo WIN_64_ZIP_HASH="$WIN_64_ZIP_HASH" >> $GITHUB_OUTPUT | |
| echo WIN_64_ARM_EXE_HASH="$WIN_64_ARM_EXE_HASH" >> $GITHUB_OUTPUT | |
| echo WIN_64_ARM_ZIP_HASH="$WIN_64_ARM_ZIP_HASH" >> $GITHUB_OUTPUT | |
| shell: bash | |
| staging-release-create-docs-pr: | |
| name: Create docs updates for new release | |
| needs: | |
| - staging-release-images | |
| - staging-release-windows-checksums | |
| permissions: | |
| contents: none | |
| environment: release | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Release 2.0 - not latest | |
| if: startsWith(inputs.version, '2.0') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: fluent/fluent-bit-docs | |
| ref: 2.0 | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| - name: Release 2.1 - not latest | |
| if: startsWith(inputs.version, '2.1') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: fluent/fluent-bit-docs | |
| ref: 2.1 | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| - name: Release 2.2 - not latest | |
| if: startsWith(inputs.version, '2.2') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: fluent/fluent-bit-docs | |
| ref: 2.2 | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| - name: Release 3.0 - not latest | |
| if: startsWith(inputs.version, '3.0') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: fluent/fluent-bit-docs | |
| ref: 3.0 | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| - name: Release 3.1 and latest | |
| # TODO: change to 3.1 branch once 4.0 series is ready | |
| if: startsWith(inputs.version, '3.1') | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: fluent/fluent-bit-docs | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| # - name: Release 4.0 and latest | |
| # if: startsWith(inputs.version, '4.0') | |
| # uses: actions/checkout@v4 | |
| # with: | |
| # repository: fluent/fluent-bit-docs | |
| # token: ${{ secrets.GH_PA_TOKEN }} | |
| - name: Ensure we have the script we need | |
| run: | | |
| if [[ ! -f update-release-version-docs.sh ]] ; then | |
| git checkout update-release-version-docs.sh -- master | |
| fi | |
| shell: bash | |
| - name: Update versions | |
| # Uses https://github.com/fluent/fluent-bit-docs/blob/master/update-release-version-docs.sh | |
| run: | | |
| ./update-release-version-docs.sh | |
| shell: bash | |
| env: | |
| NEW_VERSION: ${{ inputs.version }} | |
| WIN_32_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe32-hash }} | |
| WIN_32_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip32-hash }} | |
| WIN_64_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-exe64-hash }} | |
| WIN_64_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-zip64-hash }} | |
| WIN_64_ARM_EXE_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-exe64-hash }} | |
| WIN_64_ARM_ZIP_HASH: ${{ needs.staging-release-windows-checksums.outputs.windows-arm-zip64-hash }} | |
| - name: Raise docs PR | |
| id: cpr | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: 'release: update to v${{ inputs.version }}' | |
| signoff: true | |
| delete-branch: true | |
| title: 'release: update to v${{ inputs.version }}' | |
| # We need workflows permission so have to use the GH_PA_TOKEN | |
| token: ${{ secrets.GH_PA_TOKEN }} | |
| labels: ci,automerge | |
| body: | | |
| Update release ${{ inputs.version }} version. | |
| - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request | |
| draft: false | |
| - name: Check outputs | |
| if: ${{ steps.cpr.outputs.pull-request-number }} | |
| run: | | |
| echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" | |
| echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" | |
| staging-release-create-version-update-pr: | |
| name: Create version update PR for new release | |
| needs: | |
| - staging-release-create-release | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| environment: release | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Release 2.0 | |
| if: startsWith(inputs.version, '2.0') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 2.0 | |
| - name: Release 2.1 | |
| if: startsWith(inputs.version, '2.1') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 2.1 | |
| - name: Release 2.2 | |
| if: startsWith(inputs.version, '2.2') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 2.2 | |
| - name: Release 3.0 | |
| if: startsWith(inputs.version, '3.0') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 3.0 | |
| - name: Release 3.1 | |
| if: startsWith(inputs.version, '3.1') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 3.1 | |
| - name: Release 4.0 | |
| if: startsWith(inputs.version, '4.0') | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: master | |
| # Get the new version to use | |
| - name: 'Get next minor version' | |
| id: semvers | |
| uses: "WyriHaximus/github-action-next-semvers@v1" | |
| with: | |
| version: ${{ inputs.version }} | |
| strict: true | |
| - run: ./update_version.sh | |
| shell: bash | |
| env: | |
| NEW_VERSION: ${{ steps.semvers.outputs.patch }} | |
| # Ensure we use the PR action to do the work | |
| DISABLE_COMMIT: 'yes' | |
| - name: Raise FB PR to update version | |
| id: cpr | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| commit-message: 'release: update to ${{ steps.semvers.outputs.patch }}' | |
| signoff: true | |
| delete-branch: true | |
| title: 'release: update to ${{ steps.semvers.outputs.patch }}' | |
| labels: ci,automerge | |
| body: | | |
| Update next release to ${{ steps.semvers.outputs.patch }} version. | |
| - Created by ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - Auto-generated by create-pull-request: https://github.com/peter-evans/create-pull-request | |
| draft: false | |
| - name: Check outputs | |
| if: ${{ steps.cpr.outputs.pull-request-number }} | |
| run: | | |
| echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" | |
| echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" |