diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index cd5d72d..d24c087 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,7 +2,7 @@ name: Go on: push: - branches: [ "master" ] + branches: [ "*" ] pull_request: branches: [ "master" ] @@ -16,9 +16,11 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Setup Go ${{ matrix.go-version }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml new file mode 100644 index 0000000..47b963f --- /dev/null +++ b/.github/workflows/goreleaser.yml @@ -0,0 +1,44 @@ +name: goreleaser + +on: + push: + tags: + - '*' + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - + name: Set up Go + uses: actions/setup-go@v5 + + - + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: $GITHUB_ACTOR + password: ${{ secrets.GH_GORELEASER }} + + - + name: Download Syft + uses: anchore/sbom-action/download-syft@v0.17.4 + + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: '~> v2' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GH_GORELEASER }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 78999d0..778c9da 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,3 @@ vendor bin dist -.goreleaser.yml - -dist/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..13ca4e8 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,93 @@ +# yaml-language-server: $schema=https://goreleaser.com/static/schema.json +version: 2 +before: + hooks: + # You may remove this if you don't use go modules. + - go mod tidy +universal_binaries: + - replace: true +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + - freebsd + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - -X main.version={{.Version}} + - -X main.commit={{.ShortCommit}} + - -X main.Date={{.CommitDate}} + +archives: + - format: tar.gz + format_overrides: + - goos: windows + format: zip + - goos: darwin + format: zip + +checksum: + name_template: 'checksums.txt' + +snapshot: + version_template: "{{ incpatch .Version }}-next" + +changelog: + use: github + sort: asc + filters: + exclude: + - '^docs:' + - typo + - Goreleaser + - Dockerfile + - CI + - '^test:' + +sboms: + - artifacts: archive + +nfpms: + - file_name_template: "{{ .ConventionalFileName }}" + formats: + - deb + - rpm + - apk + - archlinux + dependencies: + - "bash" + maintainer: "Peter Teich " + vendor: "Peter Teich" + homepage: "https://github.com/pteich" + description: "Export Data from ElasticSearch to CSV/JSON using a Lucene Query (e.g. from Kibana) or a raw JSON Query string" + license: "MIT" + +dockers: + - image_templates: ["ghcr.io/pteich/elastic-query-export:{{ .Version }}"] + dockerfile: Dockerfile + build_flag_templates: + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.description={{ .ProjectName }} + - --label=org.opencontainers.image.url=https://github.com/pteich/elastic-query-export + - --label=org.opencontainers.image.source=https://github.com/pteich/elastic-query-export + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=MIT + +brews: + - name: elastic-query-export + homepage: https://github.com/pteich + repository: + owner: pteich + name: homebrew-elastic-query-export + +release: + draft: true + replace_existing_draft: true + replace_existing_artifacts: true \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5669c9e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:latest +LABEL org.opencontainers.image.source https://github.com/pteich/elastic-query-export + +COPY elastic-query-export /usr/bin/elastic-query-export + +ENTRYPOINT ["/usr/bin/elastic-query-export"] \ No newline at end of file diff --git a/README.md b/README.md index 4f1e51a..e748625 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,24 @@ Export Data from ElasticSearch to CSV by Raw or Lucene Query (e.g. from Kibana). Works with ElasticSearch 6+ (OpenSearch works too) and makes use of ElasticSearch's Scroll API and Go's -concurrency possibilities to work as fast as possible. +concurrency features to work as fast as possible. ## Install Download a pre-compiled binary for your operating system from here: https://github.com/pteich/elastic-query-export/releases You need just this binary. It works on OSX (Darwin), Linux and Windows. -### Arch +There are also prebuilt RPM, DEB and APK packages for your Linux distribution. + +### Brew + +Use Brew to install: +```shell +brew tap pteich/tap +brew install elastic-query-export +``` + +### Arch AUR ```shell yay -S elastic-query-export-bin diff --git a/go.mod b/go.mod index 53b7966..12cf255 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,11 @@ module github.com/pteich/elastic-query-export -go 1.20 +go 1.22 require ( github.com/olivere/elastic/v7 v7.0.32 github.com/pteich/configstruct v1.5.1 - golang.org/x/sync v0.6.0 + golang.org/x/sync v0.8.0 gopkg.in/cheggaaa/pb.v2 v2.0.7 ) @@ -15,8 +15,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/stretchr/testify v1.7.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect gopkg.in/fatih/color.v1 v1.7.0 // indirect gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect diff --git a/go.sum b/go.sum index 6e3d70f..934dd47 100644 --- a/go.sum +++ b/go.sum @@ -2,17 +2,15 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -22,27 +20,19 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pteich/configstruct v1.5.0 h1:aWSCFr2+z+oiECflHXb5/0aL8tGYFoRCiPQljl/pgvg= -github.com/pteich/configstruct v1.5.0/go.mod h1:P+V8K8Ivy8v9QfonDTW4a9o8b5g4JKixhP5+onxItfo= github.com/pteich/configstruct v1.5.1 h1:Yq9pdX3VBrjxF+WD4+WVFnQuMeHd+xodIFkAdgrYceI= github.com/pteich/configstruct v1.5.1/go.mod h1:yEmGYGCQTox72iuiThXENgdEmtll8UyTpRJu+YbjjVo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e h1:CsOuNlbOuf0mzxJIefr6Q4uAUetRUwZE4qt7VfzP+xo= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/VividCortex/ewma.v1 v1.1.1 h1:tWHEKkKq802K/JT9RiqGCBU5fW3raAPnJGTE9ostZvg= gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -56,8 +46,6 @@ gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIs gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=