Merge pull request #100 from stefanprodan/feat-ignore

Add support for `timoni.ignore`

api/v1alpha1/ignore.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2023 Stefan Prodan
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+const (
+	IgnoreFile            = "timoni.ignore"
+	DefaultIgnorePatterns = `# VCS
+.git/
+.gitignore
+.gitmodules
+.gitattributes
+
+# Go
+vendor/
+go.mod
+go.sum
+
+# CUE
+*_tool.cue
+
+# Timoni
+timoni.ignore
+`
+)

cmd/timoni/mod_init.go

@@ -202,5 +202,5 @@ func initModuleFromTemplate(mName, mTmpl, src string, dst string) (err error) {
 		}
 	}
 
-	return
+	return os.WriteFile(filepath.Join(dst, apiv1.IgnoreFile), []byte(apiv1.DefaultIgnorePatterns), 0600)
 }

cmd/timoni/mod_pull_test.go

@@ -56,12 +56,21 @@ func Test_PullMod(t *testing.T) {
 	fsErr := filepath.Walk(modPath, func(path string, info fs.FileInfo, err error) error {
 		if !info.IsDir() {
 			tmpPath := filepath.Join(tmpDir, strings.TrimPrefix(path, modPath))
-			if _, err := os.Stat(tmpPath); err != nil && os.IsNotExist(err) {
+			if _, err := os.Stat(tmpPath); err != nil && os.IsNotExist(err) && !strings.Contains(tmpPath, "ignore") {
 				return fmt.Errorf("file '%s' should exist in pulled module", path)
 			}
 		}
 
 		return nil
 	})
 	g.Expect(fsErr).ToNot(HaveOccurred())
+
+	// Walk the pulled module and check ignored files
+	fsErr = filepath.Walk(tmpDir, func(path string, info fs.FileInfo, err error) error {
+		if strings.Contains(info.Name(), "ignore") {
+			return fmt.Errorf("file '%s' should not exist in pulled module", path)
+		}
+		return nil
+	})
+	g.Expect(fsErr).ToNot(HaveOccurred())
 }

cmd/timoni/mod_push.go

@@ -30,6 +30,7 @@ import (
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/yaml"
 
+	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 	"github.com/stefanprodan/timoni/internal/engine"
 	"github.com/stefanprodan/timoni/internal/flags"
 )
@@ -149,6 +150,12 @@ func pushModCmdRun(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	ps, err := engine.ReadIgnoreFile(path)
+	if err != nil {
+		return fmt.Errorf("reading %s failed: %w", apiv1.IgnoreFile, err)
+	}
+	pushModArgs.ignorePaths = append(pushModArgs.ignorePaths, ps...)
+
 	digestURL, err := ociClient.Push(ctx, url, path, meta, pushModArgs.ignorePaths)
 	if err != nil {
 		return fmt.Errorf("pushing module failed: %w", err)

cmd/timoni/testdata/module/ignore/ignore.txt

@@ -0,0 +1 @@
+test

cmd/timoni/testdata/module/timoni.ignore

@@ -0,0 +1,19 @@
+# VCS
+.git/
+.gitignore
+.gitmodules
+.gitattributes
+
+# Go
+vendor/
+go.mod
+go.sum
+
+# CUE
+*_tool.cue
+
+# Timoni
+timoni.ignore
+
+# Test dir
+ignore/

docs/module.md

@@ -20,6 +20,7 @@ myapp
 │   ├── deployment.cue # Kubernetes Deployment template
 │   └── service.cue # Kubernetes Service template
 ├── timoni.cue # Timoni entry point
+├── timoni.ignore # Ignore rules
 └── values.cue # Timoni values placeholder
 ```
 
@@ -76,6 +77,34 @@ timoni: {
 }
 ```
 
+## Ignore
+
+The `timoni.ignore` file contains rules in the
+[.gitignore pattern format](https://git-scm.com/docs/gitignore#_pattern_format).
+The paths matching the defined rules are excluded when publishing
+the module to a container registry.
+
+When publishing modules as OCI artifacts, it is recommended to use the following ignore patterns:
+
+```.gitignore
+# VCS
+.git/
+.gitignore
+.gitmodules
+.gitattributes
+
+# Go
+vendor/
+go.mod
+go.sum
+
+# CUE
+*_tool.cue
+
+# Timoni
+timoni.ignore
+```
+
 ## Values
 
 The `values.cue` file is a placeholder for the user-supplied values.

go.mod

@@ -7,11 +7,13 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.0
 	github.com/distribution/distribution/v3 v3.0.0-20230327091844-0c958010ace2
 	github.com/fluxcd/pkg/oci v0.23.0
+	github.com/fluxcd/pkg/sourceignore v0.3.3
 	github.com/fluxcd/pkg/ssa v0.27.0
 	github.com/google/go-containerregistry v0.14.0
 	github.com/mattn/go-shellwords v1.0.12
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/onsi/gomega v1.27.6
+	github.com/otiai10/copy v1.11.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.7.0
@@ -73,7 +75,6 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fluxcd/go-git/v5 v5.0.0-20221219190809-2e5c9d01cfc4 // indirect
-	github.com/fluxcd/pkg/sourceignore v0.3.3 // indirect
 	github.com/fluxcd/pkg/tar v0.2.0 // indirect
 	github.com/fluxcd/pkg/version v0.2.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect

go.sum

@@ -412,6 +412,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
 github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc=
+github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=

internal/engine/testdata/module/ignore/ignore.txt

@@ -0,0 +1 @@
+test

internal/engine/testdata/module/ignore_tool.cue

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"tool/cli"
+)
+
+command: test: {
+	task: print: cli.Print & {
+		text: "test"
+	}
+}

internal/engine/testdata/module/timoni.ignore

@@ -0,0 +1,8 @@
+# Timoni ignore
+timoni.ignore
+
+# CUE tools
+*_tool.cue
+
+# Dirs
+ignore/

internal/engine/utils.go

@@ -17,109 +17,57 @@ limitations under the License.
 package engine
 
 import (
-	"fmt"
-	"io"
+	"bufio"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"cuelang.org/go/cue"
+	"github.com/fluxcd/pkg/sourceignore"
+	cp "github.com/otiai10/copy"
+
+	apiv1 "github.com/stefanprodan/timoni/api/v1alpha1"
 )
 
 // CopyModule copies the given module to the destination directory,
-// while excluding symlinks. The destination directory must not exit.
-func CopyModule(src string, dst string) (err error) {
-	src = filepath.Clean(src)
-	dst = filepath.Clean(dst)
+// while excluding files that match the timoni.ignore patterns.
+func CopyModule(srcDir string, dstDir string) (err error) {
+	srcDir = filepath.Clean(srcDir)
+	dstDir = filepath.Clean(dstDir)
 
-	si, err := os.Stat(src)
+	domain := strings.Split(srcDir, string(filepath.Separator))
+	ps, err := sourceignore.ReadIgnoreFile(filepath.Join(srcDir, apiv1.IgnoreFile), domain)
 	if err != nil {
 		return err
 	}
-	if !si.IsDir() {
-		return fmt.Errorf("source %s is not a directory", src)
-	}
-
-	_, err = os.Stat(dst)
-	if err != nil && !os.IsNotExist(err) {
-		return
-	}
-	if err == nil {
-		return fmt.Errorf("destination %s already exists", dst)
-	}
-
-	err = os.MkdirAll(dst, si.Mode())
-	if err != nil {
-		return
-	}
-
-	entries, err := os.ReadDir(src)
-	if err != nil {
-		return
-	}
-
-	for _, entry := range entries {
-		srcPath := filepath.Join(src, entry.Name())
-		dstPath := filepath.Join(dst, entry.Name())
+	matcher := sourceignore.NewMatcher(ps)
 
-		if entry.IsDir() {
-			err = CopyModule(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		} else {
-			if fi, fiErr := entry.Info(); fiErr != nil || !fi.Mode().IsRegular() {
-				return
-			}
-
-			err = CopyModuleFile(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		}
+	opt := cp.Options{
+		Skip: func(info os.FileInfo, src, dest string) (bool, error) {
+			return matcher.Match(strings.Split(src, string(filepath.Separator)), info.IsDir()), nil
+		},
 	}
 
-	return
+	return cp.Copy(srcDir, dstDir, opt)
 }
 
-// CopyModuleFile copies a file from source to destination
-// while preserving permissions.
-func CopyModuleFile(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		if e := out.Close(); e != nil {
-			err = e
+// ReadIgnoreFile returns the ignore patters found in the module root.
+func ReadIgnoreFile(moduleRoot string) ([]string, error) {
+	path := filepath.Join(moduleRoot, apiv1.IgnoreFile)
+	var ps []string
+	if f, err := os.Open(path); err == nil {
+		defer f.Close()
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			s := scanner.Text()
+			if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+				ps = append(ps, s)
+			}
 		}
-	}()
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		return
+	} else if !os.IsNotExist(err) {
+		return nil, err
 	}
-
-	err = out.Sync()
-	if err != nil {
-		return
-	}
-
-	si, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	err = os.Chmod(dst, si.Mode())
-	if err != nil {
-		return
-	}
-
-	return
+	return ps, nil
 }
 
 // ExtractValueFromFile compiles the given file and