From 4fa8e8bbce4cdcbd9a40a226c90c757a509d54cf Mon Sep 17 00:00:00 2001 From: Japsty Date: Tue, 15 Oct 2024 16:40:00 +0300 Subject: [PATCH] pack: added .packignore In some cases, there are files that may be useful, but should not be part of artifact. The ability to add files and directories to the .packignore file has been added, which allows you to ignore these files and directories when packing. Closes: [812](https://github.com/tarantool/tt/issues/812) --- cli/pack/common.go | 38 ++++++- cli/pack/common_test.go | 20 ++++ cli/pack/ignore.go | 100 ++++++++++++++++++ cli/pack/ignore_test.go | 63 +++++++++++ .../bundle_with_packignore/.packignore | 4 + .../testdata/bundle_with_packignore/app.lua | 0 cli/pack/testdata/bundle_with_packignore/app2 | 1 + .../bundle_with_packignore/instances_enabled | 1 + .../testdata/bundle_with_packignore/modules | 1 + .../testdata/bundle_with_packignore/tt.yaml | 11 ++ 10 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 cli/pack/ignore.go create mode 100644 cli/pack/ignore_test.go create mode 100644 cli/pack/testdata/bundle_with_packignore/.packignore create mode 100644 cli/pack/testdata/bundle_with_packignore/app.lua create mode 120000 cli/pack/testdata/bundle_with_packignore/app2 create mode 120000 cli/pack/testdata/bundle_with_packignore/instances_enabled create mode 120000 cli/pack/testdata/bundle_with_packignore/modules create mode 100644 cli/pack/testdata/bundle_with_packignore/tt.yaml diff --git a/cli/pack/common.go b/cli/pack/common.go index 3f352da37..3d62bd377 100644 --- a/cli/pack/common.go +++ b/cli/pack/common.go @@ -277,14 +277,30 @@ func getDestAppDir(bundleEnvPath, appName string, // copyApplications copies applications from current env to the result bundle. func copyApplications(bundleEnvPath string, packCtx *PackCtx, - cliOpts, newOpts *config.CliOpts) error { - var err error + cliOpts, newOpts *config.CliOpts, toIgnore map[string]struct{}) error { for appName, instances := range packCtx.AppsInfo { if len(instances) == 0 { return fmt.Errorf("application %q does not have any instances", appName) } inst := instances[0] appPath := inst.AppDir + + projectPath := filepath.Dir(packCtx.configFilePath) + relAppPath, err := filepath.Rel(projectPath, appPath) + if err != nil { + return err + } + relAppPathUnix := filepath.ToSlash(relAppPath) + + ignore, err := shouldIgnore(relAppPathUnix, toIgnore) + if err != nil { + return err + } + if ignore { + log.Infof("Application %s found in .packignore, skipping it...", appName) + continue + } + if inst.IsFileApp { appPath = inst.InstanceScript resolvedAppPath, err := filepath.EvalSymlinks(appPath) @@ -363,6 +379,12 @@ func prepareBundle(cmdCtx *cmdcontext.CmdCtx, packCtx *PackCtx, packCtx.AppList = getAppNamesToPack(packCtx) log.Infof("Apps to pack: %s", strings.Join(packCtx.AppList, " ")) + projectPath := filepath.Dir(packCtx.configFilePath) + ignorePatterns, err := readPackIgnore(projectPath) + if err != nil { + return "", fmt.Errorf("failed to read .packignore: %v", err) + } + if bundleEnvPath, err = updateEnvPath(bundleEnvPath, packCtx, cliOpts); err != nil { return "", err } @@ -373,16 +395,22 @@ func prepareBundle(cmdCtx *cmdcontext.CmdCtx, packCtx *PackCtx, return "", fmt.Errorf("error copying binaries: %s", err) } - if err = copyApplications(bundleEnvPath, packCtx, cliOpts, newOpts); err != nil { + if err = copyApplications(bundleEnvPath, packCtx, cliOpts, + newOpts, ignorePatterns); err != nil { return "", fmt.Errorf("error copying applications: %s", err) } if packCtx.Archive.All { - if err = copyArtifacts(*packCtx, bundleEnvPath, newOpts, packCtx.AppsInfo); err != nil { + if err = copyArtifacts(*packCtx, bundleEnvPath, newOpts, + packCtx.AppsInfo, ignorePatterns); err != nil { return "", fmt.Errorf("failed copying artifacts: %s", err) } } + if err = removeIgnoredFiles(bundleEnvPath, ignorePatterns); err != nil { + return "", fmt.Errorf("failed to remove ignored files: %v", err) + } + if buildRocks { err = buildAppRocks(cmdCtx, packCtx, cliOpts, bundleEnvPath) if err != nil && !os.IsNotExist(err) { @@ -425,7 +453,7 @@ func copyAppSrc(packCtx *PackCtx, cliOpts *config.CliOpts, srcAppPath, dstAppPat // copyArtifacts copies all artifacts from the current bundle configuration // to the passed package structure from the passed path. func copyArtifacts(packCtx PackCtx, basePath string, newOpts *config.CliOpts, - appsInfo map[string][]running.InstanceCtx) error { + appsInfo map[string][]running.InstanceCtx, toIgnore map[string]struct{}) error { for _, appName := range packCtx.AppList { for _, inst := range appsInfo[appName] { diff --git a/cli/pack/common_test.go b/cli/pack/common_test.go index e7fc81b68..f5788dc9c 100644 --- a/cli/pack/common_test.go +++ b/cli/pack/common_test.go @@ -1068,6 +1068,26 @@ func Test_prepareBundle(t *testing.T) { {assert.FileExists, "app/tt.yaml"}, }, }, + { + name: "Packing env with packignore:.", + params: params{ + configPath: "testdata/bundle_with_packignore/tt.yaml", + tntExecutable: tntExecutable, + packCtx: PackCtx{Name: "app"}, + }, + wantErr: false, + checks: []check{ + {assert.DirExists, "instances.enabled"}, + {assert.NoFileExists, "instances.enabled/app"}, + + {assert.NoDirExists, "modules"}, + + {assert.DirExists, "app2"}, + {assert.NoDirExists, "app2/var"}, + + {assert.NoFileExists, "app.lua"}, + }, + }, } for _, tt := range tests { diff --git a/cli/pack/ignore.go b/cli/pack/ignore.go new file mode 100644 index 000000000..a881a2dc6 --- /dev/null +++ b/cli/pack/ignore.go @@ -0,0 +1,100 @@ +package pack + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" +) + +// readPackIgnore reads the .packignore file and returns a slice of ignore patterns. +func readPackIgnore(projectPath string) (map[string]struct{}, error) { + ignoreFilePath := filepath.Join(projectPath, ".packignore") + file, err := os.Open(ignoreFilePath) + if err != nil { + if os.IsNotExist(err) { + return map[string]struct{}{}, nil + } + return nil, err + } + defer file.Close() + + patterns := make(map[string]struct{}) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + patterns[line] = struct{}{} + } + + if err := scanner.Err(); err != nil { + return nil, err + } + return patterns, nil +} + +// shouldIgnore checks if the given file path matches any of the ignore patterns. +func shouldIgnore(path string, patterns map[string]struct{}) (bool, error) { + for pattern := range patterns { + pattern = filepath.ToSlash(pattern) + filePath := filepath.ToSlash(path) + + if strings.HasSuffix(pattern, "/") { + if strings.HasPrefix(filePath, pattern) { + return true, nil + } + continue + } + + match, err := filepath.Match(pattern, filePath) + if err != nil { + return false, err + } + if match { + return true, nil + } + } + return false, nil +} + +// removeIgnoredFiles walks through the bundle directory and removes files or directories +// that match the ignore patterns. +func removeIgnoredFiles(bundleEnvPath string, patterns map[string]struct{}) error { + return filepath.Walk(bundleEnvPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(bundleEnvPath, path) + if err != nil { + return err + } + + relPathUnix := filepath.ToSlash(relPath) + + ignore, err := shouldIgnore(relPathUnix, patterns) + if err != nil { + return err + } + + if ignore { + if info.IsDir() { + err = os.RemoveAll(path) + if err != nil { + return fmt.Errorf("failed to remove directory %q: %v", path, err) + } + return filepath.SkipDir + } else { + err = os.Remove(path) + if err != nil { + return fmt.Errorf("failed to remove file %q: %v", path, err) + } + } + } + return nil + }) +} diff --git a/cli/pack/ignore_test.go b/cli/pack/ignore_test.go new file mode 100644 index 000000000..e510eff61 --- /dev/null +++ b/cli/pack/ignore_test.go @@ -0,0 +1,63 @@ +package pack + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPackIgnore(t *testing.T) { + projectDir, err := os.MkdirTemp("", "test") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(projectDir) + + packignoreFile := ` + .txt + logs/ + secret.key + temp* + *.bak + ` + packignorePath := filepath.Join(projectDir, ".packignore") + err = os.WriteFile(packignorePath, []byte(packignoreFile), 0644) + if err != nil { + t.Fatalf("Failed to write .packignore file: %v", err) + } + + patterns, err := readPackIgnore(projectDir) + if err != nil { + t.Fatalf("Failed to read .packignore: %v", err) + } + + tests := []struct { + path string + expected bool + }{ + {"file.txt", true}, + {"document.txt", true}, + {"image.png", false}, + {"logs/error.log", true}, + {"logs/", true}, + {"logs", false}, + {"secret.key", true}, + {"config.yaml", false}, + {"tempfile", true}, + {"temporary", true}, + {"template", true}, + {"data.bak", true}, + {"backup.bak", true}, + {"main.go", false}, + } + + for _, test := range tests { + ignore, err := shouldIgnore(test.path, patterns) + if err != nil { + t.Errorf("Error in shouldIgnore for path %q: %v", test.path, err) + } + if ignore != test.expected { + t.Errorf("For path %q, expected ignore=%v, got %v", test.path, test.expected, ignore) + } + } +} diff --git a/cli/pack/testdata/bundle_with_packignore/.packignore b/cli/pack/testdata/bundle_with_packignore/.packignore new file mode 100644 index 000000000..e198e0c08 --- /dev/null +++ b/cli/pack/testdata/bundle_with_packignore/.packignore @@ -0,0 +1,4 @@ +instances.enabled/app +modules +app2/var +app.lua \ No newline at end of file diff --git a/cli/pack/testdata/bundle_with_packignore/app.lua b/cli/pack/testdata/bundle_with_packignore/app.lua new file mode 100644 index 000000000..e69de29bb diff --git a/cli/pack/testdata/bundle_with_packignore/app2 b/cli/pack/testdata/bundle_with_packignore/app2 new file mode 120000 index 000000000..deebc8de1 --- /dev/null +++ b/cli/pack/testdata/bundle_with_packignore/app2 @@ -0,0 +1 @@ +../../../../test/integration/pack/test_bundles/bundle1/app2 \ No newline at end of file diff --git a/cli/pack/testdata/bundle_with_packignore/instances_enabled b/cli/pack/testdata/bundle_with_packignore/instances_enabled new file mode 120000 index 000000000..c02d4c2d9 --- /dev/null +++ b/cli/pack/testdata/bundle_with_packignore/instances_enabled @@ -0,0 +1 @@ +../../../../test/integration/pack/test_bundles/bundle1/instances_enabled \ No newline at end of file diff --git a/cli/pack/testdata/bundle_with_packignore/modules b/cli/pack/testdata/bundle_with_packignore/modules new file mode 120000 index 000000000..e7b46c2b3 --- /dev/null +++ b/cli/pack/testdata/bundle_with_packignore/modules @@ -0,0 +1 @@ +../../../../test/integration/pack/test_bundles/bundle1/modules \ No newline at end of file diff --git a/cli/pack/testdata/bundle_with_packignore/tt.yaml b/cli/pack/testdata/bundle_with_packignore/tt.yaml new file mode 100644 index 000000000..fae23cd96 --- /dev/null +++ b/cli/pack/testdata/bundle_with_packignore/tt.yaml @@ -0,0 +1,11 @@ +env: + bin_dir: bin + inc_dir: include + instances_enabled: . + tarantoolctl_layout: false +app: + run_dir: var/run + log_dir: var/log + wal_dir: var/lib + memtx_dir: var/lib + vinyl_dir: var/lib