From be46f1a7c0cdffee1d07fa2ee8d526f62fa2fac0 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Tue, 21 Feb 2023 20:40:25 +0100 Subject: [PATCH 1/4] ignore excluded paths when transferring files with scp --- .../communicator/ssh/communicator.go | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/sdk-internals/communicator/ssh/communicator.go b/sdk-internals/communicator/ssh/communicator.go index ed46dd89d..3892bb1e6 100644 --- a/sdk-internals/communicator/ssh/communicator.go +++ b/sdk-internals/communicator/ssh/communicator.go @@ -685,7 +685,7 @@ func (c *comm) scpUploadDirSession(dst string, src string, excl []string) error return err } - return scpUploadDir(src, entries, w, r) + return scpUploadDir(src, entries, w, r, excl) } if src[len(src)-1] != '/' { @@ -960,10 +960,24 @@ func scpUploadDirProtocol(name string, w io.Writer, r *bufio.Reader, f func() er return err } -func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader) error { +func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader, excl []string) error { for _, fi := range fs { realPath := filepath.Join(root, fi.Name()) + // Check if this file is excluded + excluded := false + for _, e := range excl { + if e == realPath { + excluded = true + break + } + } + + if excluded { + log.Printf("[DEBUG] SCP: skipping excluded file: %s", realPath) + continue + } + // Track if this is actually a symlink to a directory. If it is // a symlink to a file we don't do any special behavior because uploading // a file just works. If it is a directory, we need to know so we @@ -1015,7 +1029,7 @@ func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader) e return err } - return scpUploadDir(realPath, entries, w, r) + return scpUploadDir(realPath, entries, w, r, excl) }, fi) if err != nil { return err From e2523057769472a9bc915d939a878afd90470881 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Tue, 21 Feb 2023 21:11:00 +0100 Subject: [PATCH 2/4] ignore excluded paths when transferring files with stfp --- .../communicator/ssh/communicator.go | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/sdk-internals/communicator/ssh/communicator.go b/sdk-internals/communicator/ssh/communicator.go index 3892bb1e6..e7c319dbf 100644 --- a/sdk-internals/communicator/ssh/communicator.go +++ b/sdk-internals/communicator/ssh/communicator.go @@ -539,6 +539,12 @@ func (c *comm) sftpUploadDirSession(dst string, src string, excl []string) error if err != nil { return err } + + if excluded := isExcluded(relSrc, excl); excluded { + log.Printf("[DEBUG] SCP: skipping excluded file: %s", relSrc) + return nil + } + finalDst := filepath.Join(rootDst, relSrc) // In Windows, Join uses backslashes which we don't want to get @@ -964,16 +970,7 @@ func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader, e for _, fi := range fs { realPath := filepath.Join(root, fi.Name()) - // Check if this file is excluded - excluded := false - for _, e := range excl { - if e == realPath { - excluded = true - break - } - } - - if excluded { + if excluded := isExcluded(realPath, excl); excluded { log.Printf("[DEBUG] SCP: skipping excluded file: %s", realPath) continue } @@ -1038,3 +1035,25 @@ func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader, e return nil } + +// isExcluded checks if the given file path is excluded by the given list of +// excludes. +// It does shell style matching, so the excludes can contain wildcards. +func isExcluded(filePath string, excludes []string) bool { + // Check if this file is excluded + excluded := false + log.Printf("[DEBUG] SCP: checking if %v is excluded", excludes) + for _, excl := range excludes { + log.Printf("[DEBUG] SCP: checking if %s is excluded by %s", filePath, excl) + match, err := filepath.Match(excl, filePath) + if err != nil { + log.Printf("[DEBUG] SCP: error matching %s to %s: %s", filePath, excl, err) + continue + } + if match { + excluded = true + break + } + } + return excluded +} From d59871b6926dab92ed60f060d577c20b655461af Mon Sep 17 00:00:00 2001 From: Glyphack Date: Tue, 21 Mar 2023 20:47:32 +0100 Subject: [PATCH 3/4] test: isExclude excludes right paths --- .../communicator/ssh/communicator.go | 1 - .../communicator/ssh/communicator_test.go | 31 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/sdk-internals/communicator/ssh/communicator.go b/sdk-internals/communicator/ssh/communicator.go index e7c319dbf..e9a501ef8 100644 --- a/sdk-internals/communicator/ssh/communicator.go +++ b/sdk-internals/communicator/ssh/communicator.go @@ -1042,7 +1042,6 @@ func scpUploadDir(root string, fs []os.FileInfo, w io.Writer, r *bufio.Reader, e func isExcluded(filePath string, excludes []string) bool { // Check if this file is excluded excluded := false - log.Printf("[DEBUG] SCP: checking if %v is excluded", excludes) for _, excl := range excludes { log.Printf("[DEBUG] SCP: checking if %s is excluded by %s", filePath, excl) match, err := filepath.Match(excl, filePath) diff --git a/sdk-internals/communicator/ssh/communicator_test.go b/sdk-internals/communicator/ssh/communicator_test.go index 11d0077c5..65822aac6 100644 --- a/sdk-internals/communicator/ssh/communicator_test.go +++ b/sdk-internals/communicator/ssh/communicator_test.go @@ -230,3 +230,34 @@ func TestHandshakeTimeout(t *testing.T) { t.Fatalf("Expected handshake timeout, got: %s", err) } } + +func TestIsExclude(t *testing.T) { + testCases := []struct { + Exclude []string + Path string + Is bool + }{ + { + Exclude: []string{"foo"}, + Path: "foo", + Is: true, + }, + { + Exclude: []string{"foo/*"}, + Path: "foo/bar", + Is: true, + }, + { + Exclude: []string{"foo"}, + Path: "bar", + Is: false, + }, + } + + for _, tc := range testCases { + if isExcluded(tc.Path, tc.Exclude) != tc.Is { + t.Fatalf("Expected %s to be excluded: %t", tc.Path, tc.Is) + } + } + +} From 9f1628fc250f929e72edaacfdacc2f9e859cf73e Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 19 May 2023 21:35:06 +0200 Subject: [PATCH 4/4] test: improve test case naming --- .../communicator/ssh/communicator_test.go | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sdk-internals/communicator/ssh/communicator_test.go b/sdk-internals/communicator/ssh/communicator_test.go index 65822aac6..9b3e45fd4 100644 --- a/sdk-internals/communicator/ssh/communicator_test.go +++ b/sdk-internals/communicator/ssh/communicator_test.go @@ -233,30 +233,30 @@ func TestHandshakeTimeout(t *testing.T) { func TestIsExclude(t *testing.T) { testCases := []struct { - Exclude []string - Path string - Is bool + Exclude []string + Path string + ExpectExclude bool }{ { - Exclude: []string{"foo"}, - Path: "foo", - Is: true, + Exclude: []string{"foo"}, + Path: "foo", + ExpectExclude: true, }, { - Exclude: []string{"foo/*"}, - Path: "foo/bar", - Is: true, + Exclude: []string{"foo/*"}, + Path: "foo/bar", + ExpectExclude: true, }, { - Exclude: []string{"foo"}, - Path: "bar", - Is: false, + Exclude: []string{"foo"}, + Path: "bar", + ExpectExclude: false, }, } for _, tc := range testCases { - if isExcluded(tc.Path, tc.Exclude) != tc.Is { - t.Fatalf("Expected %s to be excluded: %t", tc.Path, tc.Is) + if isExcluded(tc.Path, tc.Exclude) != tc.ExpectExclude { + t.Fatalf("Expected %s to be excluded: %t", tc.Path, tc.ExpectExclude) } }