fix flaky E2E tests failing

cmd/run_unix.go

@@ -74,6 +74,13 @@ func killProcessGroup(process *os.Process) error {
 		if errors.Is(err, syscall.ESRCH) {
 			return nil // process group gone
 		}
+		// EPERM can occur on macOS when the process group is in a
+		// dying/zombie state. Keep polling rather than returning an
+		// error so we can fall through to SIGKILL if needed.
+		if errors.Is(err, syscall.EPERM) {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
 		return fmt.Errorf("failed to check status of process group %d: %w", pgid, err)
 	}
 	// Grace period elapsed — force kill.

pkg/version/version_test.go

@@ -14,12 +14,15 @@ limitations under the License.
 package version
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"net/http"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetVersionsGithub(t *testing.T) {
@@ -209,7 +212,7 @@ func TestGetVersionsGithub(t *testing.T) {
 		},
 	}
 	m := http.NewServeMux()
-	s := http.Server{Addr: ":12345", Handler: m, ReadHeaderTimeout: time.Duration(5) * time.Second}
+	s := http.Server{Handler: m, ReadHeaderTimeout: time.Duration(5) * time.Second}
 
 	for _, tc := range tests {
 		body := tc.ResponseBody
@@ -218,13 +221,19 @@ func TestGetVersionsGithub(t *testing.T) {
 		})
 	}
 
-	go func() {
-		s.ListenAndServe()
-	}()
+	ln, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		s.Shutdown(ctx)
+	})
+	go func() { s.Serve(ln) }()
+	addr := fmt.Sprintf("http://localhost:%d", ln.Addr().(*net.TCPAddr).Port)
 
 	for _, tc := range tests {
 		t.Run(tc.Name, func(t *testing.T) {
-			version, err := GetLatestReleaseGithub("http://localhost:12345" + tc.Path)
+			version, err := GetLatestReleaseGithub(addr + tc.Path)
 			assert.Equal(t, tc.ExpectedVer, version)
 			if tc.ExpectedErr != "" {
 				assert.EqualError(t, err, tc.ExpectedErr)
@@ -233,18 +242,16 @@ func TestGetVersionsGithub(t *testing.T) {
 	}
 
 	t.Run("error on 404", func(t *testing.T) {
-		version, err := GetLatestReleaseGithub("http://localhost:12345/non-existant/path")
+		version, err := GetLatestReleaseGithub(addr + "/non-existant/path")
 		assert.Equal(t, "", version)
-		assert.EqualError(t, err, "http://localhost:12345/non-existant/path - 404 Not Found")
+		assert.EqualError(t, err, addr+"/non-existant/path - 404 Not Found")
 	})
 
 	t.Run("error on bad addr", func(t *testing.T) {
 		version, err := GetLatestReleaseGithub("http://a.super.non.existant.domain/")
 		assert.Equal(t, "", version)
 		assert.Error(t, err)
 	})
-
-	s.Shutdown(t.Context())
 }
 
 func TestGetVersionsHelm(t *testing.T) {
@@ -318,7 +325,7 @@ entries:
 		},
 	}
 	m := http.NewServeMux()
-	s := http.Server{Addr: ":12346", Handler: m, ReadHeaderTimeout: time.Duration(5) * time.Second}
+	s := http.Server{Handler: m, ReadHeaderTimeout: time.Duration(5) * time.Second}
 
 	for _, tc := range tests {
 		body := tc.ResponseBody
@@ -327,19 +334,23 @@ entries:
 		})
 	}
 
-	go func() {
-		s.ListenAndServe()
-	}()
+	ln, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		s.Shutdown(ctx)
+	})
+	go func() { s.Serve(ln) }()
+	addr := fmt.Sprintf("http://localhost:%d", ln.Addr().(*net.TCPAddr).Port)
 
 	for _, tc := range tests {
 		t.Run(tc.Name, func(t *testing.T) {
-			version, err := GetLatestReleaseHelmChart("http://localhost:12346" + tc.Path)
+			version, err := GetLatestReleaseHelmChart(addr + tc.Path)
 			assert.Equal(t, tc.ExpectedVer, version)
 			if tc.ExpectedErr != "" {
 				assert.EqualError(t, err, tc.ExpectedErr)
 			}
 		})
 	}
-
-	s.Shutdown(t.Context())
 }

tests/e2e/standalone/list_test.go

@@ -86,9 +86,13 @@ func TestStandaloneList(t *testing.T) {
 		cmd := exec.Command(daprdPath, "--app-id", "daprd_e2e_list", "--dapr-http-port", "3555", "--dapr-grpc-port", "4555", "--app-port", "0")
 		cmd.Start()
 
-		output, err := cmdList("")
+		// Wait for daprd to register and appear in the list.
+		var output string
+		require.Eventually(t, func() bool {
+			output, err = cmdList("")
+			return err == nil && !strings.Contains(output, "No Dapr instances found")
+		}, 30*time.Second, time.Second, "daprd instance did not appear in list")
 		t.Log(output)
-		require.NoError(t, err, "dapr list failed with daprd instance")
 		listOutputCheck(t, output, false)
 
 		// TODO: remove this condition when `dapr stop` starts working for Windows.

tests/e2e/standalone/run_template_test.go

@@ -20,6 +20,7 @@ package standalone_test
 import (
 	"fmt"
 	"io/ioutil"
+	"net"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -29,6 +30,23 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// waitForPortsFree polls until all given ports are available for binding.
+// This prevents port contention between sequential subtests that use
+// hardcoded ports.
+func waitForPortsFree(t *testing.T, ports ...int) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		for _, port := range ports {
+			ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
+			if err != nil {
+				return false
+			}
+			ln.Close()
+		}
+		return true
+	}, 30*time.Second, time.Second, "ports %v not available in time", ports)
+}
+
 type AppTestOutput struct {
 	appID                    string
 	appLogContents           []string
@@ -59,18 +77,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 			"-f", runFilePath,
 		}
 
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(25 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 
@@ -122,18 +141,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 			"-f", runFilePath,
 		}
 
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(time.Second * 10):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 
@@ -191,18 +211,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 		args := []string{
 			"-f", runFilePath,
 		}
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(25 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 
@@ -254,18 +275,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 		args := []string{
 			"-f", runFilePath,
 		}
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(25 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 
@@ -318,18 +340,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 		args := []string{
 			"-f", runFilePath,
 		}
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(25 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 
@@ -378,18 +401,19 @@ func TestRunWithTemplateFile(t *testing.T) {
 		args := []string{
 			"-f", runFilePath,
 		}
+		waitForPortsFree(t, 3510, 3511)
 		outputCh := make(chan string)
 		go func() {
 			output, _ := cmdRun("", args...)
 			t.Logf("%s", output)
 			outputCh <- output
 		}()
-		time.Sleep(time.Second * 10)
+		time.Sleep(10 * time.Second)
 		cmdStopWithRunTemplate(runFilePath)
 		var output string
 		select {
 		case output = <-outputCh:
-		case <-time.After(25 * time.Second):
+		case <-time.After(30 * time.Second):
 			t.Fatal("timed out waiting for run command to finish")
 		}
 

tests/e2e/standalone/stop_with_run_template_test.go

@@ -43,7 +43,6 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 			cleanUpLogs()
 		})
 		go ensureAllAppsStartedWithRunTemplate(t)
-		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
 		// Assert dapr list contains template name
 		assertTemplateListOutput(t, "test_dapr_template")
@@ -58,7 +57,6 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 			cleanUpLogs()
 		})
 		go ensureAllAppsStartedWithRunTemplate(t)
-		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
 		output, err := cmdStopWithRunTemplate("../testdata/run-template-files")
 		assert.NoError(t, err, "failed to stop apps started with run template")
@@ -86,7 +84,6 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 			cleanUpLogs()
 		})
 		go ensureAllAppsStartedWithRunTemplate(t)
-		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
 		output, err := cmdStopWithAppID("emit-metrics", "processor")
 		assert.NoError(t, err, "failed to stop apps started with run template")
@@ -124,11 +121,18 @@ func tearDownTestSetup(t *testing.T) {
 }
 
 func getCLIPID(t *testing.T) string {
-	output, err := cmdList("json")
-	require.NoError(t, err, "failed to list apps")
-	result := []map[string]interface{}{}
-	err = json.Unmarshal([]byte(output), &result)
-	assert.Equal(t, 2, len(result))
+	var result []map[string]interface{}
+	require.Eventually(t, func() bool {
+		output, err := cmdList("json")
+		if err != nil {
+			return false
+		}
+		result = nil
+		if err := json.Unmarshal([]byte(output), &result); err != nil {
+			return false
+		}
+		return len(result) == 2
+	}, 30*time.Second, time.Second, "expected 2 apps in list")
 	return fmt.Sprintf("%v", result[0]["cliPid"])
 }
 

tests/e2e/standalone/utils.go

@@ -118,18 +118,19 @@ func executeAgainstRunningDapr(t *testing.T, f func(), daprArgs ...string) {
 
 	err := cmd.Wait()
 	hasAppCommand := !strings.Contains(daprOutput, "WARNING: no application command found")
+	terminatedBySignal := strings.Contains(daprOutput, "terminated signal received: shutting down")
 	if err != nil {
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 &&
 			strings.Contains(daprOutput, "Exited Dapr successfully") &&
-			(!hasAppCommand || strings.Contains(daprOutput, "Exited App successfully")) {
+			(!hasAppCommand || terminatedBySignal || strings.Contains(daprOutput, "Exited App successfully")) {
 			err = nil
 		}
 	}
 	require.NoError(t, err, "dapr didn't exit cleanly")
 	assert.NotContains(t, daprOutput, "The App process exited with error code: exit status", "Stop command should have been called before the app had a chance to exit")
 	assert.Contains(t, daprOutput, "Exited Dapr successfully")
-	if hasAppCommand {
+	if hasAppCommand && !terminatedBySignal {
 		assert.Contains(t, daprOutput, "Exited App successfully")
 	}
 }