diff --git a/cmd/buf/buf.go b/cmd/buf/buf.go index 317514940a..47543b33a4 100644 --- a/cmd/buf/buf.go +++ b/cmd/buf/buf.go @@ -116,6 +116,7 @@ import ( "github.com/bufbuild/buf/cmd/buf/internal/command/registry/sdk/sdkinfo" "github.com/bufbuild/buf/cmd/buf/internal/command/registry/sdk/version" "github.com/bufbuild/buf/cmd/buf/internal/command/registry/whoami" + "github.com/bufbuild/buf/cmd/buf/internal/command/scaffold" "github.com/bufbuild/buf/cmd/buf/internal/command/source/sourceedit/sourceeditdeprecate" "github.com/bufbuild/buf/cmd/buf/internal/command/stats" "github.com/bufbuild/buf/private/buf/bufcli" @@ -157,6 +158,7 @@ func newRootCommand(name string) *appcmd.Command { push.NewCommand("push", builder), convert.NewCommand("convert", builder), curl.NewCommand("curl", builder), + scaffold.NewCommand("scaffold", builder), { Use: "dep", Short: "Work with dependencies", diff --git a/cmd/buf/internal/command/scaffold/scaffold.go b/cmd/buf/internal/command/scaffold/scaffold.go new file mode 100644 index 0000000000..4ccfe77d3f --- /dev/null +++ b/cmd/buf/internal/command/scaffold/scaffold.go @@ -0,0 +1,223 @@ +// Copyright 2020-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + + "buf.build/go/app/appcmd" + "buf.build/go/app/appext" + "buf.build/go/standard/xslices" + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/bufpkg/bufconfig" + "github.com/bufbuild/protocompile/experimental/parser" + "github.com/bufbuild/protocompile/experimental/report" + "github.com/bufbuild/protocompile/experimental/source" +) + +// NewCommand returns a new scaffold Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + return &appcmd.Command{ + Use: name, + Short: "Scaffold buf configuration by analyzing Protobuf files in a git repository", + Long: `This command generates a buf.yaml with the correct module roots by analyzing .proto files. + +The first argument is the root of the git repository to scaffold. +Defaults to "." if no argument is specified. + +If a buf.yaml already exists, this command will not overwrite it, and will produce an error.`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container) + }, + ), + } +} + +func run( + ctx context.Context, + container appext.Container, +) error { + dirPath := "." + if container.NumArgs() > 0 { + dirPath = container.Arg(0) + } + if _, err := os.Stat(filepath.Join(dirPath, ".git")); err != nil { + return fmt.Errorf("%s is not the root of a git repository", dirPath) + } + exists, err := bufcli.BufYAMLFileExistsForDirPath(ctx, dirPath) + if err != nil { + return err + } + if exists { + return fmt.Errorf("buf.yaml already exists in %s, will not overwrite", dirPath) + } + fsys := os.DirFS(dirPath) + fileInfos, err := walkAndParseProtoFiles(fsys) + if err != nil { + return err + } + if len(fileInfos) == 0 { + return errors.New("no .proto files found") + } + moduleRoots := inferModuleRoots(fileInfos) + if len(moduleRoots) == 0 { + return errors.New("could not determine module roots from .proto files") + } + moduleConfigs, err := buildModuleConfigs(moduleRoots) + if err != nil { + return err + } + bufYAMLFile, err := bufconfig.NewBufYAMLFile( + bufconfig.FileVersionV2, + moduleConfigs, + nil, + nil, + nil, + bufconfig.BufYAMLFileWithIncludeDocsLink(), + ) + if err != nil { + return err + } + return bufcli.PutBufYAMLFileForDirPath(ctx, dirPath, bufYAMLFile) +} + +func buildModuleConfigs(moduleRoots []string) ([]bufconfig.ModuleConfig, error) { + moduleConfigs := make([]bufconfig.ModuleConfig, 0, len(moduleRoots)) + for _, root := range moduleRoots { + moduleConfig, err := bufconfig.NewModuleConfig( + root, + nil, + map[string][]string{".": {}}, + map[string][]string{".": {}}, + bufconfig.DefaultLintConfigV2, + bufconfig.DefaultBreakingConfigV2, + ) + if err != nil { + return nil, err + } + moduleConfigs = append(moduleConfigs, moduleConfig) + } + return moduleConfigs, nil +} + +type protoFileInfo struct { + filePath string + packageName string +} + +// walkAndParseProtoFiles walks the filesystem for .proto files and parses each one. +// Files that fail to parse or have no package declaration are silently skipped. +func walkAndParseProtoFiles(fsys fs.FS) ([]protoFileInfo, error) { + var fileInfos []protoFileInfo + err := fs.WalkDir(fsys, ".", func(filePath string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || path.Ext(filePath) != ".proto" { + return nil + } + data, err := fs.ReadFile(fsys, filePath) + if err != nil { + return err + } + sourceFile := source.NewFile(filePath, string(data)) + astFile, ok := parser.Parse(filePath, sourceFile, new(report.Report)) + if !ok || astFile == nil { + return nil + } + pkg := astFile.Package() + if pkg.IsZero() { + return nil + } + packagePath := pkg.Path() + if packagePath.IsZero() { + return nil + } + fileInfos = append(fileInfos, protoFileInfo{ + filePath: astFile.Path(), + packageName: packagePath.Canonicalized(), + }) + return nil + }) + if err != nil { + return nil, err + } + return fileInfos, nil +} + +// inferModuleRoots determines module root directories by deriving each file's +// expected import path from its package declaration, then comparing against its +// file path. The difference is the module root. +// +// Returns sorted, unique module root paths. Returns nil if no roots can be inferred. +func inferModuleRoots(files []protoFileInfo) []string { + rootMap := make(map[string]struct{}) + for _, file := range files { + packageDir := strings.ReplaceAll(file.packageName, ".", "/") + fileName := path.Base(file.filePath) + expectedImportPath := packageDir + "/" + fileName + prefix := moduleRootPrefix(file.filePath, expectedImportPath) + if prefix != "" { + rootMap[prefix] = struct{}{} + } + } + if len(rootMap) == 0 { + return nil + } + // "." means the entire repo is a single module, no other root can coexist. + if _, ok := rootMap["."]; ok { + return []string{"."} + } + roots := xslices.MapKeysToSortedSlice(rootMap) + // Remove roots that are children of other roots. A module root + // like "proto" already includes all files under "proto/internal", + // so "proto/internal" as a separate root would be invalid. + // After sorting, a parent always appears before its children. + var filtered []string + for _, root := range roots { + if !slices.ContainsFunc(filtered, func(parent string) bool { + return strings.HasPrefix(root, parent+"/") + }) { + filtered = append(filtered, root) + } + } + return filtered +} + +// moduleRootPrefix returns the prefix of filePath before importPath, +// or "" if filePath does not end with importPath. +func moduleRootPrefix(filePath, importPath string) string { + if filePath == importPath { + return "." + } + prefix, found := strings.CutSuffix(filePath, "/"+importPath) + if found { + return prefix + } + return "" +} diff --git a/cmd/buf/internal/command/scaffold/scaffold_test.go b/cmd/buf/internal/command/scaffold/scaffold_test.go new file mode 100644 index 0000000000..caf69f3988 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/scaffold_test.go @@ -0,0 +1,101 @@ +// Copyright 2020-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scaffold + +import ( + "os" + "path/filepath" + "testing" + + "buf.build/go/app/appcmd" + "buf.build/go/app/appcmd/appcmdtesting" + "buf.build/go/app/appext" + "github.com/bufbuild/buf/private/pkg/storage" + "github.com/bufbuild/buf/private/pkg/storage/storageos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScaffoldSingleRoot(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/single_root", 0, "") +} + +func TestScaffoldMultiRoot(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/multi_root", 0, "") +} + +func TestScaffoldRootModule(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/root_module", 0, "") +} + +func TestScaffoldNoProtoFiles(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/no_proto_files", 1, "no .proto files found") +} + +func TestScaffoldNestedRoots(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/nested_roots", 0, "") +} + +func TestScaffoldExistingBufYAML(t *testing.T) { + t.Parallel() + testScaffold(t, "testdata/existing_buf_yaml", 1, "buf.yaml already exists") +} + +func TestScaffoldNotGitRoot(t *testing.T) { + t.Parallel() + testScaffoldWithOptions(t, "testdata/not_git_root", 1, "is not the root of a git repository", false) +} + +func testScaffold(t *testing.T, dir string, expectCode int, expectStderr string) { + t.Helper() + testScaffoldWithOptions(t, dir, expectCode, expectStderr, true) +} + +func testScaffoldWithOptions(t *testing.T, dir string, expectCode int, expectStderr string, createGitDir bool) { + t.Helper() + storageosProvider := storageos.NewProvider() + inputBucket, err := storageosProvider.NewReadWriteBucket(filepath.Join(dir, "input")) + require.NoError(t, err) + tempDir := t.TempDir() + tempBucket, err := storageosProvider.NewReadWriteBucket(tempDir) + require.NoError(t, err) + _, err = storage.Copy(t.Context(), inputBucket, tempBucket) + require.NoError(t, err) + if createGitDir { + require.NoError(t, os.Mkdir(filepath.Join(tempDir, ".git"), 0o755)) + } + appcmdtesting.Run( + t, + func(use string) *appcmd.Command { + return NewCommand(use, appext.NewBuilder(use)) + }, + appcmdtesting.WithExpectedExitCode(expectCode), + appcmdtesting.WithExpectedStderrPartials(expectStderr), + appcmdtesting.WithArgs(tempDir), + ) + if expectCode != 0 { + return + } + got, err := os.ReadFile(filepath.Join(tempDir, "buf.yaml")) + require.NoError(t, err) + expected, err := os.ReadFile(filepath.Join(dir, "output", "buf.yaml")) + require.NoError(t, err) + assert.Equal(t, string(expected), string(got)) +} diff --git a/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/buf.yaml b/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/buf.yaml new file mode 100644 index 0000000000..ba5ddf5078 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/buf.yaml @@ -0,0 +1,3 @@ +version: v2 +modules: + - path: proto diff --git a/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/proto/foo/v1/foo.proto b/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/proto/foo/v1/foo.proto new file mode 100644 index 0000000000..403aebbdee --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/existing_buf_yaml/input/proto/foo/v1/foo.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package foo.v1; + +message Foo {} diff --git a/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-a/a/a.proto b/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-a/a/a.proto new file mode 100644 index 0000000000..bc894d5c37 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-a/a/a.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package a; + +message A {} diff --git a/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-b/b/b.proto b/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-b/b/b.proto new file mode 100644 index 0000000000..867d0a7f4a --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/multi_root/input/module-b/b/b.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package b; + +message B {} diff --git a/cmd/buf/internal/command/scaffold/testdata/multi_root/output/buf.yaml b/cmd/buf/internal/command/scaffold/testdata/multi_root/output/buf.yaml new file mode 100644 index 0000000000..8486cf7688 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/multi_root/output/buf.yaml @@ -0,0 +1,5 @@ +# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml +version: v2 +modules: + - path: module-a + - path: module-b diff --git a/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/bar/bar.proto b/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/bar/bar.proto new file mode 100644 index 0000000000..c6b4dbee2f --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/bar/bar.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package bar; + +message Bar {} diff --git a/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/internal/foo/foo.proto b/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/internal/foo/foo.proto new file mode 100644 index 0000000000..d2b2ff478d --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/nested_roots/input/proto/internal/foo/foo.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package foo; + +message Foo {} diff --git a/cmd/buf/internal/command/scaffold/testdata/nested_roots/output/buf.yaml b/cmd/buf/internal/command/scaffold/testdata/nested_roots/output/buf.yaml new file mode 100644 index 0000000000..afda6482bd --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/nested_roots/output/buf.yaml @@ -0,0 +1,4 @@ +# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml +version: v2 +modules: + - path: proto diff --git a/cmd/buf/internal/command/scaffold/testdata/no_proto_files/input/README.md b/cmd/buf/internal/command/scaffold/testdata/no_proto_files/input/README.md new file mode 100644 index 0000000000..ec258ae00f --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/no_proto_files/input/README.md @@ -0,0 +1 @@ +This directory intentionally contains no .proto files. diff --git a/cmd/buf/internal/command/scaffold/testdata/not_git_root/input/foo.proto b/cmd/buf/internal/command/scaffold/testdata/not_git_root/input/foo.proto new file mode 100644 index 0000000000..d2b2ff478d --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/not_git_root/input/foo.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package foo; + +message Foo {} diff --git a/cmd/buf/internal/command/scaffold/testdata/root_module/input/foo/v1/foo.proto b/cmd/buf/internal/command/scaffold/testdata/root_module/input/foo/v1/foo.proto new file mode 100644 index 0000000000..403aebbdee --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/root_module/input/foo/v1/foo.proto @@ -0,0 +1,5 @@ +syntax = "proto3"; + +package foo.v1; + +message Foo {} diff --git a/cmd/buf/internal/command/scaffold/testdata/root_module/output/buf.yaml b/cmd/buf/internal/command/scaffold/testdata/root_module/output/buf.yaml new file mode 100644 index 0000000000..77d7ce0572 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/root_module/output/buf.yaml @@ -0,0 +1,2 @@ +# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml +version: v2 diff --git a/cmd/buf/internal/command/scaffold/testdata/single_root/input/proto/foo/v1/foo.proto b/cmd/buf/internal/command/scaffold/testdata/single_root/input/proto/foo/v1/foo.proto new file mode 100644 index 0000000000..8030657e46 --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/single_root/input/proto/foo/v1/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo.v1; + +message Foo { + string name = 1; +} diff --git a/cmd/buf/internal/command/scaffold/testdata/single_root/output/buf.yaml b/cmd/buf/internal/command/scaffold/testdata/single_root/output/buf.yaml new file mode 100644 index 0000000000..afda6482bd --- /dev/null +++ b/cmd/buf/internal/command/scaffold/testdata/single_root/output/buf.yaml @@ -0,0 +1,4 @@ +# For details on buf.yaml configuration, visit https://buf.build/docs/configuration/v2/buf-yaml +version: v2 +modules: + - path: proto