Skip to content
Draft
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
b4edae2
Phase 1 OTEL: add OTEL_COLLECTOR agent type to Inventory API
theTibi Mar 3, 2026
2360ed6
Phase 1 OTEL: models, settings, converters, inventory AddOtelCollector
theTibi Mar 3, 2026
87e7afa
Phase 1 OTEL: agent state – otel-collector config builder and state case
theTibi Mar 3, 2026
d0a9531
Phase 1 OTEL: server – supervisord, nginx OTLP, ClickHouse schema
theTibi Mar 3, 2026
5fea0cc
Phase 1 OTEL: pmm-agent – config path and supervisor process for OTEL…
theTibi Mar 3, 2026
dfd9910
Phase 1 OTEL: pmm-admin add otel command
theTibi Mar 3, 2026
69562c0
Phase 1 OTEL: Grafana ClickHouse-OTEL datasource and dev docs
theTibi Mar 3, 2026
a5e4afc
Implement OtelCollector in the agent inventory and API
theTibi Mar 3, 2026
ecd9cfc
Add OtelCollector path to configuration tests
theTibi Mar 3, 2026
81a8f10
Refactor OTEL configuration and enhance schema management
theTibi Mar 3, 2026
d35589c
Refactor log file path checks in OTEL collector configuration
theTibi Mar 3, 2026
a6b0886
Enable OTEL Collector by default and improve parameter validation in …
theTibi Mar 4, 2026
f979e02
Enhance OTEL schema management for ClickHouse integration
theTibi Mar 4, 2026
f7d3d4c
Add otelcol-contrib binary and update installation scripts
theTibi Mar 5, 2026
8578248
Improve binary extraction process in build-client-binary script
theTibi Mar 5, 2026
9f64b95
Add OTEL Collector configuration and tests
theTibi Mar 5, 2026
5485877
Add otel-collector directory and update configuration paths
theTibi Mar 5, 2026
91ec6dc
Enhance OTEL Collector log source management and validation
theTibi Mar 5, 2026
9572312
Update OTEL Collector configuration and database schema
theTibi Mar 6, 2026
db9cee0
Enhance OTEL Collector configuration with resource attributes
theTibi Mar 6, 2026
aa1d6ed
Enhance OTEL log collection configuration and presets
theTibi Mar 6, 2026
449a3bd
Enhance OTEL Collector configuration and log source management
theTibi Mar 6, 2026
72d2721
Add pmm-managed binary and refactor setup logic
theTibi Mar 6, 2026
658f90d
Refactor setup logic and enhance configuration update handling
theTibi Mar 7, 2026
120606b
Merge origin/v3 into tibi-test
theTibi Mar 16, 2026
0a762b7
Add support for Otel Collectors in agent listing and management
theTibi Mar 16, 2026
1b4a4fb
Merge branch 'v3' into tibi-test
theTibi Mar 26, 2026
9e2823b
Add eBPF telemetry command and update dependencies
theTibi Mar 28, 2026
7371d21
fix(otel): repair swagger client imports, add_otel labels, retention …
theTibi Mar 28, 2026
494e6b9
build(admin): verify swagger client *_parameters.go before pmm-admin …
theTibi Mar 28, 2026
6fb97fc
feat(otel): add OpenTelemetry Collector Contrib version to vars
theTibi Mar 28, 2026
17f3fff
refactor(telemetry): update eBPF telemetry command and ClickHouse exp…
theTibi Mar 29, 2026
65d3dfb
refactor(otel): replace AddOtelCommand with structured subcommands fo…
theTibi Mar 29, 2026
dca6865
refactor(otel): enhance baseOtelConfigYaml function for improved pipe…
theTibi Mar 29, 2026
65e02f2
feat(otel): add TODOs for OTEL command removal and management
theTibi Mar 30, 2026
b34d2e1
feat(otel): implement log parser preset management in API
theTibi Mar 30, 2026
1db9088
Merge branch 'v3' into tibi-test
theTibi Apr 2, 2026
6a44b81
fix(build): sync go.sum for OTLP/OpenTelemetry deps (cenkalti/backoff)
theTibi Apr 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions admin/commands/inventory/list_agents.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ var acceptableAgentTypes = map[string][]string{
types.AgentTypeQANPostgreSQLPgStatMonitorAgent: {types.AgentTypeName(types.AgentTypeQANPostgreSQLPgStatMonitorAgent), "qan-postgresql-pgstatmonitor-agent"},
types.AgentTypeRDSExporter: {types.AgentTypeName(types.AgentTypeRDSExporter), "rds-exporter"},
types.AgentTypeRTAMongoDBAgent: {types.AgentTypeName(types.AgentTypeRTAMongoDBAgent), "rta-mongodb-agent"},
types.AgentTypeOtelCollector: {types.AgentTypeName(types.AgentTypeOtelCollector), "otel-collector"},
}

type listResultAgent struct {
Expand Down Expand Up @@ -283,6 +284,18 @@ func (cmd *ListAgentsCommand) RunCmd() (commands.Result, error) {
})
}

for _, a := range agentsRes.Payload.OtelCollector {
agentsList = append(agentsList, listResultAgent{
AgentType: types.AgentTypeOtelCollector,
AgentID: a.AgentID,
PMMAgentID: a.PMMAgentID,
ServiceID: "",
Status: getAgentStatus(a.Status),
Disabled: a.Disabled,
Port: 0,
})
}

return &listAgentsResult{
Agents: agentsList,
}, nil
Expand Down
19 changes: 19 additions & 0 deletions admin/commands/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ func agentsList(agentsRes *agents.ListAgentsOK, nodeID string) []listResultAgent
agentsList = append(agentsList, vmAgents(agentsRes, pmmAgentIDs)...)
agentsList = append(agentsList, nomadAgents(agentsRes, pmmAgentIDs)...)
agentsList = append(agentsList, rtaMongodbAgents(agentsRes, pmmAgentIDs)...)
agentsList = append(agentsList, otelCollectors(agentsRes, pmmAgentIDs)...)

return agentsList
}
Expand Down Expand Up @@ -619,3 +620,21 @@ func rtaMongodbAgents(agentsRes *agents.ListAgentsOK, pmmAgentIDs map[string]str

return agentsList
}

func otelCollectors(agentsRes *agents.ListAgentsOK, pmmAgentIDs map[string]struct{}) []listResultAgent {
var agentsList []listResultAgent
for _, a := range agentsRes.Payload.OtelCollector {
if _, ok := pmmAgentIDs[a.PMMAgentID]; ok {
agentsList = append(agentsList, listResultAgent{
AgentType: types.AgentTypeOtelCollector,
AgentID: a.AgentID,
ServiceID: "",
Status: getStatus(a.Status),
Disabled: a.Disabled,
MetricsMode: "",
Port: 0,
})
}
}
return agentsList
}
1 change: 1 addition & 0 deletions admin/commands/management/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type AddCommand struct {
PostgreSQL AddPostgreSQLCommand `cmd:"" name:"postgresql" help:"Add PostgreSQL to monitoring"`
Valkey AddValkeyCommand `cmd:"" name:"valkey" help:"Add Valkey to monitoring"`
ProxySQL AddProxySQLCommand `cmd:"" name:"proxysql" help:"Add ProxySQL to monitoring"`
Otel AddOtelCommand `cmd:"" name:"otel" help:"Add OTEL Collector for log collection (and later traces, profiles)"`
}

// AddCommonFlags is used by Kong for CLI flags and commands.
Expand Down
123 changes: 123 additions & 0 deletions admin/commands/management/add_otel.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Copyright (C) 2023 Percona LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package management

import (
"strings"

"github.com/percona/pmm/admin/agentlocal"
"github.com/percona/pmm/admin/commands"
"github.com/percona/pmm/api/inventory/v1/json/client"
agents "github.com/percona/pmm/api/inventory/v1/json/client/agents_service"
)

var addOtelResultT = commands.ParseTemplate(`
OTEL Collector added.
Agent ID : {{ .Agent.AgentID }}
PMM-Agent ID : {{ .Agent.PMMAgentID }}
Status : {{ .Agent.Status }}
Disabled : {{ .Agent.Disabled }}
`)

type addOtelResult struct {
Agent *agents.AddAgentOKBodyOtelCollector `json:"otel_collector"`
}

func (res *addOtelResult) Result() {}

func (res *addOtelResult) String() string {
return commands.RenderTemplate(addOtelResultT, res)
}

// AddOtelCommand is used by Kong for CLI flags and commands.
type AddOtelCommand struct {
PMMAgentID string `help:"Node ID where pmm-agent runs (default is autodetected)"`
LogFilePaths []string `name:"log-file-paths" help:"Comma-separated list of log file paths to collect (e.g. /var/log/mysql/error.log). Used with --parser-preset or as raw if --log-sources not set."`
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
The line is 210 characters long, which exceeds the maximum of 170 characters. (lll)

LogSources string `name:"log-sources" help:"Comma-separated path:preset pairs (e.g. /var/log/mysql/error.log:mysql_error,/other.log:raw). Preset 'raw' means no parsing. Available presets: mysql_error, nginx_access, nginx_error, grafana, pmm_managed, pmm_agent, postgres, raw. Overrides --log-file-paths and --parser-preset."`
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
The line is 342 characters long, which exceeds the maximum of 170 characters. (lll)

ParserPreset string `name:"parser-preset" help:"Parser preset for all paths from --log-file-paths. Available presets: mysql_error, nginx_access, nginx_error, grafana, pmm_managed, pmm_agent, postgres, raw. Ignored if --log-sources is set."`
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
The line is 255 characters long, which exceeds the maximum of 170 characters. (lll)

CustomLabels map[string]string `mapsep:"," help:"Custom user-assigned labels"`
}

// RunCmd runs the command for AddOtelCommand.
func (cmd *AddOtelCommand) RunCmd() (commands.Result, error) {
pmmAgentID := cmd.PMMAgentID
if pmmAgentID == "" {
status, err := agentlocal.GetStatus(agentlocal.DoNotRequestNetworkInfo)
if err != nil {
return nil, err
}
pmmAgentID = status.AgentID
}

customLabels := commands.ParseKeyValuePair(cmd.CustomLabels)

body := &agents.AddAgentParamsBodyOtelCollector{
PMMAgentID: pmmAgentID,
CustomLabels: customLabels,
}
if cmd.LogSources != "" {
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
if cmd.LogSources != "" has complex nested blocks (complexity: 10) (nestif)

// Parse path:preset pairs.
for _, pair := range strings.Split(cmd.LogSources, ",") {
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
stringsseq: Ranging over SplitSeq is more efficient (modernize)

pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
path := pair
preset := "raw"
if idx := strings.Index(pair, ":"); idx >= 0 {
path = strings.TrimSpace(pair[:idx])
preset = strings.TrimSpace(pair[idx+1:])
if preset == "" {
preset = "raw"
}
}
if path != "" {
body.LogSources = append(body.LogSources, &agents.AddAgentParamsBodyOtelCollectorLogSourcesItems0{
Path: path,
Preset: preset,
})
}
}
} else if len(cmd.LogFilePaths) != 0 {
if cmd.ParserPreset != "" {
for _, p := range cmd.LogFilePaths {
p = strings.TrimSpace(p)
if p != "" {
body.LogSources = append(body.LogSources, &agents.AddAgentParamsBodyOtelCollectorLogSourcesItems0{
Path: p,
Preset: cmd.ParserPreset,
})
}
}
} else {
body.LogFilePaths = cmd.LogFilePaths
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
SA1019: body.LogFilePaths is deprecated: use log_sources for per-path preset. If log_sources is empty, these are used with preset "raw". (staticcheck)

}
}

params := &agents.AddAgentParams{
Body: agents.AddAgentBody{
OtelCollector: body,
},
Context: commands.Ctx,
}

resp, err := client.Default.AgentsService.AddAgent(params)
if err != nil {
return nil, err
}
return &addOtelResult{
Agent: resp.Payload.OtelCollector,
}, nil
}
11 changes: 11 additions & 0 deletions agent/agents/supervisor/supervisor.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@ package supervisor

import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
"path/filepath"
"runtime/pprof"
"sort"
"strconv"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -787,6 +789,15 @@ func (s *Supervisor) processParams(agentID string, agentProcess *agentv1.SetStat
templateParams["nomad_data_dir"] = cfg.Paths.NomadDataDir
processParams.Path = cfg.Paths.Nomad
processParams.Env = append(processParams.Env, os.Environ()...)
case inventoryv1.AgentType_AGENT_TYPE_OTEL_COLLECTOR:
scheme := "https"
if cfg.Server.WithoutTLS {
scheme = "http"
}
templateParams["server_otlp_url"] = fmt.Sprintf("%s://%s/otlp", scheme, cfg.Server.Address)
templateParams["server_auth_b64"] = base64.StdEncoding.EncodeToString([]byte(cfg.Server.Username + ":" + cfg.Server.Password))
templateParams["server_insecure"] = strconv.FormatBool(cfg.Server.InsecureTLS)
processParams.Path = cfg.Paths.OtelCollector
default:
return nil, errors.Errorf("unhandled agent type %[1]s (%[1]d).", agentProcess.Type) //nolint:revive
}
Expand Down
1 change: 1 addition & 0 deletions agent/cmd/pmm-agent-entrypoint/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ func main() {
}

status = 0

if *pmmAgentPrerunFile != "" || *pmmAgentPrerunScript != "" { //nolint:nestif
l.Info("Starting pmm-agent for prerun...")
agent := commandPmmAgent([]string{"run"})
Expand Down
4 changes: 4 additions & 0 deletions agent/commands/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ func Run() {
// handle termination signals
signals := make(chan os.Signal, 1)
signal.Notify(signals, unix.SIGTERM, unix.SIGINT)

go func() {
s := <-signals
signal.Stop(signals)
Expand Down Expand Up @@ -80,6 +81,7 @@ func Run() {
var wg sync.WaitGroup
wg.Add(3)
reloadCh := make(chan bool, 1)

go func() {
defer wg.Done()
supervisor.Run(ctx)
Expand All @@ -100,6 +102,7 @@ func Run() {

cleanupTmp(cfg.Paths.TempDir, l)
wg.Wait()

select {
case <-rootCtx.Done():
return
Expand Down Expand Up @@ -171,6 +174,7 @@ func cleanupTmp(tmpRoot string, log *logrus.Entry) {
log.Warnf("Failed to read directory '%s': %s", tmpRoot, err.Error())
return
}

for _, dir := range dirs {
if dir.IsDir() {
dirPath := filepath.Join(tmpRoot, dir.Name())
Expand Down
2 changes: 1 addition & 1 deletion agent/commands/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ func Setup() {
$ pmm-agent setup
We will use server address from config, not from run's flag.
*/

l := logrus.WithField("component", "setup")

configStorage := config.NewStorage(nil)
Expand Down Expand Up @@ -94,6 +93,7 @@ func checkStatus(configFilepath string, l *logrus.Entry) (string, bool) {
fmt.Printf("Checking local pmm-agent status...\n")
status, err := localStatus()
l.Debugf("Status error: %#v", err)

switch err := err.(type) { //nolint:errorlint
case nil:
if status.ConfigFilepath == "" {
Expand Down
7 changes: 5 additions & 2 deletions agent/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,9 @@ type Paths struct {
AzureExporter string `yaml:"azure_exporter"`
ValkeyExporter string `yaml:"valkey_exporter"`

VMAgent string `yaml:"vmagent"`
Nomad string `yaml:"nomad"`
VMAgent string `yaml:"vmagent"`
Nomad string `yaml:"nomad"`
OtelCollector string `yaml:"otel_collector"`

TempDir string `yaml:"tempdir"`
NomadDataDir string `yaml:"nomad_data_dir"`
Expand Down Expand Up @@ -241,6 +242,7 @@ func get(args []string, cfg *Config, l *logrus.Entry) (string, error) { //nolint
&cfg.Paths.PTMongoDBSummary: "tools/pt-mongodb-summary",
&cfg.Paths.PTMySQLSummary: "tools/pt-mysql-summary",
&cfg.Paths.Nomad: "tools/nomad",
&cfg.Paths.OtelCollector: "tools/otelcol-contrib",
} {
if *sp == "" {
*sp = v
Expand Down Expand Up @@ -282,6 +284,7 @@ func get(args []string, cfg *Config, l *logrus.Entry) (string, error) { //nolint
"Percona Toolkit pt-mongodb-summary": &cfg.Paths.PTMongoDBSummary,
"Percona Toolkit pt-mysql-summary": &cfg.Paths.PTMySQLSummary,
"Nomad binary": &cfg.Paths.Nomad,
"OTEL Collector": &cfg.Paths.OtelCollector,
} {
if !filepath.IsAbs(*sp) {
*sp = filepath.Join(cfg.Paths.PathsBase, *sp)
Expand Down
7 changes: 7 additions & 0 deletions agent/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ func TestGet(t *testing.T) {
PTMySQLSummary: "/usr/local/percona/pmm/tools/pt-mysql-summary",
PTMongoDBSummary: "/usr/local/percona/pmm/tools/pt-mongodb-summary",
Nomad: "/usr/local/percona/pmm/tools/nomad",
OtelCollector: "/usr/local/percona/pmm/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -188,6 +189,7 @@ func TestGet(t *testing.T) {
PTMongoDBSummary: "/usr/local/percona/pmm/tools/pt-mongodb-summary",
PTMySQLSummary: "/usr/local/percona/pmm/tools/pt-mysql-summary",
Nomad: "/usr/local/percona/pmm/tools/nomad",
OtelCollector: "/usr/local/percona/pmm/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -253,6 +255,7 @@ func TestGet(t *testing.T) {
PTMySQLSummary: "/usr/local/percona/pmm/tools/pt-mysql-summary",
PTMongoDBSummary: "/usr/local/percona/pmm/tools/pt-mongodb-summary",
Nomad: "/usr/local/percona/pmm/tools/nomad",
OtelCollector: "/usr/local/percona/pmm/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -324,6 +327,7 @@ func TestGet(t *testing.T) {
PTMongoDBSummary: "/usr/local/percona/pmm/tools/pt-mongodb-summary",
PTMySQLSummary: "/usr/local/percona/pmm/tools/pt-mysql-summary",
Nomad: "/usr/local/percona/pmm/tools/nomad",
OtelCollector: "/usr/local/percona/pmm/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -393,6 +397,7 @@ func TestGet(t *testing.T) {
PTMongoDBSummary: "/base/tools/pt-mongodb-summary",
PTMySQLSummary: "/base/tools/pt-mysql-summary",
Nomad: "/base/tools/nomad",
OtelCollector: "/base/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -460,6 +465,7 @@ func TestGet(t *testing.T) {
PTMongoDBSummary: "/base/tools/pt-mongodb-summary",
PTMySQLSummary: "/base/tools/pt-mysql-summary",
Nomad: "/base/tools/nomad",
OtelCollector: "/base/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down Expand Up @@ -512,6 +518,7 @@ func TestGet(t *testing.T) {
PTMongoDBSummary: "/usr/local/percona/pmm/tools/pt-mongodb-summary",
PTMySQLSummary: "/usr/local/percona/pmm/tools/pt-mysql-summary",
Nomad: "/usr/local/percona/pmm/tools/nomad",
OtelCollector: "/usr/local/percona/pmm/tools/otelcol-contrib",
},
WindowConnectedTime: defaultWindowPeriod,
Ports: Ports{
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading