From e7582609865a18c55c067433294c2692107dc48c Mon Sep 17 00:00:00 2001 From: Matt Bostock Date: Tue, 20 Jun 2017 15:43:36 +0100 Subject: [PATCH] Marathon SD: Set port index label The changes [1][] to Marathon service discovery to support multiple ports mean that Prometheus now attempts to scrape all ports belonging to a Marathon service. You can use port definition or port mapping labels to filter out which ports to scrape but that requires service owners to update their Marathon configuration. To allow for a smoother migration path, add a `__meta_marathon_port_index` label, whose value is set to the port's sequential index integer. For example, PORT0 has the value `0`, PORT1 has the value `1`, and so on. This allows you to support scraping both the first available port (the previous behaviour) in addition to ports with a `metrics` label. For example, here's the relabel configuration we might use with this patch: - action: keep source_labels: ['__meta_marathon_port_definition_label_metrics', '__meta_marathon_port_mapping_label_metrics', '__meta_marathon_port_index'] # Keep if port mapping or definition has a 'metrics' label with any # non-empty value, or if no 'metrics' port label exists but this is the # service's first available port regex: ([^;]+;;[^;]+|;[^;]+;[^;]+|;;0) This assumes that the Marathon API returns the ports in sorted order (matching PORT0, PORT1, etc), which it appears that it does. [1]: https://github.com/prometheus/prometheus/pull/2506 --- discovery/marathon/marathon.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index d1f2b2f6c..2c1101722 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -20,6 +20,7 @@ import ( "math/rand" "net" "net/http" + "strconv" "strings" "time" @@ -43,6 +44,9 @@ const ( appLabel model.LabelName = metaLabelPrefix + "app" // imageLabel is the label that is used for the docker image running the service. imageLabel model.LabelName = metaLabelPrefix + "image" + // portIndexLabel is the integer port index when multiple ports are defined; + // e.g. PORT1 would have a value of '1' + portIndexLabel model.LabelName = metaLabelPrefix + "port_index" // taskLabel contains the mesos task name of the app instance. taskLabel model.LabelName = metaLabelPrefix + "task" @@ -323,6 +327,7 @@ func targetsForApp(app *App) []model.LabelSet { target := model.LabelSet{ model.AddressLabel: model.LabelValue(targetAddress), taskLabel: model.LabelValue(t.ID), + portIndexLabel: model.LabelValue(strconv.Itoa(i)), } if i < len(app.PortDefinitions) { for ln, lv := range app.PortDefinitions[i].Labels {