From cc3e859d9efcb8dc276562aebb249a2adad86629 Mon Sep 17 00:00:00 2001 From: Robson Roberto Souza Peixoto Date: Sat, 18 Mar 2017 17:10:44 -0300 Subject: [PATCH] Add support for multiple ports in Marathon (#2506) - create a target for every port - add meta labels for Marathon labels in portMappings and portDefinitions --- discovery/marathon/marathon.go | 61 +++++-- discovery/marathon/marathon_test.go | 239 +++++++++++++++++++++++++++- 2 files changed, 285 insertions(+), 15 deletions(-) diff --git a/discovery/marathon/marathon.go b/discovery/marathon/marathon.go index 75db582bf..5d715d2b7 100644 --- a/discovery/marathon/marathon.go +++ b/discovery/marathon/marathon.go @@ -30,6 +30,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/util/httputil" + "github.com/prometheus/prometheus/util/strutil" ) const ( @@ -45,6 +46,11 @@ const ( // taskLabel contains the mesos task name of the app instance. taskLabel model.LabelName = metaLabelPrefix + "task" + // portMappingLabelPrefix is the prefix for the application portMappings labels. + portMappingLabelPrefix = metaLabelPrefix + "port_mapping_label_" + // portDefinitionLabelPrefix is the prefix for the application portDefinitions labels. + portDefinitionLabelPrefix = metaLabelPrefix + "port_definition_label_" + // Constants for instrumentation. namespace = "prometheus" ) @@ -188,9 +194,15 @@ type Task struct { Ports []uint32 `json:"ports"` } +// PortMappings describes in which port the process are binding inside the docker container. +type PortMappings struct { + Labels map[string]string `json:"labels"` +} + // DockerContainer describes a container which uses the docker runtime. type DockerContainer struct { - Image string `json:"image"` + Image string `json:"image"` + PortMappings []PortMappings `json:"portMappings"` } // Container describes the runtime an app in running in. @@ -198,13 +210,19 @@ type Container struct { Docker DockerContainer `json:"docker"` } +// PortDefinitions describes which load balancer port you should access to access the service. +type PortDefinitions struct { + Labels map[string]string `json:"labels"` +} + // App describes a service running on Marathon. type App struct { - ID string `json:"id"` - Tasks []Task `json:"tasks"` - RunningTasks int `json:"tasksRunning"` - Labels map[string]string `json:"labels"` - Container Container `json:"container"` + ID string `json:"id"` + Tasks []Task `json:"tasks"` + RunningTasks int `json:"tasksRunning"` + Labels map[string]string `json:"labels"` + Container Container `json:"container"` + PortDefinitions []PortDefinitions `json:"portDefinitions"` } // AppList is a list of Marathon apps. @@ -285,7 +303,7 @@ func createTargetGroup(app *App) *config.TargetGroup { } for ln, lv := range app.Labels { - ln = appLabelPrefix + ln + ln = appLabelPrefix + strutil.SanitizeLabelName(ln) tg.Labels[model.LabelName(ln)] = model.LabelValue(lv) } @@ -298,15 +316,30 @@ func targetsForApp(app *App) []model.LabelSet { if len(t.Ports) == 0 { continue } - target := targetForTask(&t) - targets = append(targets, model.LabelSet{ - model.AddressLabel: model.LabelValue(target), - taskLabel: model.LabelValue(t.ID), - }) + for i := 0; i < len(t.Ports); i++ { + targetAddress := targetForTask(&t, i) + target := model.LabelSet{ + model.AddressLabel: model.LabelValue(targetAddress), + taskLabel: model.LabelValue(t.ID), + } + if i < len(app.PortDefinitions) { + for ln, lv := range app.PortDefinitions[i].Labels { + ln = portDefinitionLabelPrefix + strutil.SanitizeLabelName(ln) + target[model.LabelName(ln)] = model.LabelValue(lv) + } + } + if i < len(app.Container.Docker.PortMappings) { + for ln, lv := range app.Container.Docker.PortMappings[i].Labels { + ln = portMappingLabelPrefix + strutil.SanitizeLabelName(ln) + target[model.LabelName(ln)] = model.LabelValue(lv) + } + } + targets = append(targets, target) + } } return targets } -func targetForTask(task *Task) string { - return net.JoinHostPort(task.Host, fmt.Sprintf("%d", task.Ports[0])) +func targetForTask(task *Task, index int) string { + return net.JoinHostPort(task.Host, fmt.Sprintf("%d", task.Ports[index])) } diff --git a/discovery/marathon/marathon_test.go b/discovery/marathon/marathon_test.go index b545da172..913380ad6 100644 --- a/discovery/marathon/marathon_test.go +++ b/discovery/marathon/marathon_test.go @@ -80,7 +80,12 @@ func marathonTestAppList(labels map[string]string, runningTasks int) *AppList { Host: "mesos-slave1", Ports: []uint32{31000}, } - docker = DockerContainer{Image: "repo/image:tag"} + docker = DockerContainer{ + Image: "repo/image:tag", + PortMappings: []PortMappings{ + {Labels: labels}, + }, + } container = Container{Docker: docker} app = App{ ID: "test-service", @@ -88,6 +93,9 @@ func marathonTestAppList(labels map[string]string, runningTasks int) *AppList { RunningTasks: runningTasks, Labels: labels, Container: container, + PortDefinitions: []PortDefinitions{ + {Labels: make(map[string]string)}, + }, } ) return &AppList{ @@ -119,6 +127,12 @@ func TestMarathonSDSendGroup(t *testing.T) { if tgt[model.AddressLabel] != "mesos-slave1:31000" { t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" { + t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel]) + } default: t.Fatal("Did not get a target group.") } @@ -189,6 +203,83 @@ func TestMarathonSDRunAndStop(t *testing.T) { } } +func marathonTestAppListWithMutiplePorts(labels map[string]string, runningTasks int) *AppList { + var ( + task = Task{ + ID: "test-task-1", + Host: "mesos-slave1", + Ports: []uint32{31000, 32000}, + } + docker = DockerContainer{ + Image: "repo/image:tag", + PortMappings: []PortMappings{ + {Labels: labels}, + {Labels: make(map[string]string)}, + }, + } + container = Container{Docker: docker} + app = App{ + ID: "test-service", + Tasks: []Task{task}, + RunningTasks: runningTasks, + Labels: labels, + Container: container, + PortDefinitions: []PortDefinitions{ + {Labels: make(map[string]string)}, + {Labels: labels}, + }, + } + ) + return &AppList{ + Apps: []App{app}, + } +} + +func TestMarathonSDSendGroupWithMutiplePort(t *testing.T) { + var ( + ch = make(chan []*config.TargetGroup, 1) + client = func(client *http.Client, url, token string) (*AppList, error) { + return marathonTestAppListWithMutiplePorts(marathonValidLabel, 1), nil + } + ) + if err := testUpdateServices(client, ch); err != nil { + t.Fatalf("Got error: %s", err) + } + select { + case tgs := <-ch: + tg := tgs[0] + + if tg.Source != "test-service" { + t.Fatalf("Wrong target group name: %s", tg.Source) + } + if len(tg.Targets) != 2 { + t.Fatalf("Wrong number of targets: %v", tg.Targets) + } + tgt := tg.Targets[0] + if tgt[model.AddressLabel] != "mesos-slave1:31000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" { + t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel]) + } + tgt = tg.Targets[1] + if tgt[model.AddressLabel] != "mesos-slave1:32000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" { + t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel]) + } + default: + t.Fatal("Did not get a target group.") + } +} + func marathonTestZeroTaskPortAppList(labels map[string]string, runningTasks int) *AppList { var ( task = Task{ @@ -235,3 +326,149 @@ func TestMarathonZeroTaskPorts(t *testing.T) { t.Fatal("Did not get a target group.") } } + +func marathonTestAppListWithoutPortMappings(labels map[string]string, runningTasks int) *AppList { + var ( + task = Task{ + ID: "test-task-1", + Host: "mesos-slave1", + Ports: []uint32{31000, 32000}, + } + docker = DockerContainer{ + Image: "repo/image:tag", + } + container = Container{Docker: docker} + app = App{ + ID: "test-service", + Tasks: []Task{task}, + RunningTasks: runningTasks, + Labels: labels, + Container: container, + PortDefinitions: []PortDefinitions{ + {Labels: make(map[string]string)}, + {Labels: labels}, + }, + } + ) + return &AppList{ + Apps: []App{app}, + } +} + +func TestMarathonSDSendGroupWithoutPortMappings(t *testing.T) { + var ( + ch = make(chan []*config.TargetGroup, 1) + client = func(client *http.Client, url, token string) (*AppList, error) { + return marathonTestAppListWithoutPortMappings(marathonValidLabel, 1), nil + } + ) + if err := testUpdateServices(client, ch); err != nil { + t.Fatalf("Got error: %s", err) + } + select { + case tgs := <-ch: + tg := tgs[0] + + if tg.Source != "test-service" { + t.Fatalf("Wrong target group name: %s", tg.Source) + } + if len(tg.Targets) != 2 { + t.Fatalf("Wrong number of targets: %v", tg.Targets) + } + tgt := tg.Targets[0] + if tgt[model.AddressLabel] != "mesos-slave1:31000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel]) + } + tgt = tg.Targets[1] + if tgt[model.AddressLabel] != "mesos-slave1:32000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "yes" { + t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel]) + } + default: + t.Fatal("Did not get a target group.") + } +} + +func marathonTestAppListWithoutPortDefinitions(labels map[string]string, runningTasks int) *AppList { + var ( + task = Task{ + ID: "test-task-1", + Host: "mesos-slave1", + Ports: []uint32{31000, 32000}, + } + docker = DockerContainer{ + Image: "repo/image:tag", + PortMappings: []PortMappings{ + {Labels: labels}, + {Labels: make(map[string]string)}, + }, + } + container = Container{Docker: docker} + app = App{ + ID: "test-service", + Tasks: []Task{task}, + RunningTasks: runningTasks, + Labels: labels, + Container: container, + } + ) + return &AppList{ + Apps: []App{app}, + } +} + +func TestMarathonSDSendGroupWithoutPortDefinitions(t *testing.T) { + var ( + ch = make(chan []*config.TargetGroup, 1) + client = func(client *http.Client, url, token string) (*AppList, error) { + return marathonTestAppListWithoutPortDefinitions(marathonValidLabel, 1), nil + } + ) + if err := testUpdateServices(client, ch); err != nil { + t.Fatalf("Got error: %s", err) + } + select { + case tgs := <-ch: + tg := tgs[0] + + if tg.Source != "test-service" { + t.Fatalf("Wrong target group name: %s", tg.Source) + } + if len(tg.Targets) != 2 { + t.Fatalf("Wrong number of targets: %v", tg.Targets) + } + tgt := tg.Targets[0] + if tgt[model.AddressLabel] != "mesos-slave1:31000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "yes" { + t.Fatalf("Wrong first portMappings label from the first port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong first portDefinitions label from the first port: %s", tgt[model.AddressLabel]) + } + tgt = tg.Targets[1] + if tgt[model.AddressLabel] != "mesos-slave1:32000" { + t.Fatalf("Wrong target address: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portMappingLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong portMappings label from the second port: %s", tgt[model.AddressLabel]) + } + if tgt[model.LabelName(portDefinitionLabelPrefix+"prometheus")] != "" { + t.Fatalf("Wrong portDefinitions label from the second port: %s", tgt[model.AddressLabel]) + } + default: + t.Fatal("Did not get a target group.") + } +}