Merge pull request #3105 from sak0/dev

discovery openstack: support discovery hypervisors, add rule option.

config/config.go

@@ -1179,6 +1179,7 @@ type OpenstackSDConfig struct {
 	ProjectID        string         `yaml:"project_id"`
 	DomainName       string         `yaml:"domain_name"`
 	DomainID         string         `yaml:"domain_id"`
+	Role             OpenStackRole  `yaml:"role"`
 	Region           string         `yaml:"region"`
 	RefreshInterval  model.Duration `yaml:"refresh_interval,omitempty"`
 	Port             int            `yaml:"port"`
@@ -1187,6 +1188,32 @@ type OpenstackSDConfig struct {
 	XXX map[string]interface{} `yaml:",inline"`
 }
 
+// OpenStackRole is role of the target in OpenStack.
+type OpenStackRole string
+
+// The valid options for OpenStackRole.
+const (
+	// OpenStack document reference
+	// https://docs.openstack.org/nova/pike/admin/arch.html#hypervisors
+	OpenStackRoleHypervisor OpenStackRole = "hypervisor"
+	// OpenStack document reference
+	// https://docs.openstack.org/horizon/pike/user/launch-instances.html
+	OpenStackRoleInstance OpenStackRole = "instance"
+)
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *OpenStackRole) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	if err := unmarshal((*string)(c)); err != nil {
+		return err
+	}
+	switch *c {
+	case OpenStackRoleHypervisor, OpenStackRoleInstance:
+		return nil
+	default:
+		return fmt.Errorf("Unknown OpenStack SD role %q", *c)
+	}
+}
+
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (c *OpenstackSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	*c = DefaultOpenstackSDConfig
@@ -1195,6 +1222,9 @@ func (c *OpenstackSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) err
 	if err != nil {
 		return err
 	}
+	if c.Role == "" {
+		return fmt.Errorf("role missing (one of: instance, hypervisor)")
+	}
 	return checkOverflow(c.XXX, "openstack_sd_config")
 }
 

discovery/discovery.go

@@ -98,7 +98,7 @@ func ProvidersFromConfig(cfg config.ServiceDiscoveryConfig, logger log.Logger) m
 		app("ec2", i, ec2.NewDiscovery(c, logger))
 	}
 	for i, c := range cfg.OpenstackSDConfigs {
-		openstackd, err := openstack.NewDiscovery(c)
+		openstackd, err := openstack.NewDiscovery(c, logger)
 		if err != nil {
 			log.Errorf("Cannot initialize OpenStack discovery: %s", err)
 			continue

discovery/openstack/hypervisor.go

@@ -0,0 +1,145 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors"
+	"github.com/gophercloud/gophercloud/pagination"
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/model"
+	"golang.org/x/net/context"
+
+	"github.com/prometheus/prometheus/config"
+)
+
+const (
+	openstackLabelHypervisorHostIP   = openstackLabelPrefix + "hypervisor_host_ip"
+	openstackLabelHypervisorHostName = openstackLabelPrefix + "hypervisor_hostname"
+	openstackLabelHypervisorStatus   = openstackLabelPrefix + "hypervisor_status"
+	openstackLabelHypervisorState    = openstackLabelPrefix + "hypervisor_state"
+	openstackLabelHypervisorType     = openstackLabelPrefix + "hypervisor_type"
+)
+
+// HypervisorDiscovery discovers OpenStack hypervisors.
+type HypervisorDiscovery struct {
+	authOpts *gophercloud.AuthOptions
+	region   string
+	interval time.Duration
+	logger   log.Logger
+	port     int
+}
+
+// NewHypervisorDiscovery returns a new hypervisor discovery.
+func NewHypervisorDiscovery(opts *gophercloud.AuthOptions,
+	interval time.Duration, port int, region string, l log.Logger) *HypervisorDiscovery {
+	return &HypervisorDiscovery{authOpts: opts,
+		region: region, interval: interval, port: port, logger: l}
+}
+
+// Run implements the TargetProvider interface.
+func (h *HypervisorDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
+	// Get an initial set right away.
+	tg, err := h.refresh()
+	if err != nil {
+		h.logger.Error(err)
+	} else {
+		select {
+		case ch <- []*config.TargetGroup{tg}:
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	ticker := time.NewTicker(h.interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			tg, err := h.refresh()
+			if err != nil {
+				h.logger.Error(err)
+				continue
+			}
+
+			select {
+			case ch <- []*config.TargetGroup{tg}:
+			case <-ctx.Done():
+				return
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (h *HypervisorDiscovery) refresh() (*config.TargetGroup, error) {
+	var err error
+	t0 := time.Now()
+	defer func() {
+		refreshDuration.Observe(time.Since(t0).Seconds())
+		if err != nil {
+			refreshFailuresCount.Inc()
+		}
+	}()
+
+	provider, err := openstack.AuthenticatedClient(*h.authOpts)
+	if err != nil {
+		return nil, fmt.Errorf("could not create OpenStack session: %s", err)
+	}
+	client, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{
+		Region: h.region,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create OpenStack compute session: %s", err)
+	}
+
+	tg := &config.TargetGroup{
+		Source: fmt.Sprintf("OS_" + h.region),
+	}
+	// OpenStack API reference
+	// https://developer.openstack.org/api-ref/compute/#list-hypervisors-details
+	pagerHypervisors := hypervisors.List(client)
+	err = pagerHypervisors.EachPage(func(page pagination.Page) (bool, error) {
+		hypervisorList, err := hypervisors.ExtractHypervisors(page)
+		if err != nil {
+			return false, fmt.Errorf("could not extract hypervisors: %s", err)
+		}
+		for _, hypervisor := range hypervisorList {
+			labels := model.LabelSet{
+				openstackLabelHypervisorHostIP: model.LabelValue(hypervisor.HostIP),
+			}
+			addr := net.JoinHostPort(hypervisor.HostIP, fmt.Sprintf("%d", h.port))
+			labels[model.AddressLabel] = model.LabelValue(addr)
+			labels[openstackLabelHypervisorHostName] = model.LabelValue(hypervisor.HypervisorHostname)
+			labels[openstackLabelHypervisorHostIP] = model.LabelValue(hypervisor.HostIP)
+			labels[openstackLabelHypervisorStatus] = model.LabelValue(hypervisor.Status)
+			labels[openstackLabelHypervisorState] = model.LabelValue(hypervisor.State)
+			labels[openstackLabelHypervisorType] = model.LabelValue(hypervisor.HypervisorType)
+			tg.Targets = append(tg.Targets, labels)
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return tg, nil
+}

discovery/openstack/hypervisor_test.go

@@ -0,0 +1,84 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/config"
+)
+
+type OpenstackSDHypervisorTestSuite struct {
+	suite.Suite
+	Mock *SDMock
+}
+
+func (s *OpenstackSDHypervisorTestSuite) TearDownSuite() {
+	s.Mock.ShutdownServer()
+}
+
+func (s *OpenstackSDHypervisorTestSuite) SetupTest() {
+	s.Mock = NewSDMock(s.T())
+	s.Mock.Setup()
+
+	s.Mock.HandleHypervisorListSuccessfully()
+
+	s.Mock.HandleVersionsSuccessfully()
+	s.Mock.HandleAuthSuccessfully()
+}
+
+func TestOpenstackSDHypervisorSuite(t *testing.T) {
+	suite.Run(t, new(OpenstackSDHypervisorTestSuite))
+}
+
+func (s *OpenstackSDHypervisorTestSuite) openstackAuthSuccess() (Discovery, error) {
+	conf := config.OpenstackSDConfig{
+		IdentityEndpoint: s.Mock.Endpoint(),
+		Password:         "test",
+		Username:         "test",
+		DomainName:       "12345",
+		Region:           "RegionOne",
+		Role:             "hypervisor",
+	}
+	return NewDiscovery(&conf, log.Base())
+}
+
+func (s *OpenstackSDHypervisorTestSuite) TestOpenstackSDHypervisorRefresh() {
+	hypervisor, _ := s.openstackAuthSuccess()
+	tg, err := hypervisor.refresh()
+	assert.Nil(s.T(), err)
+	require.NotNil(s.T(), tg)
+	require.NotNil(s.T(), tg.Targets)
+	require.Len(s.T(), tg.Targets, 2)
+
+	assert.Equal(s.T(), tg.Targets[0]["__address__"], model.LabelValue("172.16.70.14:0"))
+	assert.Equal(s.T(), tg.Targets[0]["__meta_openstack_hypervisor_hostname"], model.LabelValue("nc14.cloud.com"))
+	assert.Equal(s.T(), tg.Targets[0]["__meta_openstack_hypervisor_type"], model.LabelValue("QEMU"))
+	assert.Equal(s.T(), tg.Targets[0]["__meta_openstack_hypervisor_host_ip"], model.LabelValue("172.16.70.14"))
+	assert.Equal(s.T(), tg.Targets[0]["__meta_openstack_hypervisor_state"], model.LabelValue("up"))
+	assert.Equal(s.T(), tg.Targets[0]["__meta_openstack_hypervisor_status"], model.LabelValue("enabled"))
+
+	assert.Equal(s.T(), tg.Targets[1]["__address__"], model.LabelValue("172.16.70.13:0"))
+	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_hypervisor_hostname"], model.LabelValue("cc13.cloud.com"))
+	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_hypervisor_type"], model.LabelValue("QEMU"))
+	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_hypervisor_host_ip"], model.LabelValue("172.16.70.13"))
+	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_hypervisor_state"], model.LabelValue("up"))
+	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_hypervisor_status"], model.LabelValue("enabled"))
+}

discovery/openstack/instance.go

@@ -0,0 +1,211 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openstack
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/openstack"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/floatingips"
+	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
+	"github.com/gophercloud/gophercloud/pagination"
+	"github.com/prometheus/common/log"
+	"github.com/prometheus/common/model"
+	"golang.org/x/net/context"
+
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/util/strutil"
+)
+
+const (
+	openstackLabelPrefix         = model.MetaLabelPrefix + "openstack_"
+	openstackLabelInstanceID     = openstackLabelPrefix + "instance_id"
+	openstackLabelInstanceName   = openstackLabelPrefix + "instance_name"
+	openstackLabelInstanceStatus = openstackLabelPrefix + "instance_status"
+	openstackLabelInstanceFlavor = openstackLabelPrefix + "instance_flavor"
+	openstackLabelPublicIP       = openstackLabelPrefix + "public_ip"
+	openstackLabelPrivateIP      = openstackLabelPrefix + "private_ip"
+	openstackLabelTagPrefix      = openstackLabelPrefix + "tag_"
+)
+
+// InstanceDiscovery discovers OpenStack instances.
+type InstanceDiscovery struct {
+	authOpts *gophercloud.AuthOptions
+	region   string
+	interval time.Duration
+	logger   log.Logger
+	port     int
+}
+
+// NewInstanceDiscovery returns a new instance discovery.
+func NewInstanceDiscovery(opts *gophercloud.AuthOptions,
+	interval time.Duration, port int, region string, l log.Logger) *InstanceDiscovery {
+	return &InstanceDiscovery{authOpts: opts,
+		region: region, interval: interval, port: port, logger: l}
+}
+
+// Run implements the TargetProvider interface.
+func (i *InstanceDiscovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
+	// Get an initial set right away.
+	tg, err := i.refresh()
+	if err != nil {
+		i.logger.Error(err)
+	} else {
+		select {
+		case ch <- []*config.TargetGroup{tg}:
+		case <-ctx.Done():
+			return
+		}
+	}
+
+	ticker := time.NewTicker(i.interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			tg, err := i.refresh()
+			if err != nil {
+				i.logger.Error(err)
+				continue
+			}
+
+			select {
+			case ch <- []*config.TargetGroup{tg}:
+			case <-ctx.Done():
+				return
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (i *InstanceDiscovery) refresh() (*config.TargetGroup, error) {
+	var err error
+	t0 := time.Now()
+	defer func() {
+		refreshDuration.Observe(time.Since(t0).Seconds())
+		if err != nil {
+			refreshFailuresCount.Inc()
+		}
+	}()
+
+	provider, err := openstack.AuthenticatedClient(*i.authOpts)
+	if err != nil {
+		return nil, fmt.Errorf("could not create OpenStack session: %s", err)
+	}
+	client, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{
+		Region: i.region,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create OpenStack compute session: %s", err)
+	}
+
+	// OpenStack API reference
+	// https://developer.openstack.org/api-ref/compute/#list-floating-ips
+	pagerFIP := floatingips.List(client)
+	floatingIPList := make(map[string][]string)
+	err = pagerFIP.EachPage(func(page pagination.Page) (bool, error) {
+		result, err := floatingips.ExtractFloatingIPs(page)
+		if err != nil {
+			return false, fmt.Errorf("could not extract floatingips: %s", err)
+		}
+		for _, ip := range result {
+			// Skip not associated ips
+			if ip.InstanceID != "" {
+				floatingIPList[ip.InstanceID] = append(floatingIPList[ip.InstanceID], ip.IP)
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// OpenStack API reference
+	// https://developer.openstack.org/api-ref/compute/#list-servers
+	opts := servers.ListOpts{}
+	pager := servers.List(client, opts)
+	tg := &config.TargetGroup{
+		Source: fmt.Sprintf("OS_" + i.region),
+	}
+	err = pager.EachPage(func(page pagination.Page) (bool, error) {
+		instanceList, err := servers.ExtractServers(page)
+		if err != nil {
+			return false, fmt.Errorf("could not extract instances: %s", err)
+		}
+
+		for _, s := range instanceList {
+			labels := model.LabelSet{
+				openstackLabelInstanceID: model.LabelValue(s.ID),
+			}
+			if len(s.Addresses) == 0 {
+				i.logger.Info("Got no IP address for instance %s", s.ID)
+				continue
+			}
+			for _, address := range s.Addresses {
+				md, ok := address.([]interface{})
+				if !ok {
+					i.logger.Warn("Invalid type for address, expected array")
+					continue
+				}
+				if len(md) == 0 {
+					i.logger.Debugf("Got no IP address for instance %s", s.ID)
+					continue
+				}
+				md1, ok := md[0].(map[string]interface{})
+				if !ok {
+					i.logger.Warn("Invalid type for address, expected dict")
+					continue
+				}
+				addr, ok := md1["addr"].(string)
+				if !ok {
+					i.logger.Warn("Invalid type for address, expected string")
+					continue
+				}
+				labels[openstackLabelPrivateIP] = model.LabelValue(addr)
+				addr = net.JoinHostPort(addr, fmt.Sprintf("%d", i.port))
+				labels[model.AddressLabel] = model.LabelValue(addr)
+				// Only use first private IP
+				break
+			}
+			if val, ok := floatingIPList[s.ID]; ok && len(val) > 0 {
+				labels[openstackLabelPublicIP] = model.LabelValue(val[0])
+			}
+			labels[openstackLabelInstanceStatus] = model.LabelValue(s.Status)
+			labels[openstackLabelInstanceName] = model.LabelValue(s.Name)
+			id, ok := s.Flavor["id"].(string)
+			if !ok {
+				i.logger.Warn("Invalid type for instance id, excepted string")
+				continue
+			}
+			labels[openstackLabelInstanceFlavor] = model.LabelValue(id)
+			for k, v := range s.Metadata {
+				name := strutil.SanitizeLabelName(k)
+				labels[openstackLabelTagPrefix+model.LabelName(name)] = model.LabelValue(v)
+			}
+			tg.Targets = append(tg.Targets, labels)
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return tg, nil
+}

discovery/openstack/instance_test.go

@@ -20,20 +20,21 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/config"
 )
 
-type OpenstackSDTestSuite struct {
+type OpenstackSDInstanceTestSuite struct {
 	suite.Suite
 	Mock *SDMock
 }
 
-func (s *OpenstackSDTestSuite) TearDownSuite() {
+func (s *OpenstackSDInstanceTestSuite) TearDownSuite() {
 	s.Mock.ShutdownServer()
 }
 
-func (s *OpenstackSDTestSuite) SetupTest() {
+func (s *OpenstackSDInstanceTestSuite) SetupTest() {
 	s.Mock = NewSDMock(s.T())
 	s.Mock.Setup()
 
@@ -44,26 +45,26 @@ func (s *OpenstackSDTestSuite) SetupTest() {
 	s.Mock.HandleAuthSuccessfully()
 }
 
-func TestOpenstackSDSuite(t *testing.T) {
-	suite.Run(t, new(OpenstackSDTestSuite))
+func TestOpenstackSDInstanceSuite(t *testing.T) {
+	suite.Run(t, new(OpenstackSDInstanceTestSuite))
 }
 
-func (s *OpenstackSDTestSuite) openstackAuthSuccess() (*Discovery, error) {
+func (s *OpenstackSDInstanceTestSuite) openstackAuthSuccess() (Discovery, error) {
 	conf := config.OpenstackSDConfig{
 		IdentityEndpoint: s.Mock.Endpoint(),
 		Password:         "test",
 		Username:         "test",
 		DomainName:       "12345",
 		Region:           "RegionOne",
+		Role:             "instance",
 	}
-
-	return NewDiscovery(&conf)
+	return NewDiscovery(&conf, log.Base())
 }
 
-func (s *OpenstackSDTestSuite) TestOpenstackSDRefresh() {
-	d, _ := s.openstackAuthSuccess()
+func (s *OpenstackSDInstanceTestSuite) TestOpenstackSDInstanceRefresh() {
+	instance, _ := s.openstackAuthSuccess()
+	tg, err := instance.refresh()
 
-	tg, err := d.refresh()
 	assert.Nil(s.T(), err)
 	require.NotNil(s.T(), tg)
 	require.NotNil(s.T(), tg.Targets)
@@ -83,5 +84,4 @@ func (s *OpenstackSDTestSuite) TestOpenstackSDRefresh() {
 	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_instance_name"], model.LabelValue("derp"))
 	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_instance_status"], model.LabelValue("ACTIVE"))
 	assert.Equal(s.T(), tg.Targets[1]["__meta_openstack_private_ip"], model.LabelValue("10.0.0.31"))
-
 }

discovery/openstack/mock.go

@@ -182,6 +182,75 @@ func (m *SDMock) HandleAuthSuccessfully() {
 	})
 }
 
+const hypervisorListBody = `
+{
+    "hypervisors": [
+        {
+            "status": "enabled",
+            "service": {
+                "host": "nc14.cloud.com",
+                "disabled_reason": null,
+                "id": 16
+            },
+            "vcpus_used": 18,
+            "hypervisor_type": "QEMU",
+            "local_gb_used": 84,
+            "vcpus": 24,
+            "hypervisor_hostname": "nc14.cloud.com",
+            "memory_mb_used": 24064,
+            "memory_mb": 96484,
+            "current_workload": 1,
+            "state": "up",
+            "host_ip": "172.16.70.14",
+            "cpu_info": "{\"vendor\": \"Intel\", \"model\": \"IvyBridge\", \"arch\": \"x86_64\", \"features\": [\"pge\", \"avx\", \"clflush\", \"sep\", \"syscall\", \"vme\", \"dtes64\", \"msr\", \"fsgsbase\", \"xsave\", \"vmx\", \"erms\", \"xtpr\", \"cmov\", \"smep\", \"ssse3\", \"est\", \"pat\", \"monitor\", \"smx\", \"pbe\", \"lm\", \"tsc\", \"nx\", \"fxsr\", \"tm\", \"sse4.1\", \"pae\", \"sse4.2\", \"pclmuldq\", \"acpi\", \"tsc-deadline\", \"mmx\", \"osxsave\", \"cx8\", \"mce\", \"de\", \"tm2\", \"ht\", \"dca\", \"lahf_lm\", \"popcnt\", \"mca\", \"pdpe1gb\", \"apic\", \"sse\", \"f16c\", \"pse\", \"ds\", \"invtsc\", \"pni\", \"rdtscp\", \"aes\", \"sse2\", \"ss\", \"ds_cpl\", \"pcid\", \"fpu\", \"cx16\", \"pse36\", \"mtrr\", \"pdcm\", \"rdrand\", \"x2apic\"], \"topology\": {\"cores\": 6, \"cells\": 2, \"threads\": 2, \"sockets\": 1}}",
+            "running_vms": 10,
+            "free_disk_gb": 315,
+            "hypervisor_version": 2003000,
+            "disk_available_least": 304,
+            "local_gb": 399,
+            "free_ram_mb": 72420,
+            "id": 1
+        },
+        {
+            "status": "enabled",
+            "service": {
+                "host": "cc13.cloud.com",
+                "disabled_reason": null,
+                "id": 17
+            },
+            "vcpus_used": 1,
+            "hypervisor_type": "QEMU",
+            "local_gb_used": 20,
+            "vcpus": 24,
+            "hypervisor_hostname": "cc13.cloud.com",
+            "memory_mb_used": 2560,
+            "memory_mb": 96484,
+            "current_workload": 0,
+            "state": "up",
+            "host_ip": "172.16.70.13",
+            "cpu_info": "{\"vendor\": \"Intel\", \"model\": \"IvyBridge\", \"arch\": \"x86_64\", \"features\": [\"pge\", \"avx\", \"clflush\", \"sep\", \"syscall\", \"vme\", \"dtes64\", \"msr\", \"fsgsbase\", \"xsave\", \"vmx\", \"erms\", \"xtpr\", \"cmov\", \"smep\", \"ssse3\", \"est\", \"pat\", \"monitor\", \"smx\", \"pbe\", \"lm\", \"tsc\", \"nx\", \"fxsr\", \"tm\", \"sse4.1\", \"pae\", \"sse4.2\", \"pclmuldq\", \"acpi\", \"tsc-deadline\", \"mmx\", \"osxsave\", \"cx8\", \"mce\", \"de\", \"tm2\", \"ht\", \"dca\", \"lahf_lm\", \"popcnt\", \"mca\", \"pdpe1gb\", \"apic\", \"sse\", \"f16c\", \"pse\", \"ds\", \"invtsc\", \"pni\", \"rdtscp\", \"aes\", \"sse2\", \"ss\", \"ds_cpl\", \"pcid\", \"fpu\", \"cx16\", \"pse36\", \"mtrr\", \"pdcm\", \"rdrand\", \"x2apic\"], \"topology\": {\"cores\": 6, \"cells\": 2, \"threads\": 2, \"sockets\": 1}}",
+            "running_vms": 0,
+            "free_disk_gb": 379,
+            "hypervisor_version": 2003000,
+            "disk_available_least": 384,
+            "local_gb": 399,
+            "free_ram_mb": 93924,
+            "id": 721
+        }
+    ]
+}`
+
+// HandleHypervisorListSuccessfully mocks os-hypervisors detail call
+func (m *SDMock) HandleHypervisorListSuccessfully() {
+	m.Mux.HandleFunc("/os-hypervisors/detail", func(w http.ResponseWriter, r *http.Request) {
+		testMethod(m.t, r, "GET")
+		testHeader(m.t, r, "X-Auth-Token", tokenID)
+
+		w.Header().Add("Content-Type", "application/json")
+		fmt.Fprintf(w, hypervisorListBody)
+	})
+}
+
 const serverListBody = `
 {
 	"servers": [

discovery/openstack/openstack.go

@@ -14,33 +14,15 @@
 package openstack
 
 import (
-	"fmt"
-	"net"
+	"errors"
 	"time"
 
 	"github.com/gophercloud/gophercloud"
-	"github.com/gophercloud/gophercloud/openstack"
-	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/floatingips"
-	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
-	"github.com/gophercloud/gophercloud/pagination"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
-	"github.com/prometheus/common/model"
 	"golang.org/x/net/context"
 
 	"github.com/prometheus/prometheus/config"
-	"github.com/prometheus/prometheus/util/strutil"
-)
-
-const (
-	openstackLabelPrefix         = model.MetaLabelPrefix + "openstack_"
-	openstackLabelInstanceID     = openstackLabelPrefix + "instance_id"
-	openstackLabelInstanceName   = openstackLabelPrefix + "instance_name"
-	openstackLabelInstanceStatus = openstackLabelPrefix + "instance_status"
-	openstackLabelInstanceFlavor = openstackLabelPrefix + "instance_flavor"
-	openstackLabelPublicIP       = openstackLabelPrefix + "public_ip"
-	openstackLabelPrivateIP      = openstackLabelPrefix + "private_ip"
-	openstackLabelTagPrefix      = openstackLabelPrefix + "tag_"
 )
 
 var (
@@ -63,15 +45,13 @@ func init() {
 
 // Discovery periodically performs OpenStack-SD requests. It implements
 // the TargetProvider interface.
-type Discovery struct {
-	authOpts *gophercloud.AuthOptions
-	region   string
-	interval time.Duration
-	port     int
+type Discovery interface {
+	Run(ctx context.Context, ch chan<- []*config.TargetGroup)
+	refresh() (tg *config.TargetGroup, err error)
 }
 
 // NewDiscovery returns a new OpenStackDiscovery which periodically refreshes its targets.
-func NewDiscovery(conf *config.OpenstackSDConfig) (*Discovery, error) {
+func NewDiscovery(conf *config.OpenstackSDConfig, l log.Logger) (Discovery, error) {
 	opts := gophercloud.AuthOptions{
 		IdentityEndpoint: conf.IdentityEndpoint,
 		Username:         conf.Username,
@@ -82,179 +62,16 @@ func NewDiscovery(conf *config.OpenstackSDConfig) (*Discovery, error) {
 		DomainName:       conf.DomainName,
 		DomainID:         conf.DomainID,
 	}
-
-	return &Discovery{
-		authOpts: &opts,
-		region:   conf.Region,
-		interval: time.Duration(conf.RefreshInterval),
-		port:     conf.Port,
-	}, nil
-}
-
-// Run implements the TargetProvider interface.
-func (d *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
-	// Get an initial set right away.
-	tg, err := d.refresh()
-	if err != nil {
-		log.Error(err)
-	} else {
-		select {
-		case ch <- []*config.TargetGroup{tg}:
-		case <-ctx.Done():
-			return
-		}
-	}
-
-	ticker := time.NewTicker(d.interval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ticker.C:
-			tg, err := d.refresh()
-			if err != nil {
-				log.Error(err)
-				continue
-			}
-
-			select {
-			case ch <- []*config.TargetGroup{tg}:
-			case <-ctx.Done():
-				return
-			}
-		case <-ctx.Done():
-			return
-		}
+	switch conf.Role {
+	case config.OpenStackRoleHypervisor:
+		hypervisor := NewHypervisorDiscovery(&opts,
+			time.Duration(conf.RefreshInterval), conf.Port, conf.Region, l)
+		return hypervisor, nil
+	case config.OpenStackRoleInstance:
+		instance := NewInstanceDiscovery(&opts,
+			time.Duration(conf.RefreshInterval), conf.Port, conf.Region, l)
+		return instance, nil
+	default:
+		return nil, errors.New("unknown OpenStack discovery role")
 	}
 }
-
-func (d *Discovery) refresh() (tg *config.TargetGroup, err error) {
-	t0 := time.Now()
-	defer func() {
-		refreshDuration.Observe(time.Since(t0).Seconds())
-		if err != nil {
-			refreshFailuresCount.Inc()
-		}
-	}()
-
-	provider, err := openstack.AuthenticatedClient(*d.authOpts)
-
-	if err != nil {
-		return nil, fmt.Errorf("could not create OpenStack session: %s", err)
-	}
-	client, err := openstack.NewComputeV2(provider, gophercloud.EndpointOpts{
-		Region: d.region,
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("could not create OpenStack compute session: %s", err)
-	}
-
-	opts := servers.ListOpts{}
-	pager := servers.List(client, opts)
-
-	tg = &config.TargetGroup{
-		Source: fmt.Sprintf("OS_%s", d.region),
-	}
-
-	pagerFIP := floatingips.List(client)
-	floatingIPList := make(map[string][]string)
-
-	err = pagerFIP.EachPage(func(page pagination.Page) (bool, error) {
-		result, err := floatingips.ExtractFloatingIPs(page)
-		if err != nil {
-			log.Warn(err)
-		}
-		for _, ip := range result {
-			// Skip not associated ips
-			if ip.InstanceID != "" {
-				floatingIPList[ip.InstanceID] = append(floatingIPList[ip.InstanceID], ip.IP)
-			}
-		}
-		return true, nil
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("could not describe floating IPs: %s", err)
-	}
-
-	err = pager.EachPage(func(page pagination.Page) (bool, error) {
-		serverList, err := servers.ExtractServers(page)
-		if err != nil {
-			return false, fmt.Errorf("could not extract servers: %s", err)
-		}
-
-		for _, s := range serverList {
-			if len(s.Addresses) == 0 {
-				log.Debugf("Got no IP address for instance %s", s.ID)
-				continue
-			}
-			labels := model.LabelSet{
-				openstackLabelInstanceID: model.LabelValue(s.ID),
-			}
-
-			for _, address := range s.Addresses {
-				md, ok := address.([]interface{})
-				if !ok {
-					log.Warn("Invalid type for address, expected array")
-					continue
-				}
-
-				if len(md) == 0 {
-					log.Debugf("Got no IP address for instance %s", s.ID)
-					continue
-				}
-
-				md1, ok := md[0].(map[string]interface{})
-				if !ok {
-					log.Warn("Invalid type for address, expected dict")
-					continue
-				}
-
-				addr, ok := md1["addr"].(string)
-				if !ok {
-					log.Warn("Invalid type for address, expected string")
-					continue
-				}
-
-				labels[openstackLabelPrivateIP] = model.LabelValue(addr)
-
-				addr = net.JoinHostPort(addr, fmt.Sprintf("%d", d.port))
-
-				labels[model.AddressLabel] = model.LabelValue(addr)
-
-				// Only use first private IP
-				break
-			}
-
-			if val, ok := floatingIPList[s.ID]; ok {
-				if len(val) > 0 {
-					labels[openstackLabelPublicIP] = model.LabelValue(val[0])
-				}
-			}
-
-			labels[openstackLabelInstanceStatus] = model.LabelValue(s.Status)
-			labels[openstackLabelInstanceName] = model.LabelValue(s.Name)
-			id, ok := s.Flavor["id"].(string)
-			if !ok {
-				log.Warn("Invalid type for instance id, excepted string")
-				continue
-			}
-			labels[openstackLabelInstanceFlavor] = model.LabelValue(id)
-
-			for k, v := range s.Metadata {
-				name := strutil.SanitizeLabelName(k)
-				labels[openstackLabelTagPrefix+model.LabelName(name)] = model.LabelValue(v)
-			}
-
-			tg.Targets = append(tg.Targets, labels)
-		}
-		return true, nil
-	})
-
-	if err != nil {
-		return nil, fmt.Errorf("could not describe instances: %s", err)
-	}
-
-	return tg, nil
-}

vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors/doc.go

@@ -0,0 +1,3 @@
+// Package hypervisors gives information and control of the os-hypervisors
+// portion of the compute API
+package hypervisors

vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors/requests.go

@@ -0,0 +1,13 @@
+package hypervisors
+
+import (
+	"github.com/gophercloud/gophercloud"
+	"github.com/gophercloud/gophercloud/pagination"
+)
+
+// List makes a request against the API to list hypervisors.
+func List(client *gophercloud.ServiceClient) pagination.Pager {
+	return pagination.NewPager(client, hypervisorsListDetailURL(client), func(r pagination.PageResult) pagination.Page {
+		return HypervisorPage{pagination.SinglePageBase(r)}
+	})
+}

vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors/results.go

@@ -0,0 +1,161 @@
+package hypervisors
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/gophercloud/gophercloud/pagination"
+)
+
+type Topology struct {
+	Sockets int `json:"sockets"`
+	Cores   int `json:"cores"`
+	Threads int `json:"threads"`
+}
+
+type CPUInfo struct {
+	Vendor   string   `json:"vendor"`
+	Arch     string   `json:"arch"`
+	Model    string   `json:"model"`
+	Features []string `json:"features"`
+	Topology Topology `json:"topology"`
+}
+
+type Service struct {
+	Host           string `json:"host"`
+	ID             int    `json:"id"`
+	DisabledReason string `json:"disabled_reason"`
+}
+
+type Hypervisor struct {
+	// A structure that contains cpu information like arch, model, vendor, features and topology
+	CPUInfo CPUInfo `json:"-"`
+	// The current_workload is the number of tasks the hypervisor is responsible for.
+	// This will be equal or greater than the number of active VMs on the system
+	// (it can be greater when VMs are being deleted and the hypervisor is still cleaning up).
+	CurrentWorkload int `json:"current_workload"`
+	// Status of the hypervisor, either "enabled" or "disabled"
+	Status string `json:"status"`
+	// State of the hypervisor, either "up" or "down"
+	State string `json:"state"`
+	// Actual free disk on this hypervisor in GB
+	DiskAvailableLeast int `json:"disk_available_least"`
+	// The hypervisor's IP address
+	HostIP string `json:"host_ip"`
+	// The free disk remaining on this hypervisor in GB
+	FreeDiskGB int `json:"-"`
+	// The free RAM in this hypervisor in MB
+	FreeRamMB int `json:"free_ram_mb"`
+	// The hypervisor host name
+	HypervisorHostname string `json:"hypervisor_hostname"`
+	// The hypervisor type
+	HypervisorType string `json:"hypervisor_type"`
+	// The hypervisor version
+	HypervisorVersion int `json:"-"`
+	// Unique ID of the hypervisor
+	ID int `json:"id"`
+	// The disk in this hypervisor in GB
+	LocalGB int `json:"-"`
+	// The disk used in this hypervisor in GB
+	LocalGBUsed int `json:"local_gb_used"`
+	// The memory of this hypervisor in MB
+	MemoryMB int `json:"memory_mb"`
+	// The memory used in this hypervisor in MB
+	MemoryMBUsed int `json:"memory_mb_used"`
+	// The number of running vms on this hypervisor
+	RunningVMs int `json:"running_vms"`
+	// The hypervisor service object
+	Service Service `json:"service"`
+	// The number of vcpu in this hypervisor
+	VCPUs int `json:"vcpus"`
+	// The number of vcpu used in this hypervisor
+	VCPUsUsed int `json:"vcpus_used"`
+}
+
+func (r *Hypervisor) UnmarshalJSON(b []byte) error {
+
+	type tmp Hypervisor
+	var s struct {
+		tmp
+		CPUInfo           interface{} `json:"cpu_info"`
+		HypervisorVersion interface{} `json:"hypervisor_version"`
+		FreeDiskGB        interface{} `json:"free_disk_gb"`
+		LocalGB           interface{} `json:"local_gb"`
+	}
+
+	err := json.Unmarshal(b, &s)
+	if err != nil {
+		return err
+	}
+
+	*r = Hypervisor(s.tmp)
+
+	// Newer versions pass the CPU into around as the correct types, this just needs
+	// converting and copying into place. Older versions pass CPU info around as a string
+	// and can simply be unmarshalled by the json parser
+	var tmpb []byte
+
+	switch t := s.CPUInfo.(type) {
+	case string:
+		tmpb = []byte(t)
+	case map[string]interface{}:
+		tmpb, err = json.Marshal(t)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("CPUInfo has unexpected type: %T", t)
+	}
+
+	err = json.Unmarshal(tmpb, &r.CPUInfo)
+	if err != nil {
+		return err
+	}
+
+	// These fields may be passed in in scientific notation
+	switch t := s.HypervisorVersion.(type) {
+	case int:
+		r.HypervisorVersion = t
+	case float64:
+		r.HypervisorVersion = int(t)
+	default:
+		return fmt.Errorf("Hypervisor version of unexpected type")
+	}
+
+	switch t := s.FreeDiskGB.(type) {
+	case int:
+		r.FreeDiskGB = t
+	case float64:
+		r.FreeDiskGB = int(t)
+	default:
+		return fmt.Errorf("Free disk GB of unexpected type")
+	}
+
+	switch t := s.LocalGB.(type) {
+	case int:
+		r.LocalGB = t
+	case float64:
+		r.LocalGB = int(t)
+	default:
+		return fmt.Errorf("Local GB of unexpected type")
+	}
+
+	return nil
+}
+
+type HypervisorPage struct {
+	pagination.SinglePageBase
+}
+
+func (page HypervisorPage) IsEmpty() (bool, error) {
+	va, err := ExtractHypervisors(page)
+	return len(va) == 0, err
+}
+
+func ExtractHypervisors(p pagination.Page) ([]Hypervisor, error) {
+	var h struct {
+		Hypervisors []Hypervisor `json:"hypervisors"`
+	}
+	err := (p.(HypervisorPage)).ExtractInto(&h)
+	return h.Hypervisors, err
+}

vendor/github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors/urls.go

@@ -0,0 +1,7 @@
+package hypervisors
+
+import "github.com/gophercloud/gophercloud"
+
+func hypervisorsListDetailURL(c *gophercloud.ServiceClient) string {
+	return c.ServiceURL("os-hypervisors", "detail")
+}

vendor/vendor.json

@@ -383,6 +383,12 @@
 			"revision": "caf34a65f60295108141f62929245943bd00f237",
 			"revisionTime": "2017-06-07T03:48:29Z"
 		},
+		{
+			"checksumSHA1": "SKxDWZElN5KwYPPf4QSs9pR0jKg=",
+			"path": "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/hypervisors",
+			"revision": "caf34a65f60295108141f62929245943bd00f237",
+			"revisionTime": "2017-06-07T03:48:29Z"
+		},
 		{
 			"checksumSHA1": "vTyXSR+Znw7/o/70UBOWG0F09r8=",
 			"path": "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors",