OpenStack General System Performance

Dashboard

Browbeat dashboard "General OpenStack System Performance Dashboard" for viewing Performance Metrics collected from Tripleo deployed OpenStack Clouds.
Last updated: 5 months ago

Downloads: 88

  • Grafana - OpenStack General System Performance - Google Chrome_002.png
    Grafana - OpenStack General System Performance - Google Chrome_002.png

This dashboard is used to view System Performance with "typical" OpenStack Nodes (Controllers, Computes, Deployment Nodes (Undercloud)). It is built for a collectd/graphite metrics environment. It is best to actually deploy the dashboards (and collectd configurations) using Browbeat (https://github.com/openstack/browbeat), however you can use the json in here and by modifying the included collectd configuration.

Collector Configuration Details

# Installed by Browbeat Ansible Installer
# Config type: controller

# Interval default is 10s
Interval 10

# Hostname for this machine, if not defined, use gethostname(2) system call

Hostname "overcloud-controller-0"

# Loaded Plugins:
LoadPlugin "logfile"
<Plugin "logfile">
  LogLevel "info"
  File "/var/log/collectd.log"
  Timestamp true
</Plugin>

LoadPlugin write_graphite
LoadPlugin cpu
LoadPlugin conntrack
LoadPlugin df
LoadPlugin disk
LoadPlugin exec
LoadPlugin interface
LoadPlugin irq
LoadPlugin load
LoadPlugin match_regex
LoadPlugin memory
LoadPlugin mysql
LoadPlugin numa
LoadPlugin processes
LoadPlugin swap
LoadPlugin tail
LoadPlugin turbostat
LoadPlugin unixsock
LoadPlugin uptime

# Open unix domain socket for collectdctl
<Plugin unixsock>
  SocketFile "/var/run/collectd-unixsock"
  SocketGroup "collectd"
  SocketPerms "0770"
  DeleteSocket true
</Plugin>

PreCacheChain "PreCache"
<Chain "PreCache">
  <Rule "ignore_tap">
    <Match "regex">
      Plugin "^interface$"
      PluginInstance "^tap*"
    </Match>
    Target "stop"
  </Rule>
  <Rule "ignore_interfaces_q">
    <Match "regex">
      Plugin "^interface$"
      PluginInstance "^q.*"
    </Match>
    Target "stop"
  </Rule>
  Target "return"
</Chain>

# Graphite Host Configuration
<Plugin write_graphite>
  <Carbon>
    Host "{{graphite_host}}"
    Port "2003"
    Prefix "{{graphite_prefix}}."
    Protocol "tcp"
    LogSendErrors true
    StoreRates true
    AlwaysAppendDS false
    EscapeCharacter "_"
  </Carbon>
</Plugin>

<Plugin df>
  ValuesPercentage true
</Plugin>

<Plugin disk>
  Disk "/^[hsv]d[a-z]+[0-9]?$/"
  IgnoreSelected false
</Plugin>

<Plugin mysql>
  <Database "overcloud">
    Host "localhost"
    User "root"
    Password "{{mysql_root_password.stdout}}"
    Socket "/var/lib/mysql/mysql.sock"
    InnodbStats true
  </Database>
</Plugin>

# (akrzos) Including the version of OpenStack that the process was verified as running after
# OpenStack Installation with a comment at the end of each Process/ProcessMatch statement.
# A Minus before the version means the process was not found in that version. (Ex -9)
<Plugin processes>
  # Aodh (OpenStack Installed)
  ProcessMatch "aodh-evaluator" "aodh-evaluator"                                        # 9,10
  ProcessMatch "aodh-listener" "aodh-listener"                                          # 9,10
  ProcessMatch "aodh-notifier" "python.+aodh-notifier"                                  # 9,10

  # Ceilometer (OpenStack Installed)
  ProcessMatch "ceilometer-agent-central" "python.+ceilometer-agent-central"            # -9,-10
  ProcessMatch "ceilometer-agent-compute" "python.+ceilometer-agent-compute"            # -9,-10
  ProcessMatch "ceilometer-agent-notification" "ceilometer-agent-notification"          # 9,10
  ProcessMatch "ceilometer-alarm-evaluator" "python.+ceilometer-alarm-evaluator"        # -9,-10
  ProcessMatch "ceilometer-alarm-notifier" "python.+ceilometer-alarm-notifier"          # -9,-10
  ProcessMatch "ceilometer-api" "python.+ceilometer-api"                                # 9,10
  ProcessMatch "ceilometer-collector" "ceilometer-collector"                            # 9,10
  ProcessMatch "ceilometer-polling" "ceilometer-polling"                                # 9,10

  # Ceph (OpenStack Installed)
  ProcessMatch "ceph-mon" "^/usr/bin/ceph-mon"                                          # -9,-10
  ProcessMatch "ceph-osd" "^/usr/bin/ceph-osd"                                          # -9,-10
  ProcessMatch "diamond" "python.+diamond"                                              # -9,-10
  ProcessMatch "salt-minion" "python.+salt-minion"                                      # -9,-10

  # Cinder (OpenStack Installed)
  ProcessMatch "cinder-api" "python.+cinder-api"                                        # 9,10
  ProcessMatch "cinder-scheduler" "python.+cinder-scheduler"                            # 9,10
  ProcessMatch "cinder-volume" "python.+cinder-volume"                                  # 9,10

  # Collectd (Browbeat Installed)
  ProcessMatch "collectd" "/usr/sbin/collectd"

  # Glance (OpenStack Installed)
  ProcessMatch "glance-api" "python.+glance-api"                                        # 9,10
  ProcessMatch "glance-registry" "python.+glance-registry"                              # 9,10

  # Gnocchi (OpenStack Installed)
  ProcessMatch "gnocchi-metricd" "gnocchi-metricd"                                      # 9,10
  ProcessMatch "gnocchi-statsd" "python.+gnocchi-statsd"                                # 9,10

  # Everything Else (OpenStack Installed)
  # (Processes displayed under "Everything Else" on Grafana Dashboards)
  ProcessMatch "dnsmasq" "^dnsmasq.+"                                                   # -9,-10
  ProcessMatch "haproxy" "/usr/sbin/haproxy.+/etc/haproxy/haproxy.cfg"                  # 9,10
  Process "httpd"                                                                       # 9,10
  Process "memcached"                                                                   # 9,10
  Process "mongod"                                                                      # 9,10
  ProcessMatch "mysqld" "/usr/libexec/mysqld"                                           # 9,10
  ProcessMatch "rabbitmq" "/usr/lib64/erlang/erts-.+/bin/beam.smp"                      # 9,10
  Process "redis-server"                                                                # 9,10
  ProcessMatch "tuskar-api" "python.+tuskar-api"                                        # -9,-10

  # Heat (OpenStack Installed)
  ProcessMatch "heat-api" "python.+heat-api --config-file"                              # 9,10
  ProcessMatch "heat-api-cfn" "python.+heat-api-cfn"                                    # 9,10
  ProcessMatch "heat-api-cloudwatch" "python.+heat-api-cloudwatch"                      # 9,10
  ProcessMatch "heat-engine" "python.+heat-engine"                                      # 9,10

  # Keystone (OpenStack Installed)
  ProcessMatch "keystone-all" "python.+keystone-all"                                    # -9,-10
  ProcessMatch "keystone-admin" "keystone-admin.*-DFOREGROUND"                          # 9,10
  ProcessMatch "keystone-main" "keystone-main.*-DFOREGROUND"                            # 9,10

  # Neutron (OpenStack Installed)
  ProcessMatch "neutron-dhcp-agent" "python.+neutron-dhcp-agent"                        # 9,10
  ProcessMatch "neutron-l3-agent" "python.+neutron-l3-agent"                            # 9,10
  ProcessMatch "neutron-metadata-agent" "python.+neutron-metadata-agent"                # 9,10
  ProcessMatch "neutron-ns-metadata-proxy" "python.+neutron-ns-metadata-proxy"          # -9,-10
  ProcessMatch "neutron-openvswitch-agent" "python.+neutron-openvswitch-agent"          # 9,10
  ProcessMatch "neutron-server" "python.+neutron-server"                                # 9,10

  # Nova (OpenStack Installed)
  ProcessMatch "nova-api" "python.+nova-api"                                            # 9,10
  ProcessMatch "nova-cert" "python.+nova-cert"                                          # -9,-10
  ProcessMatch "nova-compute" "python.+nova-compute"                                    # -9,-10
  ProcessMatch "nova-conductor" "python.+nova-conductor"                                # 9,10
  ProcessMatch "nova-consoleauth" "python.+nova-consoleauth"                            # 9,10
  ProcessMatch "nova-novncproxy" "python.+nova-novncproxy"                              # 9,10
  ProcessMatch "nova-scheduler" "python.+nova-scheduler"                                # 9,10

  # OVS (OpenStack Installed)
  ProcessMatch "ovs-vswitchd" "ovs-vswitchd.+openvswitch"                               # 9,10
  ProcessMatch "ovsdb-client" "/bin/ovsdb-client"                                       # 9,10
  ProcessMatch "ovsdb-server" "ovsdb-server.+openvswitch"                               # 9,10
  ProcessMatch "ovn-northd" "ovn-northd.+openvswitch"                                   # 9,10
  ProcessMatch "ovn-controller" "ovn-controller.+openvswitch"                           # 9,10
  ProcessMatch "ovn-controller-vtep" "ovn-controller-vtep.+openvswitch"                 # 9,10

  # Sahara (OpenStack Installed)
  ProcessMatch "sahara-api" "python.+sahara-api"                                        # 9,-10
  ProcessMatch "sahara-engine" "python.+sahara-engine"                                  # 9,-10

  # Swift (OpenStack Installed)
  ProcessMatch "swift-account-auditor" "python.+swift-account-auditor"                  # 9,10
  ProcessMatch "swift-account-reaper" "python.+swift-account-reaper"                    # 9,10
  ProcessMatch "swift-account-replicator" "python.+swift-account-replicator"            # 9,10
  ProcessMatch "swift-account-server" "python.+swift-account-server"                    # 9,10
  ProcessMatch "swift-container-auditor" "python.+swift-container-auditor"              # 9,10
  ProcessMatch "swift-container-updater" "python.+swift-container-updater"              # 9,10
  ProcessMatch "swift-container-replicator" "python.+swift-container-replicator"        # 9,10
  ProcessMatch "swift-container-server" "python.+swift-container-server"                # 9,10
  ProcessMatch "swift-object-auditor" "python.+swift-object-auditor"                    # 9,10
  ProcessMatch "swift-object-updater" "python.+swift-object-updater"                    # 9,10
  ProcessMatch "swift-object-replicator" "python.+swift-object-replicator"              # 9,10
  ProcessMatch "swift-object-server" "python.+swift-object-server"                      # 9,10
  ProcessMatch "swift-proxy-server" "python.+swift-proxy-server"                        # 9,10

  # Pacemaker / Corosync (OpenStack Installed)
  ProcessMatch "attrd" "/usr/libexec/pacemaker/attrd"                                   # 9,10
  ProcessMatch "cib" "/usr/libexec/pacemaker/cib"                                       # 9,10
  Process "corosync"                                                                    # 9,10
  ProcessMatch "crmd" "/usr/libexec/pacemaker/crmd"                                     # 9,10
  ProcessMatch "lrmd" "/usr/libexec/pacemaker/lrmd"                                     # 9,10
  ProcessMatch "pacemakerd" "/usr/sbin/pacemakerd"                                      # 9,10
  ProcessMatch "pcsd" "^/usr/bin/ruby.+/usr/lib/pcsd"                                   # 9,10
  ProcessMatch "pengine" "/usr/libexec/pacemaker/pengine"                               # 9,10
  ProcessMatch "stonithd" "/usr/libexec/pacemaker/stonithd"                             # 9,10
</Plugin>

<Plugin swap>
  ReportBytes true
  ValuesPercentage true
</Plugin>

# Tail plugin configuration
<Plugin "tail">
  <File "/var/log/keystone/keystone.log">
    Instance "keystone"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/httpd/keystone_wsgi_admin_error.log">
    Instance "keystone-wsgi-admin"
    <Match>
      Regex ":error"
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/httpd/keystone_wsgi_main_error.log">
    Instance "keystone-wsgi-main"
    <Match>
      Regex ":error"
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>


  <File "/var/log/nova/nova-api.log">
    Instance "nova-api"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/nova/nova-conductor.log">
    Instance "nova-conductor"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/nova/nova-consoleauth.log">
    Instance "nova-consoleauth"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/nova/nova-novncproxy.log">
    Instance "nova-novncproxy"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/nova/nova-scheduler.log">
    Instance "nova-scheduler"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

  <File "/var/log/neutron/dhcp-agent.log">
    Instance "neutron-dhcp-agent"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/neutron/l3-agent.log">
    Instance "neutron-l3-agent"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/neutron/metadata-agent.log">
    Instance "neutron-metadata-agent"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/neutron/openvswitch-agent.log">
    Instance "neutron-openvswitch-agent"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/neutron/server.log">
    Instance "neutron-server"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

  <File "/var/log/cinder/api.log">
    Instance "cinder-api"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/cinder/scheduler.log">
    Instance "cinder-scheduler"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/cinder/volume.log">
    Instance "cinder-volume"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

  <File "/var/log/glance/api.log">
    Instance "glance-api"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/glance/registry.log">
    Instance "glance-registry"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

  # Swift logs all into the same file
  <File "/var/log/swift/swift.log">
    Instance "swift"
    <Match>
      Regex "account-server: ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "account-server"
    </Match>
    <Match>
      Regex "container-server: ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "container-server"
    </Match>
    <Match>
      Regex "object-server: ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "object-server"
    </Match>
  </File>

  <File "/var/log/ceilometer/agent-notification.log">
    Instance "ceilometer-agent-notification"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/ceilometer/central.log">
    Instance "ceilometer-central"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/ceilometer/collector.log">
    Instance "ceilometer-collector"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/httpd/ceilometer_wsgi_error.log">
    Instance "ceilometer-wsgi-api"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

  <File "/var/log/gnocchi/app.log">
    Instance "gnocchi-app"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/gnocchi/metricd.log">
    Instance "gnocchi-metricd"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/gnocchi/statsd.log">
    Instance "gnocchi-statsd"
    <Match>
      Regex " ERROR "
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>
  <File "/var/log/httpd/gnocchi_wsgi_error.log">
    Instance "gnocchi-wsgi-api"
    <Match>
      Regex ":error"
      DSType "CounterInc"
      Type "counter"
      Instance "total"
    </Match>
  </File>

</Plugin>

# Include other collectd configuration files
Include "/etc/collectd.d"