initial commit

roles/monitoring/defaults/main.yml

@@ -0,0 +1,17 @@
+---
+# defaults file for roles/prometheus
+grafana_admin_user: admin
+grafana_domain: "{{ inventory_hostname }}"
+grafana_web_path: /grafana
+grafana_protocol: http
+grafana_port: 3000
+
+prometheus_domain: "{{ inventory_hostname }}"
+prometheus_web_path: /prometheus
+prometheus_port: 9090
+
+nginx_default_path: "{{ grafana_web_path }}"
+
+alertmanager_domain: "{{ inventory_hostname }}"
+alertmanager_web_path: /alertmanager
+alertmanager_port: 9093

roles/monitoring/files/prometheus/rules/hardware.rules

@@ -0,0 +1,43 @@
+groups:
+- name: Hardware alerts
+  rules:
+  - alert: Instancedown
+    expr: up == 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      title: Instance {{ $labels.instance }} is down
+      description: Failed to scrape {{ $labels.job }} on {{ $labels.instance }} for more than 5 minute. Instance seems down.
+  - alert: LowRootDiskSpace
+    expr: (node_filesystem_free_bytes{device =~ "/dev/.+"} / node_filesystem_size_bytes{device =~ "/dev/.+"} * 100) < 10
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      title: Low free root space on {{ $labels.instance }}
+      description: On {{ $labels.instance }} device {{ $labels.device }} mounted on {{ $labels.mountpoint }} has low free space of {{ $value }}%
+  - alert: LowDataDiskSpace
+    expr: (node_filesystem_free_bytes{device !~ "/dev/.+", fstype !~ "tmpfs|.*lxcfs"} / node_filesystem_size_bytes{device !~ "/dev/.+", fstype !~ "tmpfs|.*lxcfs"} * 100) < 10
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      title: Low free data space on {{ $labels.instance }}
+      description: On {{ $labels.instance }} device {{ $labels.device }} mounted on {{ $labels.mountpoint }} has low free space of {{ $value }}%
+  - alert: HighCPULoad
+    expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      title: High CPU load on instance {{ $labels.instance }}
+      description: Instance {{ $labels.instance }} has high CPU load.
+  - alert: HighMemoryUsage
+    expr: (1 - ((node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes)) * 100 > 90
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      description: Instance {{ $labels.instance }} has high memory usage
+      summary: High memory usage on {{ $labels.instance }}

roles/monitoring/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+# handlers file for roles/prometheus
+- name: restart grafana
+  service:
+    name: grafana-server
+    state: restarted
+  when: not grafana_service.changed
+
+- name: restart prometheus
+  service:
+    name: prometheus
+    state: restarted
+  when: not prometheus_service.changed

roles/monitoring/tasks/alertmanager.yml

@@ -0,0 +1,24 @@
+- name: Install Alertmanager
+  import_role:
+    name: cloudalchemy.alertmanager
+    public: yes
+  vars:
+    alertmanager_web_external_url: "http://{{ alertmanager_domain }}{{ alertmanager_web_path }}"
+    alertmanager_config_flags_extra:
+      web.route-prefix: /
+    alertmanager_web_listen_address: "0.0.0.0:{{ alertmanager_port }}"
+
+- name: Configure Nginx for Alertmaneger
+  import_role:
+    name: nginx
+  vars:
+    nginx_config_file: "{{ inventory_hostname }}.d/alertmanager.conf"
+    nginx_server:
+      locations:
+      - path: "{{ alertmanager_web_path }}/"
+        basic_auth:
+          file: .htpasswd.alertmanager
+          password: "{{ alertmanager_password }}"
+        proxy_pass:
+          port: "{{ alertmanager_port }}"
+          path: /

roles/monitoring/tasks/grafana.yml

@@ -0,0 +1,71 @@
+- name: add grafana GPG apt key
+  apt_key:
+    url: https://packages.grafana.com/gpg.key
+    state: present
+
+- block:
+  - name: add grafana repository
+    apt_repository:
+      repo: deb https://packages.grafana.com/oss/deb stable main
+      state: present
+    register: grafana_repo
+    notify: restart grafana
+
+  - name: update apt cache
+    apt:
+      update_cache: yes
+      cache_valid_time: 3600
+    when: grafana_repo is changed
+
+  - name: install grafana package
+    apt:
+      pkg: grafana
+      state: present
+    notify: restart grafana
+
+- name: copy grafana config
+  template:
+    src: grafana.ini.j2
+    dest: /etc/grafana/grafana.ini
+  become: yes
+  notify: restart grafana
+    
+- name: enable grafana service
+  systemd:
+    name: grafana-server
+    state: started
+    enabled: yes
+  register: grafana_service
+
+- name: Configure Nginx for Grafana
+  import_role:
+    name: nginx
+  vars:
+    nginx_config_file: "{{ inventory_hostname }}.conf"
+    nginx_server:
+      name: "{{ inventory_hostname }}"
+      port: 80
+      locations:
+      - path: "= /"
+        return:
+          code: 301
+          url: "http://{{ inventory_hostname }}{{ grafana_web_path }}"
+      includes:
+      - "{{ nginx_config_dir }}/{{ inventory_hostname }}.d/*.conf"
+
+- name: ensure nginx config directory exists
+  file:
+    path: "{{ nginx_config_dir }}/{{ inventory_hostname }}.d"
+    state: directory
+
+- include_role:
+    name: nginx
+    tasks_from: configure
+  vars:
+    nginx_config_file: "{{ inventory_hostname }}.d/grafana.conf"
+    nginx_server:
+      locations:
+      - path: "{{ grafana_web_path }}/"
+        proxy_pass:
+          port: "{{ grafana_port }}"
+          path: /

roles/monitoring/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- import_tasks: grafana.yml
+  become: yes
+  tags: grafana
+- import_tasks: alertmanager.yml
+  tags: alertmanager
+- import_tasks: prometheus.yml
+  tags: prometheus

roles/monitoring/tasks/prometheus.yml

@@ -0,0 +1,26 @@
+- name: Install Prometheus
+  import_role:
+    name: cloudalchemy.prometheus
+    public: yes
+  vars:
+    prometheus_config_file: "prometheus_custom.yml.j2"
+    prometheus_alert_rules: []
+    prometheus_web_external_url: "http://{{ prometheus_domain }}{{ prometheus_web_path }}"
+    prometheus_config_flags_extra:
+      web.route-prefix: /
+    prometheus_web_listen_address: "0.0.0.0:{{ prometheus_port }}"
+
+- name: Configure Nginx for Prometheus
+  import_role:
+    name: nginx
+  vars:
+    nginx_config_file: "{{ inventory_hostname }}.d/prometheus.conf"
+    nginx_server:
+      locations:
+      - path: "{{ prometheus_web_path }}/"
+        basic_auth:
+          file: .htpasswd.prometheus
+          password: "{{ prometheus_password }}"
+        proxy_pass:
+          port: "{{ prometheus_port }}"
+          path: /

roles/monitoring/templates/grafana.ini.j2

@@ -0,0 +1,627 @@
+[server]
+
+[security]
+
+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+;app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+;instance_name = ${HOSTNAME}
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+;data = /var/lib/grafana
+
+# Temporary files in `data` directory older than given duration will be removed
+;temp_data_lifetime = 24h
+
+# Directory where grafana can store logs
+;logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+;plugins = /var/lib/grafana/plugins
+
+# folder that contains provisioning config files that grafana will apply on startup and while running.
+;provisioning = conf/provisioning
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, h2, socket)
+protocol = {{ grafana_protocol }}
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port  to use
+http_port = {{ grafana_port }}
+
+# The public facing domain name used to access grafana from a browser
+domain = {{ grafana_domain }}
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+root_url = {{ grafana_protocol }}://{{ grafana_domain }}{{ grafana_web_path }}
+
+# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
+;serve_from_sub_path = false
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket path
+;socket =
+
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+;type = sqlite3
+;host = 127.0.0.1:3306
+;name = grafana
+;user = root
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+
+# Use either URL or the previous fields to configure the database
+# Example: mysql://user:secret@host:port/database
+;url =
+
+# For "postgres" only, either "disable", "require" or "verify-full"
+;ssl_mode = disable
+
+# For "sqlite3" only, path relative to data_path setting
+;path = grafana.db
+
+# Max idle conn setting default is 2
+;max_idle_conn = 2
+
+# Max conn setting default is 0 (mean not set)
+;max_open_conn =
+
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
+# Set to true to log the sql calls and execution times.
+;log_queries =
+
+# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
+;cache_mode = private
+
+#################################### Cache server #############################
+[remote_cache]
+# Either "redis", "memcached" or "database" default is "database"
+;type = database
+
+# cache connectionstring options
+# database: will use Grafana primary database.
+# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'.
+# memcache: 127.0.0.1:11211
+;connstr =
+
+#################################### Data proxy ###########################
+[dataproxy]
+
+# This enables data proxy logging, default is false
+;logging = false
+
+# How long the data proxy should wait before timing out default is 30 (seconds)
+;timeout = 30
+
+# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
+;send_user_header = false
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+;reporting_enabled = true
+
+# Set to false to disable all checks to https://grafana.net
+# for new vesions (grafana itself and plugins), check is used
+# in some UI views to notify that grafana or plugin update exists
+# This option does not cause any auto updates, nor send any information
+# only a GET request to http://grafana.com to get latest versions
+;check_for_updates = true
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+# Google Tag Manager ID, only enabled if you specify an id here
+;google_tag_manager_id =
+
+#################################### Security ####################################
+[security]
+# disable creation of admin user on first start of grafana
+;disable_initial_admin_creation = false
+
+# default admin user, created on startup
+admin_user = {{ grafana_admin_user }}
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+admin_password = {{ grafana_admin_password }}
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+# disable protection against brute force login attempts
+;disable_brute_force_login_protection = false
+
+# set to true if you host Grafana behind HTTPS. default is false.
+;cookie_secure = false
+
+# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
+;cookie_samesite = lax
+
+# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
+;allow_embedding = false
+
+# Set to true if you want to enable http strict transport security (HSTS) response header.
+# This is only sent when HTTPS is enabled in this configuration.
+# HSTS tells browsers that the site should only be accessed using HTTPS.
+# The default version will change to true in the next minor release, 6.3.
+;strict_transport_security = false
+
+# Sets how long a browser should cache HSTS. Only applied if strict_transport_security is enabled.
+;strict_transport_security_max_age_seconds = 86400
+
+# Set to true if to enable HSTS preloading option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_preload = false
+
+# Set to true if to enable the HSTS includeSubDomains option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_subdomains = false
+
+# Set to true to enable the X-Content-Type-Options response header.
+# The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
+# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
+;x_content_type_options = false
+
+# Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
+# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
+;x_xss_protection = false
+
+#################################### Snapshots ###########################
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
+# Set to true to enable this Grafana instance act as an external snapshot server and allow unauthenticated requests for
+# creating and deleting snapshots.
+;public_mode = false
+
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+#################################### Dashboards History ##################
+[dashboards]
+# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
+;versions_to_keep = 20
+
+#################################### Users ###############################
+[users]
+# disable user signup / registration
+;allow_sign_up = true
+
+# Allow non admin users to create organizations
+;allow_org_create = true
+
+# Set to true to automatically assign new users to the default organization (id 1)
+;auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+;auto_assign_org_role = Viewer
+
+# Background text for the user field on the login page
+;login_hint = email or username
+;password_hint = password
+
+# Default UI theme ("dark" or "light")
+;default_theme = dark
+
+# External user management, these options affect the organization users view
+;external_manage_link_url =
+;external_manage_link_name =
+;external_manage_info =
+
+# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
+;viewers_can_edit = false
+
+# Editors can administrate dashboard, folders and teams they create
+;editors_can_admin = false
+
+[auth]
+# Login cookie name
+;login_cookie_name = grafana_session
+
+# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days,
+;login_maximum_inactive_lifetime_days = 7
+
+# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
+;login_maximum_lifetime_days = 30
+
+# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
+;token_rotation_interval_minutes = 10
+
+# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
+;disable_login_form = false
+
+# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
+;disable_signout_menu = false
+
+# URL to redirect the user to after sign out
+;signout_redirect_url =
+
+# Set to true to attempt login with OAuth automatically, skipping the login screen.
+# This setting is ignored if multiple OAuth providers are configured.
+;oauth_auto_login = false
+
+#################################### Anonymous Auth ######################
+[auth.anonymous]
+# enable anonymous access
+;enabled = false
+
+# specify organization name that should be used for unauthenticated users
+;org_name = Main Org.
+
+# specify role for unauthenticated users
+;org_role = Viewer
+
+#################################### Github Auth ##########################
+[auth.github]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://github.com/login/oauth/authorize
+;token_url = https://github.com/login/oauth/access_token
+;api_url = https://api.github.com/user
+;team_ids =
+;allowed_organizations =
+
+#################################### Google Auth ##########################
+[auth.google]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_client_id
+;client_secret = some_client_secret
+;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
+;auth_url = https://accounts.google.com/o/oauth2/auth
+;token_url = https://accounts.google.com/o/oauth2/token
+;api_url = https://www.googleapis.com/oauth2/v1/userinfo
+;allowed_domains =
+
+#################################### Generic OAuth ##########################
+[auth.generic_oauth]
+;enabled = false
+;name = OAuth
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;email_attribute_name = email:primary
+;email_attribute_path =
+;auth_url = https://foo.bar/login/oauth/authorize
+;token_url = https://foo.bar/login/oauth/access_token
+;api_url = https://foo.bar/user
+;team_ids =
+;allowed_organizations =
+;role_attribute_path =
+;tls_skip_verify_insecure = false
+;tls_client_cert =
+;tls_client_key =
+;tls_client_ca =
+
+; Set to true to enable sending client_id and client_secret via POST body instead of Basic authentication HTTP header
+; This might be required if the OAuth provider is not RFC6749 compliant, only supporting credentials passed via POST payload
+;send_client_credentials_via_post = false
+
+#################################### SAML Auth ###########################
+[auth.saml] # Enterprise only
+# Defaults to false. If true, the feature is enabled.
+;enabled = false
+
+# Base64-encoded public X.509 certificate. Used to sign requests to the IdP
+;certificate =
+
+# Path to the public X.509 certificate. Used to sign requests to the IdP
+;certificate_path =
+
+# Base64-encoded private key. Used to decrypt assertions from the IdP
+;private_key =
+
+;# Path to the private key. Used to decrypt assertions from the IdP
+;private_key_path =
+
+# Base64-encoded IdP SAML metadata XML. Used to verify and obtain binding locations from the IdP
+;idp_metadata =
+
+# Path to the SAML metadata XML. Used to verify and obtain binding locations from the IdP
+;idp_metadata_path =
+
+# URL to fetch SAML IdP metadata. Used to verify and obtain binding locations from the IdP
+;idp_metadata_url =
+
+# Duration, since the IdP issued a response and the SP is allowed to process it. Defaults to 90 seconds.
+;max_issue_delay = 90s
+
+# Duration, for how long the SP's metadata should be valid. Defaults to 48 hours.
+;metadata_valid_duration = 48h
+
+# Friendly name or name of the attribute within the SAML assertion to use as the user's name
+;assertion_attribute_name = displayName
+
+# Friendly name or name of the attribute within the SAML assertion to use as the user's login handle
+;assertion_attribute_login = mail
+
+# Friendly name or name of the attribute within the SAML assertion to use as the user's email
+;assertion_attribute_email = mail
+
+#################################### Grafana.com Auth ####################
+[auth.grafana_com]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email
+;allowed_organizations =
+
+#################################### Auth Proxy ##########################
+[auth.proxy]
+;enabled = false
+;header_name = X-WEBAUTH-USER
+;header_property = username
+;auto_sign_up = true
+;sync_ttl = 60
+;whitelist = 192.168.1.1, 192.168.2.1
+;headers = Email:X-User-Email, Name:X-User-Name
+# Read the auth proxy docs for details on what the setting below enables
+;enable_login_token = false
+
+#################################### Basic Auth ##########################
+[auth.basic]
+;enabled = true
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+;enabled = false
+;config_file = /etc/grafana/ldap.toml
+;allow_sign_up = true
+
+# LDAP backround sync (Enterprise only)
+# At 1 am every day
+;sync_cron = "0 0 1 * * *"
+;active_sync_enabled = true
+
+#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+;from_name = Grafana
+# EHLO identity in SMTP dialog (defaults to instance_name)
+;ehlo_identity = dashboard.example.com
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+;mode = console file
+
+# Either "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+
+# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
+;filters =
+
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# This enables automated log rotate(switch of following options), default is true
+;log_rotate = true
+
+# Max line number of single file, default is 1000000
+;max_lines = 1000000
+
+# Max size shift of single file, default is 28 means 1 << 28, 256MB
+;max_size_shift = 28
+
+# Segment log daily, default is true
+;daily_rotate = true
+
+# Expired days of log file(delete after max days), default is 7
+;max_days = 7
+
+[log.syslog]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
+;network =
+;address =
+
+# Syslog facility. user, daemon and local0 through local7 are valid.
+;facility =
+
+# Syslog tag. By default, the process' argv[0] is used.
+;tag =
+
+#################################### Alerting ############################
+[alerting]
+# Disable alerting engine & UI features
+;enabled = true
+# Makes it possible to turn off alert rule execution but alerting UI is visible
+;execute_alerts = true
+
+# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
+;error_or_timeout = alerting
+
+# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
+;nodata_or_nullvalues = no_data
+
+# Alert notifications can include images, but rendering many images at the same time can overload the server
+# This limit will protect the server from render overloading and make sure notifications are sent out quickly
+;concurrent_render_limit = 5
+
+
+# Default setting for alert calculation timeout. Default value is 30
+;evaluation_timeout_seconds = 30
+
+# Default setting for alert notification timeout. Default value is 30
+;notification_timeout_seconds = 30
+
+# Default setting for max attempts to sending alert notifications. Default value is 3
+;max_attempts = 3
+
+#################################### Explore #############################
+[explore]
+# Enable the Explore section
+;enabled = true
+
+#################################### Internal Grafana Metrics ##########################
+# Metrics available at HTTP API Url /metrics
+[metrics]
+# Disable / Enable internal metrics
+;enabled           = true
+# Disable total stats (stat_totals_*) metrics to be generated
+;disable_total_stats = false
+
+# Publish interval
+;interval_seconds  = 10
+
+# Send internal metrics to Graphite
+[metrics.graphite]
+# Enable by setting the address setting (ex localhost:2003)
+;address =
+;prefix = prod.grafana.%(instance_name)s.
+
+#################################### Distributed tracing ############
+[tracing.jaeger]
+# Enable by setting the address sending traces to jaeger (ex localhost:6831)
+;address = localhost:6831
+# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
+;always_included_tag = tag1:value1
+# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
+;sampler_type = const
+# jaeger samplerconfig param
+# for "const" sampler, 0 or 1 for always false/true respectively
+# for "probabilistic" sampler, a probability between 0 and 1
+# for "rateLimiting" sampler, the number of spans per second
+# for "remote" sampler, param is the same as for "probabilistic"
+# and indicates the initial sampling rate before the actual one
+# is received from the mothership
+;sampler_param = 1
+# Whether or not to use Zipkin propagation (x-b3- HTTP headers).
+;zipkin_propagation = false
+# Setting this to true disables shared RPC spans.
+# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
+;disable_shared_zipkin_spans = false
+
+#################################### Grafana.com integration  ##########################
+# Url used to import dashboards directly from Grafana.com
+[grafana_com]
+;url = https://grafana.com
+
+#################################### External image storage ##########################
+[external_image_storage]
+# Used for uploading images to public servers so they can be included in slack/email messages.
+# you can choose between (s3, webdav, gcs, azure_blob, local)
+;provider =
+
+[external_image_storage.s3]
+;bucket =
+;region =
+;path =
+;access_key =
+;secret_key =
+
+[external_image_storage.webdav]
+;url =
+;public_url =
+;username =
+;password =
+
+[external_image_storage.gcs]
+;key_file =
+;bucket =
+;path =
+
+[external_image_storage.azure_blob]
+;account_name =
+;account_key =
+;container_name =
+
+[external_image_storage.local]
+# does not require any configuration
+
+[rendering]
+# Options to configure a remote HTTP image rendering service, e.g. using https://github.com/grafana/grafana-image-renderer.
+# URL to a remote HTTP image renderer service, e.g. http://localhost:8081/render, will enable Grafana to render panels and dashboards to PNG-images using HTTP requests to an external service.
+;server_url =
+# If the remote HTTP image renderer service runs on a different server than the Grafana server you may have to configure this to a URL where Grafana is reachable, e.g. http://grafana.domain/.
+;callback_url =
+
+[enterprise]
+# Path to a valid Grafana Enterprise license.jwt file
+;license_path =
+
+[panels]
+# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
+;disable_sanitize_html = false
+
+[plugins]
+;enable_alpha = false
+;app_tls_skip_verify_insecure = false

roles/monitoring/templates/nginx/alertmanager.conf.j2

@@ -0,0 +1,11 @@
+location {{ alertmanager_web_path }}/ {
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+
+    auth_basic "Authentication required for Alert Manager";
+    auth_basic_user_file /etc/nginx/.htpasswd-alertmanager;
+
+    proxy_pass http://localhost:{{ alertmanager_port }}/;
+}

roles/monitoring/templates/nginx/grafana.conf.j2

@@ -0,0 +1,5 @@
+{{ ansible_managed | comment }}
+
+location {{ grafana_web_path }}/ {
+    proxy_pass http://localhost:{{ grafana_port }}/;
+}

roles/monitoring/templates/nginx/prometheus.conf.j2

@@ -0,0 +1,11 @@
+location {{ prometheus_web_path }}/ {
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Host $http_host;
+    proxy_redirect off;
+
+    auth_basic "Authentication required";
+    auth_basic_user_file /etc/nginx/.htpasswd-prometheus;
+
+    proxy_pass http://localhost:{{ prometheus_port }}/;
+}

roles/monitoring/templates/prometheus.env.j2

@@ -0,0 +1,60 @@
+# Set the command-line arguments to pass to the server.
+ARGS="--web.external-url={{ prometheus_protocol }}://{{ prometheus_domain }}{{ prometheus_path }} \
+  --web.route-prefix=/ \
+  --web.listen-address='0.0.0.0:{{ prometheus_port }}'"
+
+
+# Prometheus supports the following options:
+#  --config.file="/etc/prometheus/prometheus.yml"
+#                             Prometheus configuration file path.
+#  --web.listen-address="0.0.0.0:9090"
+#                             Address to listen on for UI, API, and telemetry.
+#  --web.read-timeout=5m      Maximum duration before timing out read of the
+#                             request, and closing idle connections.
+#  --web.max-connections=512  Maximum number of simultaneous connections.
+#  --web.external-url=<URL>   The URL under which Prometheus is externally
+#                             reachable (for example, if Prometheus is served
+#                             via a reverse proxy). Used for generating
+#                             relative and absolute links back to Prometheus
+#                             itself. If the URL has a path portion, it will
+#                             be used to prefix all HTTP endpoints served by
+#                             Prometheus. If omitted, relevant URL components
+#                             will be derived automatically.
+#  --web.route-prefix=<path>  Prefix for the internal routes of web endpoints.
+#                             Defaults to path of --web.external-url.
+#  --web.local-assets="/usr/share/prometheus/web/"
+#                             Path to static asset/templates directory.
+#  --web.user-assets=<path>   Path to static asset directory, available at
+#                             /user.
+#  --web.enable-lifecycle     Enable shutdown and reload via HTTP request.
+#  --web.enable-admin-api     Enables API endpoints for admin control actions.
+#  --web.console.templates="/etc/prometheus/consoles"
+#                             Path to the console template directory,
+#                             available at /consoles.
+#  --web.console.libraries="/etc/prometheus/console_libraries"
+#                             Path to the console library directory.
+#  --storage.tsdb.path="/var/lib/prometheus/metrics2/"
+#                             Base path for metrics storage.
+#  --storage.tsdb.min-block-duration=2h
+#                             Minimum duration of a data block before being
+#                             persisted.
+#  --storage.tsdb.max-block-duration=<duration>
+#                             Maximum duration compacted blocks may span.
+#                             (Defaults to 10% of the retention period)
+#  --storage.tsdb.retention=15d
+#                             How long to retain samples in the storage.
+#  --storage.tsdb.no-lockfile
+#                             Do not create lockfile in data directory.
+#  --alertmanager.notification-queue-capacity=10000
+#                             The capacity of the queue for pending alert
+#                             manager notifications.
+#  --alertmanager.timeout=10s
+#                             Timeout for sending alerts to Alertmanager.
+#  --query.lookback-delta=5m  The delta difference allowed for retrieving
+#                             metrics during expression evaluations.
+#  --query.timeout=2m         Maximum time a query may take before being
+#                             aborted.
+#  --query.max-concurrency=20
+#                             Maximum number of queries executed concurrently.
+#  --log.level=info           Only log messages with the given severity or
+#                             above. One of: [debug, info, warn, error]

roles/monitoring/templates/prometheus.yml.j2

@@ -0,0 +1,31 @@
+global:
+  scrape_interval: 5s
+  scrape_timeout: 5s
+
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      - localhost:{{ alertmanager_port }}
+    scheme: http
+    timeout: 10s
+
+rule_files:
+- '/etc/prometheus/rules.d/*.yml'
+
+scrape_configs:
+{% for host in groups['all'] %}
+{% with
+  node_exporter_password = hostvars[host].node_exporter_password | default(node_exporter_password),
+  node_exporter_path = hostvars[host].node_exporter_path | default(node_exporter_path)
+%}
+- job_name: 'node-exporter-{{ host }}'
+  basic_auth:
+    username: admin
+    password: {{ node_exporter_password }}
+  metrics_path: {{ node_exporter_path }}
+  static_configs:
+  - targets:
+    - '{{ host }}'
+{% endwith %}
+{% endfor %}

roles/monitoring/templates/prometheus_custom.yml.j2

@@ -0,0 +1,40 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+{{ ansible_managed | comment }}
+# http://prometheus.io/docs/operating/configuration/
+
+global:
+  scrape_interval: 5s
+  scrape_timeout: 5s
+
+rule_files:
+  - {{ prometheus_config_dir }}/rules/*.rules
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets:
+      - "localhost:{{ alertmanager_port }}"
+    timeout: 10s
+  {% if prometheus_alert_relabel_configs | length > 0 %}
+  alert_relabel_configs:
+  {{ prometheus_alert_relabel_configs | to_nice_yaml(indent=2) | indent(2,False) }}
+  {% endif %}
+
+scrape_configs:
+{% for host in groups['all'] %}
+{% with
+  node_exporter_password = hostvars[host].node_exporter_password | default(node_exporter_password),
+  node_exporter_path = hostvars[host].node_exporter_path | default(node_exporter_path),
+  node_exporter_public_port = hostvars[host].node_exporter_public_port | default(node_exporter_public_port)
+%}
+- job_name: 'node-exporter-{{ host }}'
+  basic_auth:
+    username: admin
+    password: {{ node_exporter_password }}
+  metrics_path: {{ node_exporter_path }}
+  static_configs:
+  - targets:
+    - '{{ host }}:{{ node_exporter_public_port }}'
+{% endwith %}
+{% endfor %}