Prometheus support (#450)
Co-authored-by: nils måsén <nils@piksel.se> Co-authored-by: MihailITPlace <ya.halo-halo@yandex.ru> Co-authored-by: Sebastiaan Tammer <sebastiaantammer@gmail.com>pull/757/head
Simon Aronsson
4 years ago
committed by
GitHub
parent
35490c853d
commit
d7d5b25882
@ -0,0 +1,43 @@
|
||||
version: '3.7'
|
||||
|
||||
services:
|
||||
watchtower:
|
||||
container_name: watchtower
|
||||
build:
|
||||
context: ./
|
||||
dockerfile: dockerfiles/Dockerfile.dev-self-contained
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
ports:
|
||||
- 8080:8080
|
||||
command: --interval 10 --http-api-metrics --http-api-token demotoken --debug prometheus grafana parent child
|
||||
prometheus:
|
||||
container_name: prometheus
|
||||
image: prom/prometheus
|
||||
volumes:
|
||||
- ./prometheus/:/etc/prometheus/
|
||||
- prometheus:/prometheus/
|
||||
ports:
|
||||
- 9090:9090
|
||||
grafana:
|
||||
container_name: grafana
|
||||
image: grafana/grafana
|
||||
ports:
|
||||
- 3000:3000
|
||||
environment:
|
||||
GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource
|
||||
volumes:
|
||||
- grafana:/var/lib/grafana
|
||||
- ./grafana:/etc/grafana/provisioning
|
||||
parent:
|
||||
image: nginx
|
||||
container_name: parent
|
||||
child:
|
||||
image: nginx:alpine
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.depends-on: parent
|
||||
container_name: child
|
||||
|
||||
volumes:
|
||||
prometheus: {}
|
||||
grafana: {}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
@ -0,0 +1,26 @@
|
||||
> **⚠️ Experimental feature**
|
||||
>
|
||||
> This feature was added in v1.0.4 and is still considered experimental.
|
||||
> If you notice any strange behavior, please raise a ticket in the repository issues.
|
||||
|
||||
Metrics can be used to track how Watchtower behaves over time.
|
||||
|
||||
To use this feature, you have to set an [API token](arguments.md#http-api-token) and [enable the metrics API](arguments.md#http-api-metrics),
|
||||
as well as creating a port mapping for your container for port `8080`.
|
||||
|
||||
## Available Metrics
|
||||
|
||||
| Name | Type | Description |
|
||||
| ------------------------------- | ------- | --------------------------------------------------------------------------- |
|
||||
| `watchtower_containers_scanned` | Gauge | Number of containers scanned for changes by watchtower during the last scan |
|
||||
| `watchtower_containers_updated` | Gauge | Number of containers updated by watchtower during the last scan |
|
||||
| `watchtower_containers_failed` | Gauge | Number of containers where update failed during the last scan |
|
||||
| `watchtower_scans_total` | Counter | Number of scans since the watchtower started |
|
||||
| `watchtower_scans_skipped` | Counter | Number of skipped scans since watchtower started |
|
||||
|
||||
## Demo
|
||||
|
||||
The repository contains a demo with prometheus and grafana, available through `docker-compose.yml`. This demo
|
||||
is preconfigured with a dashboard, which will look something like this:
|
||||
|
||||
|
||||
@ -0,0 +1,293 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": 1,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 1,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.3.6",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "watchtower_scans_total",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Total Scans",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": null,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "watchtower_containers_scanned{instance=\"watchtower:8080\", job=\"watchtower\"}"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Scanned"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "watchtower_containers_failed{instance=\"watchtower:8080\", job=\"watchtower\"}"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Faled"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "watchtower_containers_updated{instance=\"watchtower:8080\", job=\"watchtower\"}"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "displayName",
|
||||
"value": "Updated"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 6,
|
||||
"x": 1,
|
||||
"y": 0
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 5,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "7.3.6",
|
||||
"pointradius": 2,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "watchtower_containers_scanned",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "watchtower_containers_failed",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "watchtower_containers_updated",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "C"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "Container Updates",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"decimals": 0,
|
||||
"format": "short",
|
||||
"label": "",
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": "0",
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"label": null,
|
||||
"logBase": 1,
|
||||
"max": null,
|
||||
"min": null,
|
||||
"show": true
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 1,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "7.3.6",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "watchtower_scans_skipped",
|
||||
"interval": "",
|
||||
"legendFormat": "",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "Skipped Scans",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": false,
|
||||
"schemaVersion": 26,
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "Watchtower",
|
||||
"uid": "d7bdoT-Gz",
|
||||
"version": 1
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: 'Prometheus'
|
||||
orgId: 1
|
||||
folder: ''
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /etc/grafana/provisioning/dashboards
|
||||
@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
@ -1,63 +1,76 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var (
|
||||
lock chan bool
|
||||
)
|
||||
const tokenMissingMsg = "api token is empty or has not been set. exiting"
|
||||
|
||||
func init() {
|
||||
lock = make(chan bool, 1)
|
||||
lock <- true
|
||||
// API is the http server responsible for serving the HTTP API endpoints
|
||||
type API struct {
|
||||
Token string
|
||||
hasHandlers bool
|
||||
}
|
||||
|
||||
// SetupHTTPUpdates configures the endpoint needed for triggering updates via http
|
||||
func SetupHTTPUpdates(apiToken string, updateFunction func()) error {
|
||||
if apiToken == "" {
|
||||
return errors.New("api token is empty or has not been set. not starting api")
|
||||
// New is a factory function creating a new API instance
|
||||
func New(token string) *API {
|
||||
return &API{
|
||||
Token: token,
|
||||
hasHandlers: false,
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("Watchtower HTTP API started.")
|
||||
|
||||
http.HandleFunc("/v1/update", func(w http.ResponseWriter, r *http.Request) {
|
||||
log.Info("Updates triggered by HTTP API request.")
|
||||
|
||||
_, err := io.Copy(os.Stdout, r.Body)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
// RequireToken is wrapper around http.HandleFunc that checks token validity
|
||||
func (api *API) RequireToken(fn http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", api.Token) {
|
||||
log.Errorf("Invalid token \"%s\"", r.Header.Get("Authorization"))
|
||||
log.Debugf("Expected token to be \"%s\"", api.Token)
|
||||
return
|
||||
}
|
||||
log.Println("Valid token found.")
|
||||
fn(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
if r.Header.Get("Token") != apiToken {
|
||||
log.Println("Invalid token. Not updating.")
|
||||
return
|
||||
// RegisterFunc is a wrapper around http.HandleFunc that also sets the flag used to determine whether to launch the API
|
||||
func (api *API) RegisterFunc(path string, fn http.HandlerFunc) {
|
||||
api.hasHandlers = true
|
||||
http.HandleFunc(path, api.RequireToken(fn))
|
||||
}
|
||||
|
||||
// RegisterHandler is a wrapper around http.Handler that also sets the flag used to determine whether to launch the API
|
||||
func (api *API) RegisterHandler(path string, handler http.Handler) {
|
||||
api.hasHandlers = true
|
||||
http.Handle(path, api.RequireToken(handler.ServeHTTP))
|
||||
}
|
||||
|
||||
log.Println("Valid token found. Attempting to update.")
|
||||
// Start the API and serve over HTTP. Requires an API Token to be set.
|
||||
func (api *API) Start(block bool) error {
|
||||
|
||||
select {
|
||||
case chanValue := <-lock:
|
||||
defer func() { lock <- chanValue }()
|
||||
updateFunction()
|
||||
default:
|
||||
log.Debug("Skipped. Another update already running.")
|
||||
if !api.hasHandlers {
|
||||
log.Debug("Watchtower HTTP API skipped.")
|
||||
return nil
|
||||
}
|
||||
|
||||
})
|
||||
if api.Token == "" {
|
||||
log.Fatal(tokenMissingMsg)
|
||||
}
|
||||
|
||||
log.Info("Watchtower HTTP API started.")
|
||||
if block {
|
||||
runHTTPServer()
|
||||
} else {
|
||||
go func() {
|
||||
runHTTPServer()
|
||||
}()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WaitForHTTPUpdates starts the http server and listens for requests.
|
||||
func WaitForHTTPUpdates() error {
|
||||
func runHTTPServer() {
|
||||
log.Info("Serving HTTP")
|
||||
log.Fatal(http.ListenAndServe(":8080", nil))
|
||||
os.Exit(0)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -0,0 +1,27 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/containrrr/watchtower/pkg/metrics"
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// Handler is an HTTP handle for serving metric data
|
||||
type Handler struct {
|
||||
Path string
|
||||
Handle http.HandlerFunc
|
||||
Metrics *metrics.Metrics
|
||||
}
|
||||
|
||||
// New is a factory function creating a new Metrics instance
|
||||
func New() *Handler {
|
||||
m := metrics.Default()
|
||||
handler := promhttp.Handler()
|
||||
|
||||
return &Handler{
|
||||
Path: "/v1/metrics",
|
||||
Handle: handler.ServeHTTP,
|
||||
Metrics: m,
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
package metrics_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/containrrr/watchtower/pkg/metrics"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/containrrr/watchtower/pkg/api"
|
||||
metricsAPI "github.com/containrrr/watchtower/pkg/api/metrics"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
const Token = "123123123"
|
||||
|
||||
func TestContainer(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Metrics Suite")
|
||||
}
|
||||
|
||||
func runTestServer(m *metricsAPI.Handler) {
|
||||
http.Handle(m.Path, m.Handle)
|
||||
go func() {
|
||||
http.ListenAndServe(":8080", nil)
|
||||
}()
|
||||
}
|
||||
|
||||
func getWithToken(c http.Client, url string) (*http.Response, error) {
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", Token))
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
var _ = Describe("the metrics", func() {
|
||||
httpAPI := api.New(Token)
|
||||
m := metricsAPI.New()
|
||||
httpAPI.RegisterHandler(m.Path, m.Handle)
|
||||
httpAPI.Start(false)
|
||||
|
||||
// We should likely split this into multiple tests, but as prometheus requires a restart of the binary
|
||||
// to reset the metrics and gauges, we'll just do it all at once.
|
||||
|
||||
It("should serve metrics", func() {
|
||||
metric := &metrics.Metric{
|
||||
Scanned: 4,
|
||||
Updated: 3,
|
||||
Failed: 1,
|
||||
}
|
||||
metrics.RegisterScan(metric)
|
||||
c := http.Client{}
|
||||
res, err := getWithToken(c, "http://localhost:8080/v1/metrics")
|
||||
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
contents, err := ioutil.ReadAll(res.Body)
|
||||
fmt.Printf("%s\n", string(contents))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_containers_updated 3"))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_containers_failed 1"))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_containers_scanned 4"))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_scans_total 1"))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_scans_skipped 0"))
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
metrics.RegisterScan(nil)
|
||||
}
|
||||
|
||||
res, err = getWithToken(c, "http://localhost:8080/v1/metrics")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
contents, err = ioutil.ReadAll(res.Body)
|
||||
fmt.Printf("%s\n", string(contents))
|
||||
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_scans_total 4"))
|
||||
Expect(string(contents)).To(ContainSubstring("watchtower_scans_skipped 3"))
|
||||
})
|
||||
})
|
||||
@ -0,0 +1,50 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
var (
|
||||
lock chan bool
|
||||
)
|
||||
|
||||
// New is a factory function creating a new Handler instance
|
||||
func New(updateFn func()) *Handler {
|
||||
lock = make(chan bool, 1)
|
||||
lock <- true
|
||||
|
||||
return &Handler{
|
||||
fn: updateFn,
|
||||
Path: "/v1/update",
|
||||
}
|
||||
}
|
||||
|
||||
// Handler is an API handler used for triggering container update scans
|
||||
type Handler struct {
|
||||
fn func()
|
||||
Path string
|
||||
}
|
||||
|
||||
// Handle is the actual http.Handle function doing all the heavy lifting
|
||||
func (handle *Handler) Handle(w http.ResponseWriter, r *http.Request) {
|
||||
log.Info("Updates triggered by HTTP API request.")
|
||||
|
||||
_, err := io.Copy(os.Stdout, r.Body)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case chanValue := <-lock:
|
||||
defer func() { lock <- chanValue }()
|
||||
handle.fn()
|
||||
default:
|
||||
log.Debug("Skipped. Another update already running.")
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,91 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var metrics *Metrics
|
||||
|
||||
// Metric is the data points of a single scan
|
||||
type Metric struct {
|
||||
Scanned int
|
||||
Updated int
|
||||
Failed int
|
||||
}
|
||||
|
||||
// Metrics is the handler processing all individual scan metrics
|
||||
type Metrics struct {
|
||||
channel chan *Metric
|
||||
scanned prometheus.Gauge
|
||||
updated prometheus.Gauge
|
||||
failed prometheus.Gauge
|
||||
total prometheus.Counter
|
||||
skipped prometheus.Counter
|
||||
}
|
||||
|
||||
// Register registers metrics for an executed scan
|
||||
func (metrics *Metrics) Register(metric *Metric) {
|
||||
metrics.channel <- metric
|
||||
}
|
||||
|
||||
// Default creates a new metrics handler if none exists, otherwise returns the existing one
|
||||
func Default() *Metrics {
|
||||
if metrics != nil {
|
||||
return metrics
|
||||
}
|
||||
|
||||
metrics = &Metrics{
|
||||
scanned: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "watchtower_containers_scanned",
|
||||
Help: "Number of containers scanned for changes by watchtower during the last scan",
|
||||
}),
|
||||
updated: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "watchtower_containers_updated",
|
||||
Help: "Number of containers updated by watchtower during the last scan",
|
||||
}),
|
||||
failed: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "watchtower_containers_failed",
|
||||
Help: "Number of containers where update failed during the last scan",
|
||||
}),
|
||||
total: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "watchtower_scans_total",
|
||||
Help: "Number of scans since the watchtower started",
|
||||
}),
|
||||
skipped: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "watchtower_scans_skipped",
|
||||
Help: "Number of skipped scans since watchtower started",
|
||||
}),
|
||||
channel: make(chan *Metric, 10),
|
||||
}
|
||||
|
||||
go metrics.HandleUpdate(metrics.channel)
|
||||
|
||||
return metrics
|
||||
}
|
||||
|
||||
// RegisterScan fetches a metric handler and enqueues a metric
|
||||
func RegisterScan(metric *Metric) {
|
||||
metrics := Default()
|
||||
metrics.Register(metric)
|
||||
}
|
||||
|
||||
// HandleUpdate dequeue the metric channel and processes it
|
||||
func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
|
||||
for change := range channel {
|
||||
if change == nil {
|
||||
// Update was skipped and rescheduled
|
||||
metrics.total.Inc()
|
||||
metrics.skipped.Inc()
|
||||
metrics.scanned.Set(0)
|
||||
metrics.updated.Set(0)
|
||||
metrics.failed.Set(0)
|
||||
continue
|
||||
}
|
||||
// Update metrics with the new values
|
||||
metrics.total.Inc()
|
||||
metrics.scanned.Set(float64(change.Scanned))
|
||||
metrics.updated.Set(float64(change.Updated))
|
||||
metrics.failed.Set(float64(change.Failed))
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
scrape_configs:
|
||||
- job_name: watchtower
|
||||
scrape_interval: 5s
|
||||
metrics_path: /v1/metrics
|
||||
bearer_token: demotoken
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'watchtower:8080'
|
||||
|
||||
Loading…
Reference in New Issue