From d7d5b2588277f0e07895a63c12a2117262bdf9fc Mon Sep 17 00:00:00 2001 From: Simon Aronsson Date: Wed, 6 Jan 2021 22:28:32 +0100 Subject: [PATCH] Prometheus support (#450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: nils måsén Co-authored-by: MihailITPlace Co-authored-by: Sebastiaan Tammer --- cmd/root.go | 37 +++- docker-compose.yml | 43 ++++ docs/arguments.md | 16 +- docs/assets/grafana-dashboard.png | Bin 0 -> 32994 bytes docs/metrics.md | 26 +++ docs/notifications.md | 21 +- go.mod | 1 + grafana/dashboards/dashboard.json | 293 +++++++++++++++++++++++++ grafana/dashboards/dashboard.yml | 11 + grafana/datasources/datasource.yml | 8 + internal/actions/actions_suite_test.go | 3 +- internal/actions/update.go | 81 +++++-- internal/actions/update_test.go | 10 +- internal/flags/flags.go | 9 +- mkdocs.yml | 1 + pkg/api/api.go | 93 ++++---- pkg/api/metrics/metrics.go | 27 +++ pkg/api/metrics/metrics_test.go | 77 +++++++ pkg/api/update/update.go | 50 +++++ pkg/metrics/metrics.go | 91 ++++++++ pkg/notifications/gotify.go | 2 +- pkg/notifications/msteams.go | 2 +- prometheus/prometheus.yml | 9 + 23 files changed, 812 insertions(+), 99 deletions(-) create mode 100644 docker-compose.yml create mode 100644 docs/assets/grafana-dashboard.png create mode 100644 docs/metrics.md create mode 100644 grafana/dashboards/dashboard.json create mode 100644 grafana/dashboards/dashboard.yml create mode 100644 grafana/datasources/datasource.yml create mode 100644 pkg/api/metrics/metrics.go create mode 100644 pkg/api/metrics/metrics_test.go create mode 100644 pkg/api/update/update.go create mode 100644 pkg/metrics/metrics.go create mode 100644 prometheus/prometheus.yml diff --git a/cmd/root.go b/cmd/root.go index 4308dd1..0aeeac6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,12 +1,16 @@ package cmd import ( + metrics2 "github.com/containrrr/watchtower/pkg/metrics" "os" "os/signal" "strconv" "syscall" "time" + "github.com/containrrr/watchtower/pkg/api/metrics" + "github.com/containrrr/watchtower/pkg/api/update" + "github.com/containrrr/watchtower/internal/actions" "github.com/containrrr/watchtower/internal/flags" "github.com/containrrr/watchtower/pkg/api" @@ -144,7 +148,10 @@ func PreRun(cmd *cobra.Command, args []string) { func Run(c *cobra.Command, names []string) { filter := filters.BuildFilter(names, enableLabel, scope) runOnce, _ := c.PersistentFlags().GetBool("run-once") - httpAPI, _ := c.PersistentFlags().GetBool("http-api") + enableUpdateAPI, _ := c.PersistentFlags().GetBool("http-api-update") + enableMetricsAPI, _ := c.PersistentFlags().GetBool("http-api-metrics") + + apiToken, _ := c.PersistentFlags().GetString("http-api-token") if runOnce { if noStartupMessage, _ := c.PersistentFlags().GetBool("no-startup-message"); !noStartupMessage { @@ -160,17 +167,20 @@ func Run(c *cobra.Command, names []string) { log.Fatal(err) } - if httpAPI { - apiToken, _ := c.PersistentFlags().GetString("http-api-token") + httpAPI := api.New(apiToken) - if err := api.SetupHTTPUpdates(apiToken, func() { runUpdatesWithNotifications(filter) }); err != nil { - log.Fatal(err) - os.Exit(1) - } + if enableUpdateAPI { + updateHandler := update.New(func() { runUpdatesWithNotifications(filter) }) + httpAPI.RegisterFunc(updateHandler.Path, updateHandler.Handle) + } - api.WaitForHTTPUpdates() + if enableMetricsAPI { + metricsHandler := metrics.New() + httpAPI.RegisterHandler(metricsHandler.Path, metricsHandler.Handle) } + httpAPI.Start(enableUpdateAPI) + if err := runUpgradesOnSchedule(c, filter); err != nil { log.Error(err) } @@ -189,8 +199,11 @@ func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error { select { case v := <-tryLockSem: defer func() { tryLockSem <- v }() - runUpdatesWithNotifications(filter) + metric := runUpdatesWithNotifications(filter) + metrics2.RegisterScan(metric) default: + // Update was skipped + metrics2.RegisterScan(nil) log.Debug("Skipped another update already running.") } @@ -222,7 +235,8 @@ func runUpgradesOnSchedule(c *cobra.Command, filter t.Filter) error { return nil } -func runUpdatesWithNotifications(filter t.Filter) { +func runUpdatesWithNotifications(filter t.Filter) *metrics2.Metric { + notifier.StartNotification() updateParams := t.UpdateParams{ Filter: filter, @@ -233,9 +247,10 @@ func runUpdatesWithNotifications(filter t.Filter) { LifecycleHooks: lifecycleHooks, RollingRestart: rollingRestart, } - err := actions.Update(client, updateParams) + metrics, err := actions.Update(client, updateParams) if err != nil { log.Println(err) } notifier.SendNotification() + return metrics } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b0a3373 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +version: '3.7' + +services: + watchtower: + container_name: watchtower + build: + context: ./ + dockerfile: dockerfiles/Dockerfile.dev-self-contained + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - 8080:8080 + command: --interval 10 --http-api-metrics --http-api-token demotoken --debug prometheus grafana parent child + prometheus: + container_name: prometheus + image: prom/prometheus + volumes: + - ./prometheus/:/etc/prometheus/ + - prometheus:/prometheus/ + ports: + - 9090:9090 + grafana: + container_name: grafana + image: grafana/grafana + ports: + - 3000:3000 + environment: + GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource + volumes: + - grafana:/var/lib/grafana + - ./grafana:/etc/grafana/provisioning + parent: + image: nginx + container_name: parent + child: + image: nginx:alpine + labels: + com.centurylinklabs.watchtower.depends-on: parent + container_name: child + +volumes: + prometheus: {} + grafana: {} diff --git a/docs/arguments.md b/docs/arguments.md index a36c438..4fb56c6 100644 --- a/docs/arguments.md +++ b/docs/arguments.md @@ -164,7 +164,7 @@ Environment Variable: WATCHTOWER_LABEL_ENABLE ## Without updating containers Will only monitor for new images, send notifications and invoke the [pre-check/post-check hooks](https://containrrr.dev/watchtower/lifecycle-hooks/), but will **not** update the containers. -> ### ⚠️ Please note +> **⚠️ Please note** > > Due to Docker API limitations the latest image will still be pulled from the registry. @@ -238,9 +238,7 @@ Sets an authentication token to HTTP API requests. Environment Variable: WATCHTOWER_HTTP_API_TOKEN Type: String Default: - -``` - -## Filter by scope +```## Filter by scope Update containers that have a `com.centurylinklabs.watchtower.scope` label set with the same value as the given argument. This enables [running multiple instances](https://containrrr.github.io/watchtower/running-multiple-instances). ``` @@ -250,6 +248,16 @@ Environment Variable: WATCHTOWER_SCOPE Default: - ``` +## HTTP API Metrics +Enables a metrics endpoint, exposing prometheus metrics via HTTP. See [Metrics](metrics.md) for details. + +``` + Argument: --http-api-metrics +Environment Variable: WATCHTOWER_HTTP_API_METRICS + Type: Boolean + Default: false +``` + ## Scheduling [Cron expression](https://pkg.go.dev/github.com/robfig/cron@v1.2.0?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression can be defined, but not both. An example: `--schedule "0 0 4 * * *"` diff --git a/docs/assets/grafana-dashboard.png b/docs/assets/grafana-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..faab4229cafa6bbf67e82f994fac5a6b012ddd6c GIT binary patch literal 32994 zcmce;2UJsO)HbT44hRP9C}4u3qJvVTmwTefTgUA=Pg z#+EJHpvdja^l4A>mYqla9IF!#^4;~%-pluLCeGg5z3pTEqiH!W+5p;yFpX{s?m6Oa z5>qey>1yt`i#KDj&#*7*mMa$JZylD@Uh#cF`QWU+k{8Edi5UBaTqPnKAdFegRq2_; zWdv2=74YwzFhJOrE$1(<z6ytKv0gLDTjfW9-Qi7!{Ky67Ab0q;wS&_bWdm%HY-2Etm*cBUg-$2ANx=O zZODE!E%`pFng79)>swKn&qsx(6_2lCk$!j`20pRc&-j7W6s zu)w45pBziPiV6(KIQ5p5HrO0e1Hopu@iQ^uMXOrdjQk|L>w83KXyhaz=nj>Z;)eanit?WUutaIvC3P99~2=JId zm7+UOojoT$95YV+h5n|h&1Xlv@mZp zBBnD%{m{B~qwO(`B>A6=v`AhJI$^7gxlC8 zU2)M@kyQyfm{V{m0DRVj!reN)JI%Ohb|KB9Eg5?6n-@NloR7869<@#?c!rOx6?#i^|Ed zcFaq@)fbncrtnC7T$!H`8q4ZjIiJO;X1SP?#~#?Vovwa(?~xaiV1s?Uq=l=D7}w8i zO31fGn9t4p9*?g-LSYu5ELaVnRU68XFv3(}`5O;qR=83k-rKXK`)O#U*k|wM4)TLJ zo(Qi@g^?AOCl8jN3N(-rYn72K5X2`vS~MQ9FP+p{;DsLTI#Q~jyILFVAVU`)(O8?W zlvo>U5^@cbGGDCIs#@Cb?bqdh`bCS(*TY}m1x-|UK0UjGF73iItAHEbFWP~+wnB-kqaknXz!PRB=ZXNrCMT}K;x@HuijTJ?c9kLaC z=v)Mvs7clp`x|Zdm5_`J#8+!bY4hs)0#sHhTlSq$N&8;j?Q9VE%2o8}uz)&SSbY^~ z0zz8x5BD8aaVYDTF;=cvUT*Sb2njoOUpk7yuoH}T7510d6y;@>XbK~B29~5XQwP9l ziXA9;Mx2U&`w1qQ_tlP}Fh01xj+KUm#xaK?R}O@+V#m}d>ilq1*TSw&12->HCBi+% zXrB*pNwj?7Th6?gclo{hC#afRP%j6g@r=xYITrO8tUuVHV%`nyDp`IAK6;4Y0m`8l zR3v2NWghkS0~3s4r@P%~ap=aB9jl!i;`+Pwo~ao zxSo=MRj-qb-BB?n2q)ak?6X&Inw2Y7CP;jz4vD#~aAFo$EUI$x2#VcDp(g+1_)gBN z_QgX_mSz%>zDGsbq}qq8Ii71FcON0+@C=1+nuX$;zvpO2*8{rSWZ#KH#))?Ml?eO( zLTHhKBDHv^E~sS6CB~H`wH=2w$rjPgq@~$au2M{PWa08{;^i7ww{=Ao4S05~j7i}m z;(cZ>iR26T6gRLaE?LbSo>S~vI3njeoTBo=CV>RYT6yWSv(L#> zWy2o`*ts_!9+0NWk83E*=gT-R$WREC%LOGfbAt1Yw@&%*Ok?ri#e(cRNS)B^o*76s z89OO@%K;gVUpl3mY?q(0U^Q_(IZcMgmbfT3CM2Rlu23veMXLoXQZL#^1X%_nXPlTC zux@(N@`;s8@pQS!N=7hZm2NcMe9$g&-lrt%xt_~=uNOnUEDZIxPb#~68lefJ1Wx!`HCr-Wevg=i)PuX6^oNAG<($ay|ht6uJyzP>t7Oy?pS3*n4{$!v0 zk;+Et=Q!)kEazw=2?#}}xvD$;iOK3a zvdASg-I-DYWL!xtqH7xZ*o^EF^$YmfR?GeO{+*R5GNzo0Q13XC(lt97QykImfs;Tkjg%3f4` z^m*HdiSe?@VI;g#{bbBZNlh&>=e&MPuFV33H7phL)R^ECfFl=+v3gC?1;b!A-5)O~ z>s6C#CLOtU@TrZDquwfGa)tt|bDH0Rn5;=tYe8ZBL1^q+X{zrq#mJtBGSg4K!(T3h zA1YxuW0vZbV`Jyl-drReEeupaXmI4LrAV%@3cl0Ap-c&HMOR7q6-f^}FL!JVRG^Dg z)0m73KpSgB%AWOLWUWobNA5|y9Ie1u?vfjOE*#5QDCaoEiR=nugSPWrtlzIXLe8SS z1s?_eKJv%5`Q|$OWo4s$LqcwSf)Y|7&igN2x3-3&cAQV!*x7l7qCfVaZy!qNns|@u zfk4#*yU zNNFC-_0@w8%c5a zlj3Koj_jKWE9)OR8>Mo8{OjU*CKTV2cA*M|nK`-*y9PH9pt#fW%ZSqpc*dkAxuEb1 zo{%}pc`$>awbhW za;UYZ#xEg@ZtcETv#SA#8nz@p2XS+=LHElNT@fGmo)8k50YPmY_R0xAeZQrq5p<6* zFHP^GRFU2Y>g=E(x}7KTQV9V8Bulgs_-vQR0dyf*_-p>Sn5m-jjr~HzO0V|5!=Mbo z+ag&QnzNs!J*WTKiLLj-t-iGnbH--sgtA?&#xTLzLu)NZB5LqkCBCC-r5`?p#UF9l zw?xUj#h92%+&xkxn07bFIy34yP+L`f?9SD?46s8%&VFUP~yyS z_VI)Gd^jrh{GjkoCI8}&AEl_x`wa52jU9J;fGd|B{m_{X3hTKm?=~G02!WsRutUC4 zeEk^_;FuC3JS}$ov%=oagM`sLg9gJCkaiG?7)59UFAM@NElAOIMpE$ zWA*H~GIGV|;tt4AGye+L$~`4c=#5)J4+Iq>cH^?cSH4_D_IYRv9nY>8zgWu=p0f}6 z3@)D8Xcpd+95RPDkSoR4I1aQ>VK!lH88X3Y?~oiKdD3};c?`O@mcQ9 z5XFlYAK&)gDG8|UHiJ1VVRGUxJi5M0Z`+;&d!2oQl(ytE(-YV-T>-zLgdFpal zW!CcUIep4VcTF_ECV)eHt?B7P31ezDuBRU(b2%sXAneK1mpvUKMHGW>fwbAcFlow&6fjilR#ce-Ta5kV&cHjQ+O=(t!%IB-1 zo}U!CWtl*;*~SG0(?)zbcW69su>tAr)Tmyr(m-&hw<#>u&mS^8%c*GX*8SvF$=i!t zp9E*wQT5|5g;v$V1Ktszo{=PKe%wQ)E(Rlt>I>97DsSZMOHQ*0nPObf;ACF5BOS;Q zB+Rf|C=^_2RHaYunAr$WZ)XQC3*0?3QL5%0R6xntJZl!t zy`!T_wxMH>aWWVbzcxuInNfP=3y0O?$mnRCeQkPA37C(KXL4?d{oq*GbJ&ej+1W$S zl>Aei9R_kGbP=o7^(AKE1=dHlvoGi>2XL^571`jAdI-CfD6#pA51&+ZQ9w=|PQXyc zgngDHEgp{sW-3ljCVo+$>N~oyIKfB|D6)k2SzLJ*5Zay1U)~r1={;GVd2>tpE3(5;AF`=Ys}cxx6aH{gfZ8a98e3ANX@?_G?E@j+U7N^Fl4h z(}rBHYG!f30|MeMYqJfNNF*%Uh92Jmndm|3zhE+(}#q#FXm0|Ze zpRZB^vQdExREa>+a^$?T&Et4se6v`QLicLl=Xk016LODIf#BBi-i9;Jl>CTnXyZ05 zcc|O7!*wgew|D2Uj+9-3?%TMShd&%U)6#4@QM`2wVq$N#I;2!j_A$OOLlO&w$zjy4 z?FDi`qQ)l%W;^r4TgG2m5_9ejZJ{8uOzy3Y2PulJy^F_}ES6p(x>A?R+&g{R?OItk z3I~=7At30%jtCzZ4Iky>P0(OpT~ai*8RZqW@&m7*CJuQT0MZSgE!DI1$bIv%Ph zxrP^5n~?MABQ9M1UQ~FybVZ(e>XYB>HtZ~EIRZ+$PRS1wQCipDEm6p40pQWg#EtBj za)Ef^`;WJdQGa~@uRXAhIJ-sj4J-8f)l#ncfxqA3<_RUv@5iO>MX~%yo}V`VD66Dh z{69E$ZL&$o&MJ!i2Z6xR|L-0C{}Ii8;2ioz$*ixQ)U1#wgTbtw%gZYuS`iuF`{|aJ?<M}y(t#&ruCjF z;kT>`TvF@8duTn@CX^=5Wc>ar!nc#_H5bhW?_TBuE)X9bTQuz3kbPRDMp+7=maI_C z?eWi}*NZvlB`8*0x42O%{>sRqM2y6#a=11%BUM7UlJRP3_6sJ~bxGEB%IctyiJh#= z^jLb$rH$vlHo5+jt%lQrRw5pnwrTF9F4EmqrV`t0bT|G&KV}q}7*jhWU7+0;! z!I8UpDM2(v=gz10d-02d7W0ihLm8ZE^kAbHyJ)$MOMF#%#IPa5Z;<6CeVZyJLoKjQ zg=a6L@Fn|_u)79oa1i)Vy5NpjQd<^6$DwEWY}w;cf-$@aAPLT?czDuhW=6}xNEfxH zu;t;z;5sXjY#RzPT|pk=^$HEEOgH-=*@(DJGlu(4;Ta>>EsS0RNW@5!j;3e_`aoeu4{sqGrbwUO}^wa^Qy&5jl&l#v$`Ow--_^XANL0f zr*8se!ZZ{SGfsJK!)_Kvr*FzqPf_tKTQraS*pD+2qTL8R-yyY70>$?0Q!l*NA z?d@PU#AjB`zK7aFX&4tH_|lvks(48A7qz$s0@dR|_Tpq|x7>l57=d`N9$qv7k+-&N zfPJ3ja|i4}JuT+)iP#5oMMJ&o`>UMD(AkK?idLl*aNDPKwR<>M2M=USHj&PDxkrV4 zn9b_qo_^5rhzHxl?(pX$1i}23RWOz*s3=RpzaZaC|%*kEo=iK7Bv~GC(pHYGKhX&}>`oWXO@UBt=U_Zz* z>5b9CP?OKTKFd+O%M(PAvxoS8n5Bu=nR$`%K6G+kkp1MON)@ z;zj;IM)0nTUZQvNi0pgo;pmSk8*fG|3#zZ`TE?o7G8J639m|uOj8@oRmQJU! z1llJp7V0OT)tg}0#86Y+HI74M6&TEZ1%Rq*)a{oL2qFa z+D|c_5!YaHkPBvucjj9ZLM(nEJnq6kWRIllu%b+{2KmQ|lRf4c8ou*vP;P*U%d{Nj z7wyr?xRm4+;}2Zd4_4B*ZV}pW*6*aL%_bf$%Gu_O!oJ*<=(j06yZTRM;L62_N@+a% zy!O54c|8Nt-QTd18O>J~PrQ6wvA8)1Svb}2aK|w;Wq4wqI2-Dl`LVR5RV}g5XXFdP znLc4&8>iB5UZ5(5BgHIumsYik!WO2<#^dkye$zm1Q~Mew4=%Iwxu+WiPk^1v*w1R7-xozqV;-l>HzSw?7|cO-8^{t5DylL`w+^s98XBi@aFm+PRIf#6J-4+LGbdEwl+iiMx+jALCk+b7I=F5|jWbN#1-WOQ zZ0c1hKVSW!%A(a<5SDRQLT_c7WlQmfM|<_Hxn_W`$G#*Xde>Ql?EAMyKmqXOXA2#m z00$z9htJiazz15o^31Iy?&NYk(wOAl-|SC&^InkjcwRQXRsu&m%muSl%?Sf~1Ed~G z_;#PE9I}q0M4^m#T>eI+d@DaKi*WLIUO}4+CwX1m=lsF_Q%$n)f^{}jaHMgVb4Pt} zto*H;L=@`Cg4Yc+{}k^r1ymr+fLH*ervgu*w-rE5r;k(Xd{nbQx4!|7{tzsK?8bO> zqI;o$=1dP%Mg~laR<9{6DI3(pkGJOu@4(%kDuZkpoZE<{&y+do;exL04fxCT;``9( zs%8`4eEYFJ|0>eTmn%~qJ+(@lYS~GyK$R@kAjuOt33?85@2>2oLPVr>v%WM@!LH60 z${h1-EwZ3TEjIB$kc{G1NsqqKuiG1a1ve@YaCSa3xN<9!22f zt=)-$Zgkl2z;^M-0A-}Wh6y-s;3W5)DNP5-aaNny(ha@bUWq)S@D4t8E>=Y`Z}B$| z5eRecj^+#UEif{g0zI;SGO56g#1$n;()69JyY1g4yglZvxjVV;9&1?iZg0@1T=4e$Q|_rIJdsFe%ETj*~f7$BC$ zm%vIWUNS|gEXwJGI`zhJMh#y>b5YAu z+%GT66``7yv!+%KFK2dzs=C<5UdPPtgi_=3gBDsWvL6Lo#_hHs>NmZ}pm*;ypI6z}uhYHzGR4Pi_IHW5fPz80soGopV4mIc zg{0I|#nK)lg@%Vv(YxgMOG`5X96nZ4Q;Jt2`(61{oQm1B^Nk4wq2ax>#GR@I>cft9 z4#+`XJf5`z=*lE2Bgq0{c{K(uzTs;!@jj#dgD1sTz-`sv18s}@>$p$%Dz9Gzt*u}Z zoAk29S*ND}-(LunX-7MAVL-FA=GJ}~(RESIeYN#XiQ}dH2Z_mJpq)O13A$$1RQUm% z8=2D>*Q`vr-!oW?;Ihu#laU9QfL4HF(fgf^t^+GgS!5%)P+1ToxCwG2Vd%^P#Y*Wm z&P#c60d_5@Q)=DJ6KU@hLq>cmKd7WaO*$(!xx7`!nUfHLnkcs**5xLixWy^Wd|3l;q zq)^WW*NF#Edy2WjI{fFOp5L=!qrfjauz!CdGe_P)8UH`Q`~N3IW}E?vm@UH%!SFyV zS|Cm&ISzi}ExtgNRE-N~L$QQ>`~C8QR>YPJ3lnE*T;PKXDls)hyzGS_ay*cr9RDr| zApcoUPj2S|r5|baIG6&fgu47{c~9|F+8w=Ux7a_Hjq3B&ahABK3(B#>$DcNDySfq9 z0A_BaKyQvyHM2lG9=sI=L&m)N$#!4&c<+Yso&G&3$`A1ayIpx+{bHt6T5_>jGkm7> z-M@V5lQJjBAV*oBdtzu$lHHVM_zgAtYs5$I0Jj4)<=DfsB#ZD~#|U*TUK?FlJ~gI_ zgn1C18c>NB98pWM)g7=57sS`KqvLlo>h;j$ok%DAz+h9eHUcL2f(rOgQNAdys=!f*7s1xOx zqP>`Dh;jJ#4Q2`*r+Sb>v#^oj1LrtnV6|uTays?aO9fWL%rD*y07v0~Cc;iX@Resx zpN*~nKq8(Y>o}AVAJtT`ik5+omFKNDAl^M?riObtp@#+@k9{P315Pf^H~a7qfp?$U zL-`VpA2XJ^KUuuiE+@uv}JY1@X(3 z$h!S{L(We00PvAAsz_O;>}z(JT8+3p)CHn}Hp$k&dUO%O--={@efPu`r6cE4wq2YB zqNn~h09Ir;&D|VArgtdx7MV9|TEO?RLSOEWe~y!)7N_5e9{h4v3h%V?ZFiQ1@i{VL z?oFPN%#7KezvuxD)cP)=qugfJYW`tW~$?~tx=+U2yz3VzX z(581=-8=F^%C_|{u5;)7ekKM4v~{;X7odUJZ$P+($f3kqmVVl$N}I?dt@o^swcm?d z`3Cp_rj9?~3H>WbSnSsU0Dqo*W6CnbADl2)!(m|cBp~0?fIbp_V+Um8>1xDB=?OoD z@#FHhTbpZIN0MM^&p{Hl*{J0F5l#!OX05M--=DM-`tsIy`K{lV?-^4AzV3zXKc={N z>i)5t|9N^?X|uBJ3<@PWVZZqN-G z@M!xQuZR{`e(XE+K5TkT8@5``BKiboEUKiaQKXGiRdQb;Qlfyu5-Hi=D@RZEXZr4c zyZB||D^#;7BTuj)vqpeUHpXm7ilXX_Ap>$RZAQHzN{WbF(ey zq|w&(NbooOBp zLcZ_=hZmu%cW(WP>TXVS#Ymd>Y-5iZK5=(sVo&4J*9K-}I)7X1rwy>WK0LJ%LF%p$ zTRb(Tl4nggKKp@>EW4e|CJ8QlV|J2bvu=~fUhjOeV+K9q|kCl)B!}_)BV@+STW}7MZ z(X{^j&j)fs2PFCig#(q8ICtzGVq90#h_LD@ET9-)`|m4?(!vg_`&ROLDss+B*uy^hp zL0ao8V+qboEdE`Wclm+Q7@8CJ(p`3P)YPPokOEnQ?Qn01L;aFOV@|bXWpQs$)_-vV zIG`p>T;Y>x@9I;4gannw0KCTQDt4P#INa9T{=IX0HpoIpn*Ju_Kji$V*>t5)R2{g8 zOFt{Xka5ZKKnfz^g}wx~r%Rw6fLvbq zYRACgi}HufXQ9q9jB8*1!#K|qb4C{BlV3qP3kpBvnaC{3-*9KDAA`HCm-0oVEmMfR z=J(&cB*p0siOL>XYQpXH+0dL{ONX~2)ZfB zzo2nhI*RWm^mwm}18mC;Irq2DE2nJYb;l-oFf^^=Wm?I*QM1O?_ajVf_wVlF9h7R> z(rPs#poV%1dz5dB>xEWk9Yi3*-W={ylMI*Kdn2XjS(CQG{mCRP;H0ttI7yz`Eb`is zzw&0gX+qmap+>Kn_sMyF$io)H>>pIS?m}ZsQ!uqD68w+(PL^xzZ#dEVE<5Rp0++ie zN(CRK?@UJvs5ZPsHI_aYtlzZ`C4R)ugpC?H|0qg@_6iysfH-~nB2lI^oTA`;w%9bx z85=js*(HjM(I4iI}5YMKF z$qKyzhAlR}d3Ag~mnff742EzQ=8om#-zBL@+`o|Fy!rqL>+B$CAz9{*!s=*|A2>z& zpJDRAPACT+hUKjojmRr+&piZt(Ie7pH5u+Z)@Ert5F{P$ZmT^vXv&Nlp7nrY0{D#B zo2aqdW3t=*EQ;l?;D5vc{Yy$n?WT&otir+tX9B=L0CnSD89iqQL;;3}P0P86JL2Bh zq}E}t=yT*NH@tmXVpBI*{Ga+RYqQNZ*t;Z zbrM&so!iHb1&k8== z0ptb%yANG<67h(6IWfnX&tI+sLxC=0QSSe@ujLk0kYlnCnbqA+#iG&CWFlI})EjNAdeB@d&{@y01oY^sX` z=Q@)0Ak~;`zkm@iLGUz_*)C+tI{N0>rM?;Vd2vN?LqB{coEsv8l5h|0li)OtflS!9 zZwf=E67M1?eEmt-xQR2$uzYJTf9E4%z8ifVKoNEAyj(FuyKFAoiZF6i`_{1quBqzgawi3;)6t7iUm zHQIvtV9eGe*JQUwS%|Yk)U+R~)3W*`X%aFT5*8kLfK$k2LHNBw=*oaWD-eS5&HSn; zO!=ky*wN9C4QJQ0iro`$OAUK--;_+R+)mDEk1zrRwrWb3!93TV^n7XW2Ohsn+=Fc9 zPeiF`7fkmxdM$qOGct{3B%T2*JI_J6Ge)1AD%%pOOI_@_ijTGUl2c+%Cy`?8hKXV> zJOeeJ!c3pU#RI#!SE}ecZkEACvgjB(7pCSTxs%v*Y# zV24t5^?B^~r`<&6ouH;TFvHQ4-Ivf()${ z8h-tIP)iZAJ~RclMz`r&V+)5rG|cD~G|qn8?*)FxgHkEo1N(H{g6JfOQjsx)=?P{H zfTOzj%N?L+^`p)bx}@osw3;|#yJRl=P4j!fXv>ER%>DBI1|xv0l$dbk(W&}O<~7jL z&n0p#D_+Rz2DFD2+8Wf%Z~R3G8PM&>UtZqM3N_@3HyGc}s#Q%NZSzYQno3OfaWmWf z(vRj)_i$XI(J8_5LDzMC!|6$rzkZ80Z%bVutu?kj9`BhqNs`lvauU2O`!^XSMh1eT zl=*r2E|!!(dTv?MbD(iSi<`T(_2p;UG8$9G&L}gIeDhn-PZ5Hv%u5(dn~aHvapz5U zj+#G+#q&C-$#61B2X63f4@{f_3<5}m#?0_tqYoU3g}*oZaJ#Wk>)?P4uBS^U+I}i_ z_)^P0t7>`}ko;~+7E*+^ootT)(S+Q%uLq@;Sw%)xcqTZPOMi}I0d(pWpjxv)&(@PD zYF=Y0ApfM*&eGk&WLzKD(IfdaLM<@{&ORhogqgsE0Z`@dC;KLIp0jZ<8DUaAw*1+= zar*S{a^ACgfwO%)Z))Vm+XI|oTK{AFQT$54Ub zs|IP9R>|YnQ7YsdbnKt-*8wLF&1=y{ZV(~>jnNBTP=^y^Hw1Mv{OJdk=u0f-vcMhA^yhs{FZZnYo$Y_^h26Xsri294S{Af?Y8zpf>z#U)|u0tuoMLep27NOn@ z4YMnR9Kn! zChcU3hSJp}X`p__J(w%`mj#1SD$kUW{_e{SrlM;LdW{u9hjTw>g$Gi3`|Eq~rGU@R zGCp*U9+29&8l1i~C5f1yCIXvgYiib|rGUbBRgSJh2xD@;@7Xak&!l==` zc&l#A;Zn=&JRe@QkTg%(YRr>_k3&H6E{CZrntZgQ5wYRr@w-9zoe+L?6vm$IZ(<3< zOJiP`h_+gtkudL^i!>+iBgG7VKE1FxgnVmOzU$kxo6BZwUK=|)#VzE9f2xupN>Y)* zu<}&gOUW})HU~#p>la#iTu~=%HNy4^gvrorwVr5aVmkhAkspvkB0566Og#`YofUVU zk6e5@Rr3tG$&tL1roP-G7$^f;2}qe$^lXwAb8P&`CvDl5fliIo+jpkT@7>T8Pr^sQ z!PG*wdX1Z9U^{`Vg~5|>Rb{@|0F*Z_pAv1=pBg*vYL+!fX_WnjFRS6Mlt2u;`iHso z_g}|8k@1ZM;!6@X+B_3l)bo3M@dKacgua{p@?M?4lJ{~t_gHzjG|a``W9@;~V8Y;u zmAJnpCO5+yXlKQglRz$jH`6kuHT&BQpsk;8zTbU5U?~WMmyr%Rw^kV0NVmj(^CHM% zP6r}F>!rd2*Gum|L4I0&Vc_kkN5gcct4YZzTU%-v2uk}A0P!R-tM3Jh^Teb5DhpJ^ zwQJgT-u7lQH^XMUEMLG3_RIPurLskSl(_o0NHGcX4jPxsEds3O1$P-e&e);X)rQO~ zcI`Je29!)%EezvBXp6dx?!C^Q7|F&GxYgquNBF(_ZX7Cs`E5YzO^I&DrK(F!C@7Pq z?cM!wSjKoehCGn;o1FmT8dhy+_!pQqw&HFYO}F9C0!yn}2cBVG#nT3jS)%;`G{+`` z%YUPo*8`C^exvO(psbgJ`lpcT=b+YQfPcI3%Rd4L1ZY`Fz^3X3cb*zL|Mn0}+!p`z zOVjffAZ+BVeENgn{rQG)_-P#tk+b?-3e=NUe|v`IxmFISEYtP?X3swwO=_we0KD}k zD8Nsxvm_o3&o7qu7_kwpxw@b4jErQ_@w+Ozffcb>tg<#cB0b}Mau+iPCe4=ot;Un4 z^1L&np+MN)iTj!Yz#k9t4Q;WCc;~+Cq?k?d?i?>{r0sYD8nduoXcNndzsC%u=^l`0oi$V zDi8aDbTWF@0JA+lxeu8z|2H*LAq7uiJ1nGRK_?*Alb!uJb?xdrL|&=%)0>0ZUpt|i z#=mn6=0fcGG6}yI+om~(s-6BCO^Dv(94L(e*?irE3x76Ne)-eH`iDa1U8XiM0U?|c zZsCwP)7mrO>AyYk=NQ$`vfNFiaSbdgLqKPPQlSdBu*hud{J%5s=Tz0t5CpZZrHVxd z3%jl}KGvthMa-2UO(?$m8;aSO@6015%(iL-Mb+Di=a!Xj=#V+xFD1z z!qxrkgN@VB&bg5#t_lB61yCusY|9K?B0hJ(8bDU7%V*}~#(?=Ji?EC|=R!did;6o` zYrh?)-B|s+<<1@I%3lJx%i5%nQqDA0*I=H`*FS+Gm^miM>~R4Zf)@=U3GrSnD9IoF zjG-vd=5UW@+%{u&*b8Hk_Vi+>;0@`Fl;)TZJv!XzR1@{!C${AWrWM!wV*u`t4=5?8 zf6*DEMpBpw{MfHsg@hK}gh;_-pT!967|OW!_{!&%Ylut)z_9!81-B>YZcm zwv{drhtq|ByJZN>+EA><>eP~^sXI1d$h@Ov$NzlP>AbN>{-VAkShCX^zlJO+k!UV{ z)ut=cISjS*BNqaH;5;iyvNz-wH)JoczaFbsZWq_H`ceP8c-35Y8wXG$M9e~*p_@|1 z4Ph5K5Jk;;dGov*0{Tc0#l_;KaYE&TLczcXE66?LD&G+d)O;#xuRQ$3O4nV8wI(P| zq0UyeqEE(NSG&>IMf-?f_)#cve2?&K^YBat2AgVO^3-)P%=D67^!M)20tlE{wA8I% z{L-dIn1(j+kbTVK6lhUEyJWLkv1i8FC)pn&ZLs-{l49Qc$hF^<^aL2@=llHBXWLCj zU)&dtm>Vs+usHkG3le>oA0rF&PBtY{IU~)gVovjx$*2+Kp#F+PuV|y>&fkqR>pn`w z!?V3~h7?e!_Q%iG*%PoHF6TO@g{RD>)D!%{KFXn1=*ZAInK&Cu&OGwC>;Ei5@y(jl zIgnHLLSF>CF8(4QpILj&*4yRlUpws8dJcCITHKk@-z(e5FBp-Xrnfz*`UeZkpVq(5* z)Ps>h-1r#;%0DS1F9)Xg-S{{YR&E!1{_ou;-wDU113D4ggl*lpuU6Qax-+298$Nzv z0Q<_n*nM4|%F4o}_3U3S&0$xTFGY{*859rk?x~%Q!X1n{DrWuL_XYKcdyK&aLMx@H zesGo7NhY?(kj0ue}Ad7rdVctkB9Bk zwkvaHJY|#xb%{2~VE5qo_Dx|k@nQ4uv%)Fg;oUHC2N(4o%%CERXO-B7IdjEr>sX4x=RO#J#W>PraYY4KXbd4)|z z=^6pVdc6mZNgG@n4hv!h8}KM>QqXnDzGg+S{L*SwJ;g?BrFGQCcM`cc#BA&hh`frw z40%1KUh~_`S9AMVOTguS+h74u5HjyaU;mA->v8ibFf8z+f0tI%2i0z%HtdUC*PtWv zW^a2Qteqn)UX9%Z?EqwQ>{oZzcOv`F+AcA(e(4QmwW*yK!$FKk{}<}Qz;UK0)4502 zsk4uNg>IN`v-#hsOEVzsK!$>(n0954>;;ddSx8F%Lr?b}{nZ1#8CWXEwT|b4j++$b z;7S}UoCp0I8fQ1TpgpkthhXca!2~;%wj`tE^JNRIha3D>ql?PZ3!IN$YFQZKR+ZZX za_+XxzBeaQF90f_lbM6d@Rf178~rys+u;XhlNYk-kd*GMT*^DxIIG1FvT`c1>L`uX znE`^sj~;sphk-_>xbMk%cfj4u@sv%B1Eo|O0`Jw$Xd2iTU=F`a2misTgdDqG6$tb8 z6HWs5IDWL&F1U8jcXPLqzbY_kLYUIhETp0YWlhQjIBX_KJN67FauQgIYE&<6@-T}Zr@%XR`e{*t1v7s z@&8c{5435t7dK*YbKba<9K`4dub_AE@qap^DPVxKyIsY^+jM|2B1oyOCr2q0d2 zX-#GU!@riVET|-TWmPmP^txdNY}AUt@EF(kl89F7AiN(K^0zRmC~oTDWzHW3fK6-F zB;&o8GGt&)jK)&w(!*-4y#=HgGfTve+c->Ar;ep1OKt~s-pjz<2&cm-&=1&;{k~H1c-~=^N~p@cpnO@HKHyE5 zz0?0iyGfRG&2Ic_>0qCQ@I}rX%SmSm7NGS8q$@z{jrk@C8+9lpAuluL>#XACcaV)C zhzkqzyuEF59bGe z+oC9lYev}tYpvFS!EApZMfaVQBVEe?W;o-ohuTY0OQN=|_fUx4?*o?xDOuXHWX#X5 z0Uv!ic-l@9Uxc;dJ8-Nrr)+Ry-26d*@o!?ewjY+BhU3LHysb@NUyQiZgE8^mSI!X# z24=mg<(F0j&2iH<*vBV6JjQpW;}53G!N0rQHNsg)83vVRI?cq+(z0MhULF|H*~$nYIjJcQQl1<>u=vr?d8U@{Ob+QXGSU(( z3zzhWcjuL}3P8G5O1Cu(6eJ?K}>aRC(7j!&<&$Z0$O*F}V> zJ5X3QuMK;8Vsk?hEz?0Am(H&7*iN^;JpL(eQ^fn_&nfkH_L`@@IHcmj=8V1JX|Ov zTH5utedN^6#I7aOm4kqO0_G9*E19UPK(*O40QgB3^iF9+%FV(y_Rx07d+DFxaSse2 z@g=Y!V3;-QGX|LE$YJLPBs-!LVK(q_{KV+ET+i7IpSw6A3DuJTX9c;sW`=T#F)2Tl>)L03J$P1bLn|Dz@Mk#8W&-RFo( zV3xhA0@N_x(oMKghr$>1Eu7P0)h@CMJS;QzEvO9`%4>L)NQ-jLs<0SXnfcJ6&3T2b&vfeW2o+TJ{-Um}Ld1%*Pak0d}q?cI*RX z=*-;pWsQ}K2ZyG)E>WC-1sCy!5~_ZsO>Wyv#S*5|%X4;~6ASEps1jct+vHRozD2c| zw>*mc?o>suUygWFgcbhsGn!?Mpj3c9{TLC-840weTlwq~?P}-(sWB)Fb4x*dlH$9q z!RPKaj`_#gC`@5L-S^}9N^-ZCUAEm#;w#AU?z>01m0G)(w*@+Y5I&=^NN19q>(pWS zlea2yao`Cws*5B^egT4t>sfmkEjsrM&ok%gD}mLYJ0Z-M@DP+ZaJWlkjjWx})<>Hx zOARO9h{fUZYpnsh8YxGaPaS!F?N_w|7--G0?k|L)Y7^eu_}-}zY9j-$j(Co{;{RFhd?{wKH))LJJ? zA`n_G4m|EF<&bd9=jjEVWq2 zE3>wOzpw|%X%RO76KRG(<=sz7IRC6cD=4&)r_t2;U`5pO5@OU z0?cb9o)nApIh@{Q!?V)8^ETnE{vN-h%l+zo`oGst`RHRN5)G<2>46PzZy;y&um|?w z($!zc1(@Bmb>Q$nWW6P@3)504xp?ZcX840Z@a_8Bb`O-H&&+CeFfs=R=Q{1!^&L(R zW>cK}Qr-PK1H&;^lg{JNQ{}9}T(QZ{;o!yxCBRB&A9-nB`^wojQNJ}&>MBmKSVUd5 z$#UVo;$P=5+4*AJ)J>tZ^p_hcXAUD5UjyDwwcnqIyY(JVelLb2aHj+A<{LI>$V-Uz zMJroq$nQncT!*DsCFXs(ufEHRS%`#cWD5SWDWLpQBb6T^<9>C9dQdJrFrcQ})!*-= zf87HjYUzye+Mu&V2^n_0{%<~ynd7#w109cbKV8N#X`ZQ&m$QjjVCGmZT&2C7ccKAh ze1sO~ag57Nw;#4DGNMEe@O^pwz6|wz>2Fh?zLN$%P4~p^HQw29Q1%#wSS(O>KK&c9 z%KKi|Hhg)tCLZpkq3{YPDW94eyYF~;dFsZ9@1}yrOL#Qdz}z~SgUL}=DK}2z?t}7M zMaBWZSMB=H{UbHw zqZdNUTLIw6#B6))C#QW7p;jH4h;4#Tf1Rt-+svPp9Pk)~s{xiq`XCDhZy+$D?$EH* zj(-$Bavf;n1-wYEk#-mC*3XK8aZFP{*4&GRLXJymXZ@(P_yaUHt`GxnyXO!JkqBTYI)#+9M zHEfz7y^nvyB^jN`D3t~T>&H_D@oK+uA(F7P1;Kdcab>H#{1OQ2kIbI3@P4;VhVckP# zV9VFqXWmHn-lkic#6Nte`H!LIx}+$yW$j1&2ExdNVD?f z%Y*)hhVK5w{cSHICZjOtk;K(^T2S^rkTbIg|5V&mi6|5^j|lNbIK-?mdr=W zk3QYRBw*G$L=mv14`*~slYkXXU>#x7G2+IbV5NT)pKM98+0i{qMDAp#n<M-4_$k}?{q{h! z+I(Ojb<^X1=6IX66i4*0!OrnJ$JT10s)PE!*kGdkyH-Wyq0IWocjNw*zy$J zd#49v65k)U^0nnc-76{pFotB$m+od}6h9YlekR(5YMDGX0HW)+;yv(JUpM+eiwLvd z@xRgtR8<;c`n6*=Xx!ucXSA!k@YR)H2Kl3&=?=Wb%Pa#I1-$J~+&3WDvGWc4uj-R+ zbb(h0vr+S%ulIY;=gn4_{oB(4^xx|~c`wd;+KjnzvA&_9mwPWGPpxDV#4g;Sqamf; zg>>F929{v5MWI6RxlP9KlXCz#k6-m&-wwI1lgUp#|JV0)|2{5+WD))J14cS1QOTdD zj5%sk1~Ca;{1;9FEgC>$TOV#r1M=Z+56I(R=$)(H;O3M}abOR=k9(zPFUNzDev7_`Z5Y`=C|6sfWn6> zjM2So4OHw`kMWL{;B2rCO`Wp!mq2MVxxT{oqQH9{pUG>G5wTd43WZPUarpio&YlCe zSgvrcr?LN{bW^|##kgW&e4F&Mx#_1{$JStG*?EQu+(1>o$FkSOn#(iJv#m7JhjuSR zCKnx^2gwyo<9G=yr2(Ty|L7#X{J+}!@_4A%@9|0{3c0k98I=~5?b;cGEYX4_$(G8# z@57AkmhF~MQVfPr_9ezHCP~Ct$2!B<8G9IG=l6{6=T`TAzOUcw_j~>R{Qc);p7(Q} z=RD_}=bZDL#ojC7t|f|X`5?5tVOY4SBSxFAOnM5;2D}fJvQ)}(WPE1VvSfn1>_<6w z*UvGa$6+!ul|ITT~UJfA>?Zqz%(#)|66|t++?I#yMH*VrJ zU?|vBGIzNluaPy1WrhnGFEw|Jyu3hYybE38@sV5o9IO_vYcZ-d;tW&$uU8qb^ITcIqKQ`-;ufip?$(XqsNm>CWmv2S z@A3PSM@y^CpKEBl{{iF;hM~|2Ck!&HRW4?CN4R|nO+e*J{dRuF^D_{i21Q;UrDluNTRi;iJ8uA z$Vot8AbRfa8X|w|M&)H42q_Y;{;2q~Pit(GR$I*n3oomma#Rrcm}O|>r+)3WsmIqP zY-5QzA7)X`+sDLpS=YzkbV52oUb3#|*iY+jl%u9$6zZl?UdsCRGPo~DUj2Un5>1 zuj^5ADF4FW>_U&!^Cca}F8gbkAM^b;Bnxi^>^%9MW;OpbF|-2TLlR zLW!*lBq_xyCh>fl&udRrl#XH9G z>V=vk1>kyicshLgNe07PMxY!5)c}}cq6c5+FrhrVTa-bDT4!Lf@IJ8DGRr&Ck5ps= z`qeuBq4jfPJxla`J;Mkh%e(>^Cx(5AqrGF+^JaP>Fp)O z$*fi1*z|Oaa;PCY5hf!uRtslv~K(?&)9qjtGWM)&R5DmK7pVtrp(AG`+G-DxXuh@HDW0*zcnc zT;U%3XT`&3O3YrLj9@*f-h}*m^>Nz$UAx)ueq`LseieH2_RYs9)6TGS-l)y0b&7W~ zStfT)$xCf;xUY7~!|{klfx-qV&8>Mnc-hvnEE8pc$goJQBQ@PZs!(pxnqN_>x!EagVf%(vwNiwv1)6iM3P0ZX$*W zA7sv+?YdvQyaKhLbw25UjuR@3@oib@qHV<<*B5zt?<5HMo7H!)-UM}s7DD8xci)?t zpZj)T?%>|4+GnNL<9q^DDB67$*)g&YRD75eqTbF=fK)#9i^Salh+T=0GhKhy_~X-R z8DR{`cDB$vZ^r^%G z{BWd4*KM@R_}zv1oD=4$-=EaiE{x zY6aK^5Koq#lauN_6i?{qI&NN`4~n~rR;;;kPAocIU&LW^1|n?Vlk)u3u+$|kbS+xG zsIWA~G~*f1UA@9iwUy)QuZ?DQ)qwNFB^`j<=*%Jxqi%OO-g7)Za=^6Z%KkOhUC zBA(K*t7ygeRbnYn+d@&5*YbYkOuv@9ST)o+HMy=?t-g_uA!7%aVdvE4#gRc7bZu*7 zw1=cF&gHry!Ai5Ty+>w$#92L<;`~W!U3g1!fO2?ZGYdi~?0vjWIc#a5>lYm6aK+Zf z`f~8P?7y-*kbN5^y6nceN_o{}F<+2z;X>*Da{IC($xGo)&2mjAgb^Z!@lL7r>NVY+ zW29=MszgsbbXA-^0Yehp`hOPP%hPb=u{N|B}fN;JFuR`(dWg|9% zK<%QU%ZLplKTv~KRk=|V8-dHwgP9LOd2#7LB|xQF$~tk+`Lguat%UjFZvTTx3`%Tc zCL{&6dyWVTSX+`a@NkG~yvc0~P5aE4hzQ5`det4r;rw`w9Sr9Y4*H`YA0~B&s z!g&AJ$Xv)9>BFdzWBL^BsE>SyJKotc!85$3sRH2CY+K&H&|&lal_d>6Y~HOOf*LlF7oltQtV#i2bhJf zzWxC&&vfp=iTmwt>BRQ~ZG|uz4}x}NPokQ8gK+sNwIcKJF&X!pHL0$^=1Qld7ck{> zqXxD{(V6u~8HVJKg$%}xNf$~1!Bkfm#B6(123wPhkmRbIPO9fnV~<5^9P1dt$2iW^ zjI`{{h}~33Z8(Wu07uCPj`H+D_+zZ(Cz9j4oF$Mw-|MFL(hK0%Zp({+L8T4-%7QsZ z=I|11G^^o5&-QF@x=D$7-rt(wUl=BhBI>VIR^{Z7^M~rZu}IbPi=8Q~nwxf+M~lQ# zi#%My9*xv>vEJnQE`HK7zs)m0Y8g&BI?5(}O0BzHuU2O1Ups7u8d@2+JB%KBHZpim z5T8(6E>oq_jfP+v^Pd(W8Os~CWC=nRD6{M1+^O|-1;5+d{rQ{x{s=x3n4&Y_1>yV_ zQ{z$1^L5|el?@I9eAa3}f)Q?jFKX8vq>^U zElE{eM8ya}?zX80u6PniD%H7PB+*wS4< zD6k=si5m;PQNd0q>n-7=7>`SClPj9?}q4NRfF*YD`FfcS`&0>d-#UOqWU&t+}ZfKf!)W8ZnhFnzYMY zM$hSLmNyWRkARRw9fYe4ei5^kBwZ^1)1gam0OZt5g{Je&25@R~(R%NX&%bQ?mHuyN z7)nWLRlvKrIQQk?$Tj%b-Y-mNT~-x%J*4<*SrK8UuqH89wD>qXd1N+LM>wi4Mrct0 zz`UpwJ_PbROe%t)&V1C8^oQFj=i~m2=-gH(pm=|kCT2E0>F+Ac>^*OCt|VBNqc3kd z0C7?Y>+KLgD}0m*!V~4HcoZb+Ka?gW0|=Jmuh3YSEUfl2$g|}HSweI%DNeoX^e~9y zhKhafYxbr^^@|Bj#<4BQs#8T4gd)90va(v^^uFtu&80x$;vLSWU$;6W2YY;k&yl7# z>T5IyYhT!%*}cE5#*xWi?R--bTzSyiGMz?ncy|u_Z#hv7Z`rdOo7GAlU%v%nBK`*3 z@;G_$W|BN*Qh0DYLPZ`!j|OddpI5rDq%}tLz{Ro&d0SMjP4JHO6voE6EIs-ed~oFq zK1nZC*8embmL6J4k%6e*Gw!JRP*@o5%9GwL-L$drz(mY1qW?D6YIt~AtmF)U{@V}e zA^=oL9>wwCV$A=>eO>!k`v3td#|B{$E8I#Yx2hYFmn&~qcmT=z*XowTwxsqvJ_}9y+E{p zx8$mpWc~GZj7ZYCGUkxZk>e!=xCzm+kO}$9-a|;#8+4IZoc`LpQR+taGb5>Xq$XX- z@KFx$m@1BL#;wC${YEEQ#zJ_voa#FmD|?mxBW`c>1xJyVBtQ9|MtZoO$NE~s&OfWO z9QxKdwmi%xQ~Tn4ahS+9ZN6F)6B9@5?2(fhzdU}I-!}kS0}_=Kl)`^I1dl1Ox3Pf4 z3?!08o=AXq$kXT1%=x9tdd5fj>VKeoQw9Nni^kT)ud)k8 zHKed1q8pFVNC^*Kb!p0w_8_Ocoryh&23dC`&m^Y@gt*)Nb}&K_(gDaPzp2x;9DhB^ zL@JX^j>%ig7VM-6Yew%YFOG3GERsI=VIc#nf$(wtl;u+k@65TL8(q=03>$qnG{ey( zsqbZ$*@N6FjBB>v9OaNHZiwM1cb)Ybd`avYbe^P1pU91?FRe%caO2icaBHtP-QZ$r z)4;U>F`=g*1&JvL`iL%v^0&+_zHu<%@%}IEkF)RPsJ;V#xHW}wz<3jBm&C3&H#zSK z^ImJ58|LA=>MaXmVH5Y=$o1WH9ox!6UdKl%Y)wQqm`~M~H(pQJ=-4`xxiQ3GY50e1 zdm^iQnlfbaZFy4)8(Z|T@ENx(SwG^9{=C{7#Mzj#aShAZtQ88wxODztP+%~t?5O!z zE4E1zy)@~7o!#thwy%CgXe#>ng|LpXr%#qa=s`XhoL5ddHoU~k%zaEca=qNTdiq=YoMgL7cCRk0Y6 z`ndi{8MF(x#Zpqh`4=cTbB3iKCFG$?!}u4xZ5Nt`%dpgVj`bK zLqjgQ$Q%+^#t+plVvuKx7)FpTzwt?RS-&^+)U-+x{D-d|@p1gJORobgTEyO>a2d5Z z20{CHLnDv=jq0Z58{t=l!x59`0=34&#Hc0Xl_@*2QLA#LHBQ>~>%AiutMw9y{*4+f zV#AOPX*IK7+TBstBqV!sOD)0IT>isGV4>B-wejPBnt(NH!Agq1ZWe6?HscDHa2xYl zV(wpf%ID@&^+;uw!p2ew{rroo#eT&h^FiT{R#bafSLD>gNcN09nh7|YhOh*=YV!Ev zNtq-&MsMgDwNu0@(s_k~O}J;SmbY7v!}#JG>dFq0UZ!X}H4^*hz#%?E4Qi)4+*)y) zmEI9z-3k^g+>%Ah{j!KJapm)8Rf=DgITC3Z$i+TF$5!-<Odh%a}>F6B12ZzaLaM z_f%_V-JX*@?ql^`3g(l%FxS>L`;OLQ&^`xpeyBaiS`w!(>z^i(Gk-MukCgNw4!8IP z<@#-_6;=r)Fn(n>hYf%wGq1RD zdM6*;B%5s($?k+E;&NS`2nQh~Tr@mA5izd2LZkcZGyB;rY zvhgiAy>}~PGr2p--Ee$fo5u#l%OFwcKkc%XpS*ow;`;O1DV$U@ZsnE*ZtG<-?YS{y z&8hDv&zS^~Ye~UCC_;vL8N~ZhVEKRML%t!O^8{KkMmcT>dM>x`gN2$ePEfCIHM-LAF^7DBp!A)=_RP>qS1(vCUUl@7`^!(MuC-h!GXn2vOqje)?}!bl+7Ew5 zeZw{*7r9yHH#Q{Ux)f_NGed9kvvkww)12wYREYP=R|!Ks6CAs9mTEY_@|P_~i5pYAK?5 zgWpa?c^Qz$+m~D{m9dmfbkn>G^8K8@!)5^B$eP0_>UzWd^;*lthH{Kg=~i^$Xl6fU zHg-mt;UtEcD<@>=#hec)O(bGAau@4MKzLous<({|OWj`BXD(#CO}6t|a!(a{s>V5^ z2-fnttjHhjdW`f)JIVn*z3D`$8o!*c(#jgp>~EHC3p>2;@Ju-hH-NT>c!(Vow*Yo3w_8f9r^f z>zKNTF?}Pg(m_YwYmznNnq&6#--W19fe6v1`I&@sF zw`d>aWwU?V@b@e1Ng%W`V?w-?ME_ApGejN8cI~xSoeW)LgClK{gxJ z*>9DKE8*yz77?$iHp5PnV(VBqCeOGw$7105AT7M7xArw`^e`@lT&t)tqsU4I65>OZGmb5&sa%6z( zGjCM$H5!i=kj0jiJd0rkl7c8m+0d+yWZM3n5U2U7b&sw@N0)n_mhVP7Wbeem#=+$# zrwjR~bqVXoBh%9%l*%#6q)+F%4{gDnusFDdC?B()x~}1bH@Z}kEUofgo+%ah+Y#jv zL*ibszi?bQys|C6BhkL>^USl-gENSqR(%|`A|!s%AVFEr6y%GNe6nMaj2H5!d*x05 z&%nzw@)%fCOL7i1@azXcUgMfX$v!!2^hCt_M*dHzu92TE_h4g7O3AhtfEgj_$sMkPOH6zYC zj}4+LZs>1|me*NlLbEl13Dy|R0$wRY7BLGF6S%b8#khzssjlaL|K8KgM1DA_L=}p< z5hqHb8*pS_i$;s{)E3DZ9rV18h0M~yCoRM0Fcs{qe3Sm4Ky_$Mnm%;LYNKlmZ_#u) z`637!o@*4#B=tJ`?#t5beU8&fQZlp&#L9PXN8M62Fr==PJ-i;0lN~qy0%-EUD@ec5 zW9S;oQIL|b7wScpzBhwswP%ZB+H0Q7EPnv5G;3#d&hEr$0EaeR_4Q$88l_wEKY>Ol)}R zDnjr%a4!&_CpS=p#&XV0u*&A|xDV({{f;n@??ik2u7`x}n!LG^Xdg2hVJx}sQZzW+ z&kBmSKm8R=M^q_^mx|1y&QcZg7Jt#QitCiv^ot&9giQ#((kB>ZbZ*tflapk7Y7g*k z18?J-o8EK}BY^sc_##kDvj-B@I+Rj$ZQa%FkBT=>8o<95sQS4)b??DRd8+*|=Q$Y!ZHL0m5UPT-C z`ZZg@@5{Yi?LD9p;vZ{t0Zk{ky#&L@hxwkv-OuuK9HSdO^_QPbK`k9tmji$=p3h3$D^i{2HcrpmPi#VN+d)q}?%e6`P9>-QrnXfYu7 z&LnO|0y&xfYAjq~d(bXdt68qNej4_z%Z7AEelrWXIdCe~P9w=zP+?=`We?T^hlK#- zF4Y2smwpsqStarvrw)ZrH?##c1cvMIf8V7sJ~4Ppd26}fSE}D<+aEG;aiwLK`$FV| zW>1NPE%_p?CHaFP+d7XXM&6Va>dj}`!lxXU|423vdMSy8+55vf*?Fb;&q5|z@^;6U)oMJ#W!DfBE{ z*58CKt@oDXO2o*jg!~ zXvyIk<2ij9Gq!D1U)X`onPmn1yM_d!i*Q!|{bir6nkd?#7g%0jGvDoDEtp6Lf+g|@ zb6R{8ck2x3lZM{Ua)Ly{ZLg-5x=p-1;l3?m;vu^VH~9f+N~Y-n^xV*`oFli)s3V+Tp$T}?OqoC~5@ zeHws+lUvTl;OW+SPSx1Z+3Pd6Iaune%~ca?US%fqh;fU*(Une|;pS(wbo#TYx21>s zG9?i!Utnx|zxz$HYbl01ktDmW>?Q#5m@Hr}H`cS$#*p+0re@m~W1i+-!h8;qEM07S z6N8WXb|@O1(}=NI9$m+n1w;EcW(>=PJ+sSvM(ZslMYQj7%7x9z9xs)XHhareJ_ixD z4<1pdZUSlo>RgrlKHXacTkSgcjTI)?D^(otrjUu3eQsYA1cgV#a0ao5`*>yT0^pq) zx>BhhMJhl)l9Rsl`*Y{%o)sASXFv`HBIV)cabbx-sDZ*|!h!ta3zQPiiHM%}4_-p- z_LC&;&vaxKSM1F!vc<4ygtz()I|ASz^_L{f&%ANx6+fb4{#aGXQHm;kr-vM(MPZL8 z-y+PNfl*r`4{3~k_02ZO90@!o)&2gAG>^IC`aDph|mE8u2|xc2^`Mn3Ov> z1}n4elRd8)pD|zf!DG4vE~adYPSfCjUv~s`BQyGeUo-PizMr0rf6UlhG3&>}--TB5 zU;YBN_D8@3KTYN5%*G^Ey*oP49=T^9Gk2CP`z;nSc6kD~!rg5%!)D*L1}*EIDV*vO z^Cy-CKn^O1!?ad?&5&|)Q6P|j%{)rM>MHh4OQQ>qAWHKX5DZ+JJsGF9kif})qarTyU zlIi#DHT=9ip|KWaasIAdIt({A<@^Sdj6dIA7gz5>v_=LF*zhr{KEA-itXyF7=#LZy z%8j6b?!gviojV0^a1K7kUzKYQ#`~v)T9t1RcK9OrjuK}$b+H;g92mAc&l}j{jBu@HtS1Lex@39A#?h> zv+pmy0~as*fur$<2q&ewImkI;Jl>OC zQWe561`!p4(d49M4a>&?Q|(j(mkQb&&G$|Yis8UgB`Ut}r1ym#8vmKS&!464vf=xX zLf2`oMSXWMfI52LUn0cY-Uawy>HbaC{vdb$(N#rUi0zSEcIUrP)(2!PTR9%yh&azUT- zxbYbI+%gPv;fDEbNh4mczg-bc9&{fYeIRi4NQQFvTZXGrv}4-yPW1PE7PoUNCZIpv zzGBWKPapmEs{M{vFHIE~3cO(E-g5?RZ_^7DpR{=0iw(=p*PZzcKT02YM%x}ZBu-P) z%qgd~M-x78nt$(uJ@li))Q`4O&wuoidV8xP7|QqMh9r2mED8#bADIpZEv4u)?s~@u z-rznCu&036TQT2yfpyoOrxKDBX*a5!1TZst%yp%_KPdgDX_cGGQB**6=#9F(<@az> zb^Lr@7t7^6J5P82nQo!Q^h@vA&d(RH8mT$(I2SOgU*G;{bCv$1x7Gi!A&{a1G%k$KYMQ`bJK+zgQwn!~1+(}3{UP4W(nS7Z zrfkLuvfhEerjHB*zt`&J6z=}=`0OR{gw zdNnvBT{ZXW^@IQEsNKe{PdQb^83plTjrEG{ottK}N~l+nDOCM2OAyo`+O7G7uBD@$ zFVe4RK$%pa1t>!gt@Kx$?Kf^J+^9cOvZuQZ-H!&R **⚠️ Experimental feature** +> +> This feature was added in v1.0.4 and is still considered experimental. +> If you notice any strange behavior, please raise a ticket in the repository issues. + +Metrics can be used to track how Watchtower behaves over time. + +To use this feature, you have to set an [API token](arguments.md#http-api-token) and [enable the metrics API](arguments.md#http-api-metrics), +as well as creating a port mapping for your container for port `8080`. + +## Available Metrics + +| Name | Type | Description | +| ------------------------------- | ------- | --------------------------------------------------------------------------- | +| `watchtower_containers_scanned` | Gauge | Number of containers scanned for changes by watchtower during the last scan | +| `watchtower_containers_updated` | Gauge | Number of containers updated by watchtower during the last scan | +| `watchtower_containers_failed` | Gauge | Number of containers where update failed during the last scan | +| `watchtower_scans_total` | Counter | Number of scans since the watchtower started | +| `watchtower_scans_skipped` | Counter | Number of skipped scans since watchtower started | + +## Demo + +The repository contains a demo with prometheus and grafana, available through `docker-compose.yml`. This demo +is preconfigured with a dashboard, which will look something like this: + +![grafana metrics](assets/grafana-dashboard.png) \ No newline at end of file diff --git a/docs/notifications.md b/docs/notifications.md index afa23bd..57603cb 100644 --- a/docs/notifications.md +++ b/docs/notifications.md @@ -1,4 +1,3 @@ - # Notifications Watchtower can send notifications when containers are updated. Notifications are sent via hooks in the logging system, [logrus](http://github.com/sirupsen/logrus). @@ -12,14 +11,14 @@ The types of notifications to send are set by passing a comma-separated list of > There is currently a [bug](https://github.com/spf13/viper/issues/380) in Viper, which prevents comma-separated slices to be used when using the environment variable. A workaround is available where we instead put quotes around the environment variable value and replace the commas with spaces, as `WATCHTOWER_NOTIFICATIONS="slack msteams"` -> If you're a `docker-compose` user, make sure to specify environment variables' values in your `.yml` file without double quotes (`"`). +> If you're a `docker-compose` user, make sure to specify environment variables' values in your `.yml` file without double quotes (`"`). > > This prevents unexpected errors when watchtower starts. ## Settings - `--notifications-level` (env. `WATCHTOWER_NOTIFICATIONS_LEVEL`): Controls the log level which is used for the notifications. If omitted, the default log level is `info`. Possible values are: `panic`, `fatal`, `error`, `warn`, `info`, `debug` or `trace`. -- Watchtower will post a notification every time it is started. This behavior [can be changed](https://containrrr.github.io/watchtower/arguments/#without_sending_a_startup_message) with an argument. +- Watchtower will post a notification every time it is started. This behavior [can be changed](https://containrrr.github.io/watchtower/arguments/#without_sending_a_startup_message) with an argument. ## Available services @@ -47,7 +46,7 @@ docker run -d \ -e WATCHTOWER_NOTIFICATION_EMAIL_FROM=fromaddress@gmail.com \ -e WATCHTOWER_NOTIFICATION_EMAIL_TO=toaddress@gmail.com \ -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER=smtp.gmail.com \ - -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT=587 \ + -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT=587 \ -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_USER=fromaddress@gmail.com \ -e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PASSWORD=app_password \ -e WATCHTOWER_NOTIFICATION_EMAIL_DELAY=2 \ @@ -56,19 +55,19 @@ docker run -d \ The previous example assumes, that you already have an SMTP server up and running you can connect to. If you don't or you want to bring up watchtower with your own simple SMTP relay the following `docker-compose.yml` might be a good start for you. -The following example assumes, that your domain is called `your-domain.com` and that you are going to use a certificate valid for `smtp.your-domain.com`. This hostname has to be used as `WATCHTOWER_NOTIFICATION_EMAIL_SERVER` otherwise the TLS connection is going to fail with `Failed to send notification email` or `connect: connection refused`. We also have to add a network for this setup in order to add an alias to it. If you also want to enable DKIM or other features on the SMTP server, you will find more information at [freinet/postfix-relay](https://hub.docker.com/r/freinet/postfix-relay). +The following example assumes, that your domain is called `your-domain.com` and that you are going to use a certificate valid for `smtp.your-domain.com`. This hostname has to be used as `WATCHTOWER_NOTIFICATION_EMAIL_SERVER` otherwise the TLS connection is going to fail with `Failed to send notification email` or `connect: connection refused`. We also have to add a network for this setup in order to add an alias to it. If you also want to enable DKIM or other features on the SMTP server, you will find more information at [freinet/postfix-relay](https://hub.docker.com/r/freinet/postfix-relay). Example including an SMTP relay: ```yaml --- -version: "3.8" +version: '3.8' services: watchtower: image: containrrr/watchtower:latest container_name: watchtower environment: - WATCHTOWER_MONITOR_ONLY: "true" + WATCHTOWER_MONITOR_ONLY: 'true' WATCHTOWER_NOTIFICATIONS: email WATCHTOWER_NOTIFICATION_EMAIL_FROM: from-address@your-domain.com WATCHTOWER_NOTIFICATION_EMAIL_TO: to-address@your-domain.com @@ -90,9 +89,9 @@ services: - 25 environment: MAILNAME: somename.your-domain.com - TLS_KEY: "/etc/ssl/domains/your-domain.com/your-domain.com.key" - TLS_CRT: "/etc/ssl/domains/your-domain.com/your-domain.com.crt" - TLS_CA: "/etc/ssl/domains/your-domain.com/intermediate.crt" + TLS_KEY: '/etc/ssl/domains/your-domain.com/your-domain.com.key' + TLS_CRT: '/etc/ssl/domains/your-domain.com/your-domain.com.crt' + TLS_CA: '/etc/ssl/domains/your-domain.com/intermediate.crt' volumes: - /etc/ssl/domains/your-domain.com/:/etc/ssl/domains/your-domain.com/:ro networks: @@ -172,7 +171,7 @@ docker run -d \ `-e WATCHTOWER_NOTIFICATION_GOTIFY_TOKEN` or `--notification-gotify-token` can also reference a file, in which case the contents of the file are used. -If you want to disable TLS verification for the Gotify instance, you can use either `-e WATCHTOWER_NOTIFICATION_GOTIFY_TLS_SKIP_VERIFY=true` or `--notification-gotify-tls-skip-verify`. +If you want to disable TLS verification for the Gotify instance, you can use either `-e WATCHTOWER_NOTIFICATION_GOTIFY_TLS_SKIP_VERIFY=true` or `--notification-gotify-tls-skip-verify`. ### [containrrr/shoutrrr](https://github.com/containrrr/shoutrrr) diff --git a/go.mod b/go.mod index 0e37602..02d1253 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect github.com/pkg/errors v0.8.1 // indirect + github.com/prometheus/client_golang v0.9.3 github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 github.com/sirupsen/logrus v1.4.1 github.com/spf13/cobra v0.0.3 diff --git a/grafana/dashboards/dashboard.json b/grafana/dashboards/dashboard.json new file mode 100644 index 0000000..998485b --- /dev/null +++ b/grafana/dashboards/dashboard.json @@ -0,0 +1,293 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "expr": "watchtower_scans_total", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Scans", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "watchtower_containers_scanned{instance=\"watchtower:8080\", job=\"watchtower\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Scanned" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "watchtower_containers_failed{instance=\"watchtower:8080\", job=\"watchtower\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Faled" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "watchtower_containers_updated{instance=\"watchtower:8080\", job=\"watchtower\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Updated" + } + ] + } + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 1, + "y": 0 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "watchtower_containers_scanned", + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "watchtower_containers_failed", + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "watchtower_containers_updated", + "interval": "", + "legendFormat": "", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container Updates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 0, + "y": 4 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.3.6", + "targets": [ + { + "expr": "watchtower_scans_skipped", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Skipped Scans", + "type": "stat" + } + ], + "refresh": false, + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Watchtower", + "uid": "d7bdoT-Gz", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/dashboard.yml b/grafana/dashboards/dashboard.yml new file mode 100644 index 0000000..9f7232c --- /dev/null +++ b/grafana/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'Prometheus' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards \ No newline at end of file diff --git a/grafana/datasources/datasource.yml b/grafana/datasources/datasource.yml new file mode 100644 index 0000000..8049912 --- /dev/null +++ b/grafana/datasources/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true \ No newline at end of file diff --git a/internal/actions/actions_suite_test.go b/internal/actions/actions_suite_test.go index 7cbd71b..ffa6e2a 100644 --- a/internal/actions/actions_suite_test.go +++ b/internal/actions/actions_suite_test.go @@ -1,10 +1,11 @@ package actions_test import ( - "github.com/containrrr/watchtower/internal/actions" "testing" "time" + "github.com/containrrr/watchtower/internal/actions" + "github.com/containrrr/watchtower/pkg/container" "github.com/containrrr/watchtower/pkg/container/mocks" diff --git a/internal/actions/update.go b/internal/actions/update.go index e37e671..9320d6a 100644 --- a/internal/actions/update.go +++ b/internal/actions/update.go @@ -5,6 +5,7 @@ import ( "github.com/containrrr/watchtower/internal/util" "github.com/containrrr/watchtower/pkg/container" "github.com/containrrr/watchtower/pkg/lifecycle" + metrics2 "github.com/containrrr/watchtower/pkg/metrics" "github.com/containrrr/watchtower/pkg/sorter" "github.com/containrrr/watchtower/pkg/types" log "github.com/sirupsen/logrus" @@ -14,8 +15,10 @@ import ( // used to start those containers have been updated. If a change is detected in // any of the images, the associated containers are stopped and restarted with // the new image. -func Update(client container.Client, params types.UpdateParams) error { +func Update(client container.Client, params types.UpdateParams) (*metrics2.Metric, error) { log.Debug("Checking containers for updated images") + metric := &metrics2.Metric{} + staleCount := 0 if params.LifecycleHooks { lifecycle.ExecutePreChecks(client, params) @@ -23,9 +26,11 @@ func Update(client container.Client, params types.UpdateParams) error { containers, err := client.ListContainers(params.Filter) if err != nil { - return err + return nil, err } + staleCheckFailed := 0 + for i, targetContainer := range containers { stale, err := client.IsContainerStale(targetContainer) if stale && !params.NoRestart && !params.MonitorOnly && !targetContainer.IsMonitorOnly() && !targetContainer.HasImageInfo() { @@ -34,13 +39,20 @@ func Update(client container.Client, params types.UpdateParams) error { if err != nil { log.Infof("Unable to update container %q: %v. Proceeding to next.", containers[i].Name(), err) stale = false + staleCheckFailed++ + metric.Failed++ } containers[i].Stale = stale + + if stale { + staleCount++ + } } containers, err = sorter.SortByDependencies(containers) + metric.Scanned = len(containers) if err != nil { - return err + return nil, err } checkDependencies(containers) @@ -55,24 +67,32 @@ func Update(client container.Client, params types.UpdateParams) error { } if params.RollingRestart { - performRollingRestart(containersToUpdate, client, params) + metric.Failed += performRollingRestart(containersToUpdate, client, params) } else { - stopContainersInReversedOrder(containersToUpdate, client, params) - restartContainersInSortedOrder(containersToUpdate, client, params) + metric.Failed += stopContainersInReversedOrder(containersToUpdate, client, params) + metric.Failed += restartContainersInSortedOrder(containersToUpdate, client, params) } + + metric.Updated = staleCount - (metric.Failed - staleCheckFailed) + if params.LifecycleHooks { lifecycle.ExecutePostChecks(client, params) } - return nil + return metric, nil } -func performRollingRestart(containers []container.Container, client container.Client, params types.UpdateParams) { +func performRollingRestart(containers []container.Container, client container.Client, params types.UpdateParams) int { cleanupImageIDs := make(map[string]bool) + failed := 0 for i := len(containers) - 1; i >= 0; i-- { if containers[i].Stale { - stopStaleContainer(containers[i], client, params) - restartStaleContainer(containers[i], client, params) + if err := stopStaleContainer(containers[i], client, params); err != nil { + failed++ + } + if err := restartStaleContainer(containers[i], client, params); err != nil { + failed++ + } cleanupImageIDs[containers[i].ImageID()] = true } } @@ -80,50 +100,63 @@ func performRollingRestart(containers []container.Container, client container.Cl if params.Cleanup { cleanupImages(client, cleanupImageIDs) } + return failed } -func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) { +func stopContainersInReversedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int { + failed := 0 for i := len(containers) - 1; i >= 0; i-- { - stopStaleContainer(containers[i], client, params) + if err := stopStaleContainer(containers[i], client, params); err != nil { + failed++ + } } + return failed } -func stopStaleContainer(container container.Container, client container.Client, params types.UpdateParams) { +func stopStaleContainer(container container.Container, client container.Client, params types.UpdateParams) error { if container.IsWatchtower() { log.Debugf("This is the watchtower container %s", container.Name()) - return + return nil } if !container.Stale { - return + return nil } if params.LifecycleHooks { if err := lifecycle.ExecutePreUpdateCommand(client, container); err != nil { log.Error(err) log.Info("Skipping container as the pre-update command failed") - return + return err } } if err := client.StopContainer(container, params.Timeout); err != nil { log.Error(err) + return err } + return nil } -func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams) { +func restartContainersInSortedOrder(containers []container.Container, client container.Client, params types.UpdateParams) int { imageIDs := make(map[string]bool) - for _, staleContainer := range containers { - if !staleContainer.Stale { + failed := 0 + + for _, c := range containers { + if !c.Stale { continue } - restartStaleContainer(staleContainer, client, params) - imageIDs[staleContainer.ImageID()] = true + if err := restartStaleContainer(c, client, params); err != nil { + failed++ + } + imageIDs[c.ImageID()] = true } if params.Cleanup { cleanupImages(client, imageIDs) } + + return failed } func cleanupImages(client container.Client, imageIDs map[string]bool) { @@ -134,7 +167,7 @@ func cleanupImages(client container.Client, imageIDs map[string]bool) { } } -func restartStaleContainer(container container.Container, client container.Client, params types.UpdateParams) { +func restartStaleContainer(container container.Container, client container.Client, params types.UpdateParams) error { // Since we can't shutdown a watchtower container immediately, we need to // start the new one while the old one is still running. This prevents us // from re-using the same container name so we first rename the current @@ -142,17 +175,19 @@ func restartStaleContainer(container container.Container, client container.Clien if container.IsWatchtower() { if err := client.RenameContainer(container, util.RandName()); err != nil { log.Error(err) - return + return nil } } if !params.NoRestart { if newContainerID, err := client.StartContainer(container); err != nil { log.Error(err) + return err } else if container.Stale && params.LifecycleHooks { lifecycle.ExecutePostUpdateCommand(client, newContainerID) } } + return nil } func checkDependencies(containers []container.Container) { diff --git a/internal/actions/update_test.go b/internal/actions/update_test.go index 1a53aad..f1b8e85 100644 --- a/internal/actions/update_test.go +++ b/internal/actions/update_test.go @@ -59,7 +59,7 @@ var _ = Describe("the update action", func() { When("there are multiple containers using the same image", func() { It("should only try to remove the image once", func() { - err := actions.Update(client, types.UpdateParams{Cleanup: true}) + _, err := actions.Update(client, types.UpdateParams{Cleanup: true}) Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1)) }) @@ -75,7 +75,7 @@ var _ = Describe("the update action", func() { time.Now(), ), ) - err := actions.Update(client, types.UpdateParams{Cleanup: true}) + _, err := actions.Update(client, types.UpdateParams{Cleanup: true}) Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImageCount).To(Equal(2)) }) @@ -83,7 +83,7 @@ var _ = Describe("the update action", func() { When("performing a rolling restart update", func() { It("should try to remove the image once", func() { - err := actions.Update(client, types.UpdateParams{Cleanup: true, RollingRestart: true}) + _, err := actions.Update(client, types.UpdateParams{Cleanup: true, RollingRestart: true}) Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1)) }) @@ -121,7 +121,7 @@ var _ = Describe("the update action", func() { }) It("should not update those containers", func() { - err := actions.Update(client, types.UpdateParams{Cleanup: true}) + _, err := actions.Update(client, types.UpdateParams{Cleanup: true}) Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImageCount).To(Equal(1)) }) @@ -151,7 +151,7 @@ var _ = Describe("the update action", func() { }) It("should not update any containers", func() { - err := actions.Update(client, types.UpdateParams{MonitorOnly: true}) + _, err := actions.Update(client, types.UpdateParams{MonitorOnly: true}) Expect(err).NotTo(HaveOccurred()) Expect(client.TestData.TriedToRemoveImageCount).To(Equal(0)) }) diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 2f7a89f..d45f384 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -130,10 +130,15 @@ func RegisterSystemFlags(rootCmd *cobra.Command) { "Restart containers one at a time") flags.BoolP( - "http-api", + "http-api-update", "", - viper.GetBool("WATCHTOWER_HTTP_API"), + viper.GetBool("WATCHTOWER_HTTP_API_UPDATE"), "Runs Watchtower in HTTP API mode, so that image updates must to be triggered by a request") + flags.BoolP( + "http-api-metrics", + "", + viper.GetBool("WATCHTOWER_HTTP_API_METRICS"), + "Runs Watchtower with the Prometheus metrics API enabled") flags.StringP( "http-api-token", diff --git a/mkdocs.yml b/mkdocs.yml index 1d1506b..f628fbc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,5 +28,6 @@ nav: - 'Stop signals': 'stop-signals.md' - 'Lifecycle hooks': 'lifecycle-hooks.md' - 'Running multiple instances': 'running-multiple-instances.md' + - 'Metrics': 'metrics.md' plugins: - search diff --git a/pkg/api/api.go b/pkg/api/api.go index 12d12c3..987e4bd 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -1,63 +1,76 @@ package api import ( - "errors" - "io" - "net/http" - "os" - + "fmt" log "github.com/sirupsen/logrus" + "net/http" ) -var ( - lock chan bool -) +const tokenMissingMsg = "api token is empty or has not been set. exiting" -func init() { - lock = make(chan bool, 1) - lock <- true +// API is the http server responsible for serving the HTTP API endpoints +type API struct { + Token string + hasHandlers bool } -// SetupHTTPUpdates configures the endpoint needed for triggering updates via http -func SetupHTTPUpdates(apiToken string, updateFunction func()) error { - if apiToken == "" { - return errors.New("api token is empty or has not been set. not starting api") +// New is a factory function creating a new API instance +func New(token string) *API { + return &API{ + Token: token, + hasHandlers: false, } +} - log.Println("Watchtower HTTP API started.") - - http.HandleFunc("/v1/update", func(w http.ResponseWriter, r *http.Request) { - log.Info("Updates triggered by HTTP API request.") - - _, err := io.Copy(os.Stdout, r.Body) - if err != nil { - log.Println(err) +// RequireToken is wrapper around http.HandleFunc that checks token validity +func (api *API) RequireToken(fn http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", api.Token) { + log.Errorf("Invalid token \"%s\"", r.Header.Get("Authorization")) + log.Debugf("Expected token to be \"%s\"", api.Token) return } + log.Println("Valid token found.") + fn(w, r) + } +} - if r.Header.Get("Token") != apiToken { - log.Println("Invalid token. Not updating.") - return - } +// RegisterFunc is a wrapper around http.HandleFunc that also sets the flag used to determine whether to launch the API +func (api *API) RegisterFunc(path string, fn http.HandlerFunc) { + api.hasHandlers = true + http.HandleFunc(path, api.RequireToken(fn)) +} - log.Println("Valid token found. Attempting to update.") +// RegisterHandler is a wrapper around http.Handler that also sets the flag used to determine whether to launch the API +func (api *API) RegisterHandler(path string, handler http.Handler) { + api.hasHandlers = true + http.Handle(path, api.RequireToken(handler.ServeHTTP)) +} - select { - case chanValue := <-lock: - defer func() { lock <- chanValue }() - updateFunction() - default: - log.Debug("Skipped. Another update already running.") - } +// Start the API and serve over HTTP. Requires an API Token to be set. +func (api *API) Start(block bool) error { - }) + if !api.hasHandlers { + log.Debug("Watchtower HTTP API skipped.") + return nil + } + if api.Token == "" { + log.Fatal(tokenMissingMsg) + } + + log.Info("Watchtower HTTP API started.") + if block { + runHTTPServer() + } else { + go func() { + runHTTPServer() + }() + } return nil } -// WaitForHTTPUpdates starts the http server and listens for requests. -func WaitForHTTPUpdates() error { +func runHTTPServer() { + log.Info("Serving HTTP") log.Fatal(http.ListenAndServe(":8080", nil)) - os.Exit(0) - return nil } diff --git a/pkg/api/metrics/metrics.go b/pkg/api/metrics/metrics.go new file mode 100644 index 0000000..4faad4a --- /dev/null +++ b/pkg/api/metrics/metrics.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "github.com/containrrr/watchtower/pkg/metrics" + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Handler is an HTTP handle for serving metric data +type Handler struct { + Path string + Handle http.HandlerFunc + Metrics *metrics.Metrics +} + +// New is a factory function creating a new Metrics instance +func New() *Handler { + m := metrics.Default() + handler := promhttp.Handler() + + return &Handler{ + Path: "/v1/metrics", + Handle: handler.ServeHTTP, + Metrics: m, + } +} diff --git a/pkg/api/metrics/metrics_test.go b/pkg/api/metrics/metrics_test.go new file mode 100644 index 0000000..c1a4df0 --- /dev/null +++ b/pkg/api/metrics/metrics_test.go @@ -0,0 +1,77 @@ +package metrics_test + +import ( + "fmt" + "github.com/containrrr/watchtower/pkg/metrics" + "io/ioutil" + "net/http" + "testing" + + "github.com/containrrr/watchtower/pkg/api" + metricsAPI "github.com/containrrr/watchtower/pkg/api/metrics" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +const Token = "123123123" + +func TestContainer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Metrics Suite") +} + +func runTestServer(m *metricsAPI.Handler) { + http.Handle(m.Path, m.Handle) + go func() { + http.ListenAndServe(":8080", nil) + }() +} + +func getWithToken(c http.Client, url string) (*http.Response, error) { + req, _ := http.NewRequest("GET", url, nil) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", Token)) + return c.Do(req) +} + +var _ = Describe("the metrics", func() { + httpAPI := api.New(Token) + m := metricsAPI.New() + httpAPI.RegisterHandler(m.Path, m.Handle) + httpAPI.Start(false) + + // We should likely split this into multiple tests, but as prometheus requires a restart of the binary + // to reset the metrics and gauges, we'll just do it all at once. + + It("should serve metrics", func() { + metric := &metrics.Metric{ + Scanned: 4, + Updated: 3, + Failed: 1, + } + metrics.RegisterScan(metric) + c := http.Client{} + res, err := getWithToken(c, "http://localhost:8080/v1/metrics") + + Expect(err).NotTo(HaveOccurred()) + contents, err := ioutil.ReadAll(res.Body) + fmt.Printf("%s\n", string(contents)) + Expect(string(contents)).To(ContainSubstring("watchtower_containers_updated 3")) + Expect(string(contents)).To(ContainSubstring("watchtower_containers_failed 1")) + Expect(string(contents)).To(ContainSubstring("watchtower_containers_scanned 4")) + Expect(string(contents)).To(ContainSubstring("watchtower_scans_total 1")) + Expect(string(contents)).To(ContainSubstring("watchtower_scans_skipped 0")) + + for i := 0; i < 3; i++ { + metrics.RegisterScan(nil) + } + + res, err = getWithToken(c, "http://localhost:8080/v1/metrics") + Expect(err).NotTo(HaveOccurred()) + contents, err = ioutil.ReadAll(res.Body) + fmt.Printf("%s\n", string(contents)) + + Expect(string(contents)).To(ContainSubstring("watchtower_scans_total 4")) + Expect(string(contents)).To(ContainSubstring("watchtower_scans_skipped 3")) + }) +}) \ No newline at end of file diff --git a/pkg/api/update/update.go b/pkg/api/update/update.go new file mode 100644 index 0000000..463b082 --- /dev/null +++ b/pkg/api/update/update.go @@ -0,0 +1,50 @@ +package update + +import ( + "io" + "net/http" + "os" + + log "github.com/sirupsen/logrus" +) + +var ( + lock chan bool +) + +// New is a factory function creating a new Handler instance +func New(updateFn func()) *Handler { + lock = make(chan bool, 1) + lock <- true + + return &Handler{ + fn: updateFn, + Path: "/v1/update", + } +} + +// Handler is an API handler used for triggering container update scans +type Handler struct { + fn func() + Path string +} + +// Handle is the actual http.Handle function doing all the heavy lifting +func (handle *Handler) Handle(w http.ResponseWriter, r *http.Request) { + log.Info("Updates triggered by HTTP API request.") + + _, err := io.Copy(os.Stdout, r.Body) + if err != nil { + log.Println(err) + return + } + + select { + case chanValue := <-lock: + defer func() { lock <- chanValue }() + handle.fn() + default: + log.Debug("Skipped. Another update already running.") + } + +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..3a235af --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,91 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var metrics *Metrics + +// Metric is the data points of a single scan +type Metric struct { + Scanned int + Updated int + Failed int +} + +// Metrics is the handler processing all individual scan metrics +type Metrics struct { + channel chan *Metric + scanned prometheus.Gauge + updated prometheus.Gauge + failed prometheus.Gauge + total prometheus.Counter + skipped prometheus.Counter +} + +// Register registers metrics for an executed scan +func (metrics *Metrics) Register(metric *Metric) { + metrics.channel <- metric +} + +// Default creates a new metrics handler if none exists, otherwise returns the existing one +func Default() *Metrics { + if metrics != nil { + return metrics + } + + metrics = &Metrics{ + scanned: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "watchtower_containers_scanned", + Help: "Number of containers scanned for changes by watchtower during the last scan", + }), + updated: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "watchtower_containers_updated", + Help: "Number of containers updated by watchtower during the last scan", + }), + failed: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "watchtower_containers_failed", + Help: "Number of containers where update failed during the last scan", + }), + total: promauto.NewCounter(prometheus.CounterOpts{ + Name: "watchtower_scans_total", + Help: "Number of scans since the watchtower started", + }), + skipped: promauto.NewCounter(prometheus.CounterOpts{ + Name: "watchtower_scans_skipped", + Help: "Number of skipped scans since watchtower started", + }), + channel: make(chan *Metric, 10), + } + + go metrics.HandleUpdate(metrics.channel) + + return metrics +} + +// RegisterScan fetches a metric handler and enqueues a metric +func RegisterScan(metric *Metric) { + metrics := Default() + metrics.Register(metric) +} + +// HandleUpdate dequeue the metric channel and processes it +func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) { + for change := range channel { + if change == nil { + // Update was skipped and rescheduled + metrics.total.Inc() + metrics.skipped.Inc() + metrics.scanned.Set(0) + metrics.updated.Set(0) + metrics.failed.Set(0) + continue + } + // Update metrics with the new values + metrics.total.Inc() + metrics.scanned.Set(float64(change.Scanned)) + metrics.updated.Set(float64(change.Updated)) + metrics.failed.Set(float64(change.Failed)) + } +} diff --git a/pkg/notifications/gotify.go b/pkg/notifications/gotify.go index 47bab40..bb475bf 100644 --- a/pkg/notifications/gotify.go +++ b/pkg/notifications/gotify.go @@ -87,5 +87,5 @@ func (n *gotifyTypeNotifier) GetURL() string { func (n *gotifyTypeNotifier) StartNotification() {} func (n *gotifyTypeNotifier) SendNotification() {} -func (n *gotifyTypeNotifier) Close() {} +func (n *gotifyTypeNotifier) Close() {} func (n *gotifyTypeNotifier) Levels() []log.Level { return nil } diff --git a/pkg/notifications/msteams.go b/pkg/notifications/msteams.go index 0c99072..63c6aaa 100644 --- a/pkg/notifications/msteams.go +++ b/pkg/notifications/msteams.go @@ -63,6 +63,6 @@ func (n *msTeamsTypeNotifier) GetURL() string { func (n *msTeamsTypeNotifier) StartNotification() {} func (n *msTeamsTypeNotifier) SendNotification() {} -func (n *msTeamsTypeNotifier) Close() {} +func (n *msTeamsTypeNotifier) Close() {} func (n *msTeamsTypeNotifier) Levels() []log.Level { return nil } func (n *msTeamsTypeNotifier) Fire(entry *log.Entry) error { return nil } diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..1a30df0 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,9 @@ +scrape_configs: + - job_name: watchtower + scrape_interval: 5s + metrics_path: /v1/metrics + bearer_token: demotoken + static_configs: + - targets: + - 'watchtower:8080' +