cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Instrument resource pool checkout failures

Browse source 
As I've got no idea why the resource pools sometimes stop working.
This commit is contained in:
Christopher Baines 2024-02-10 10:11:29 +00:00
parent 26f517d9c2
commit 76712e2b00
2 changed files with 22 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

22
guix-data-service/utils.scm
View file

@ -123,7 +123,8 @@
(sleep 5)
(destructor/safe args)))))
(let ((channel (make-channel)))
(let ((channel (make-channel))
(checkout-failure-count 0))
(spawn-fiber
(lambda ()
(while #t
@ -164,8 +165,12 @@
(wrap-operation
(put-operation reply new-resource)
(const #t))
(wrap-operation (sleep-operation 0.2)
(wrap-operation (sleep-operation 1)
(const #f))))))
(unless checkout-success?
(set! checkout-failure-count
(+ 1 checkout-failure-count)))
(loop (cons new-resource resources)
(if checkout-success?
available
@ -183,8 +188,12 @@
(wrap-operation
(put-operation reply (car available))
(const #t))
(wrap-operation (sleep-operation 0.2)
(wrap-operation (sleep-operation 1)
(const #f))))))
(unless checkout-success?
(set! checkout-failure-count
(+ 1 checkout-failure-count)))
(if checkout-success?
(loop resources
(cdr available)
@ -222,9 +231,10 @@
resources-last-used)))
(('stats reply)
(let ((stats
`((resources . ,(length resources))
(available . ,(length available))
(waiters . ,(length waiters)))))
`((resources . ,(length resources))
(available . ,(length available))
(waiters . ,(length waiters))
(checkout-failure-count . ,checkout-failure-count))))
(perform-operation
(choice-operation

7
guix-data-service/web/controller.scm
View file

@ -190,7 +190,12 @@
(waiters . ,(make-gauge-metric
registry
"resource_pool_waiters_total"
#:labels '(pool_name)))))
#:labels '(pool_name)))
(checkout-failure-count
. ,(make-gauge-metric
registry
"resource_pool_checkout_failures_total"
#:labels '(pool_name)))))
(gc-metrics-updater
(get-gc-metrics-updater registry))