cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Instrument resource pool checkout failures

Browse source 
As I've got no idea why the resource pools sometimes stop working.
This commit is contained in:
Christopher Baines 2024-02-10 10:11:29 +00:00
parent 26f517d9c2
commit 76712e2b00
2 changed files with 22 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

22
guix-data-service/utils.scm
View file

@ -123,7 +123,8 @@
(sleep 5) (sleep 5)
(destructor/safe args))))) (destructor/safe args)))))
(let ((channel (make-channel))) (let ((channel (make-channel))
(checkout-failure-count 0))
(spawn-fiber (spawn-fiber
(lambda () (lambda ()
(while #t (while #t
@ -164,8 +165,12 @@
(wrap-operation (wrap-operation
(put-operation reply new-resource) (put-operation reply new-resource)
(const #t)) (const #t))
(wrap-operation (sleep-operation 0.2) (wrap-operation (sleep-operation 1)
(const #f)))))) (const #f))))))
(unless checkout-success?
(set! checkout-failure-count
(+ 1 checkout-failure-count)))
(loop (cons new-resource resources) (loop (cons new-resource resources)
(if checkout-success? (if checkout-success?
available available
@ -183,8 +188,12 @@
(wrap-operation (wrap-operation
(put-operation reply (car available)) (put-operation reply (car available))
(const #t)) (const #t))
(wrap-operation (sleep-operation 0.2) (wrap-operation (sleep-operation 1)
(const #f)))))) (const #f))))))
(unless checkout-success?
(set! checkout-failure-count
(+ 1 checkout-failure-count)))
(if checkout-success? (if checkout-success?
(loop resources (loop resources
(cdr available) (cdr available)
@ -222,9 +231,10 @@
resources-last-used))) resources-last-used)))
(('stats reply) (('stats reply)
(let ((stats (let ((stats
`((resources . ,(length resources)) `((resources . ,(length resources))
(available . ,(length available)) (available . ,(length available))
(waiters . ,(length waiters))))) (waiters . ,(length waiters))
(checkout-failure-count . ,checkout-failure-count))))
(perform-operation (perform-operation
(choice-operation (choice-operation

7
guix-data-service/web/controller.scm
View file

@ -190,7 +190,12 @@
(waiters . ,(make-gauge-metric (waiters . ,(make-gauge-metric
registry registry
"resource_pool_waiters_total" "resource_pool_waiters_total"
#:labels '(pool_name))))) #:labels '(pool_name)))
(checkout-failure-count
. ,(make-gauge-metric
registry
"resource_pool_checkout_failures_total"
#:labels '(pool_name)))))
(gc-metrics-updater (gc-metrics-updater
(get-gc-metrics-updater registry)) (get-gc-metrics-updater registry))