cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve deleting derivations

Browse source 
Drop the batch size to get rid of warnings about memory usage and improve the
logging by adding duration information.
This commit is contained in:
Christopher Baines 2022-07-08 20:54:12 +01:00
parent 788571f53f
commit 39487cd7e6
1 changed files with 35 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

69
guix-data-service/data-deletion.scm
View file

@ -336,7 +336,7 @@ FROM (
WHERE commit = ''"))))) WHERE commit = ''")))))
(define* (delete-unreferenced-derivations #:key (define* (delete-unreferenced-derivations #:key
(batch-size 1000000)) (batch-size 100000))
(define (delete-builds-for-derivation-output-details-set (define (delete-builds-for-derivation-output-details-set
conn conn
derivation-output-details-set-id) derivation-output-details-set-id)
@ -491,10 +491,11 @@ DELETE FROM derivations WHERE id = $1"
(lambda (conn) (lambda (conn)
(define (delete-batch conn) (define (delete-batch conn)
(let* ((derivations (let* ((derivations
(map car (with-time-logging "fetching batch of derivations"
(exec-query (map car
conn (exec-query
" conn
"
SELECT DISTINCT derivation_id SELECT DISTINCT derivation_id
FROM derivation_outputs FROM derivation_outputs
WHERE NOT EXISTS ( WHERE NOT EXISTS (
@ -514,36 +515,37 @@ WHERE NOT EXISTS (
SELECT 1 FROM guix_revision_system_test_derivations SELECT 1 FROM guix_revision_system_test_derivations
WHERE derivation_id = derivation_outputs.derivation_id WHERE derivation_id = derivation_outputs.derivation_id
) LIMIT $1" ) LIMIT $1"
(list (number->string batch-size))))) (list (number->string batch-size))))))
(derivations-count (length derivations))) (derivations-count (length derivations)))
(simple-format (current-error-port)
"Looking at ~A derivations\n"
derivations-count)
(let ((deleted-count (let ((deleted-count
(fold (with-time-logging
(lambda (count result) (simple-format #f
(+ result count)) "Looking at ~A derivations"
0 derivations-count)
(map (fold
(lambda (derivation-id) (lambda (count result)
(unless (string->number derivation-id) (+ result count))
(error 0
(simple-format #f "derivation-id: ~A is not a number" (map
derivation-id))) (lambda (derivation-id)
(unless (string->number derivation-id)
(error
(simple-format #f "derivation-id: ~A is not a number"
derivation-id)))
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(with-postgresql-transaction (with-postgresql-transaction
conn conn
(lambda (conn) (lambda (conn)
(exec-query (exec-query
conn conn
" "
SET CONSTRAINTS derivations_by_output_details_set_derivation_id_fkey DEFERRED") SET CONSTRAINTS derivations_by_output_details_set_derivation_id_fkey DEFERRED")
(maybe-delete-derivation conn (maybe-delete-derivation conn
derivation-id)))))) derivation-id))))))
derivations)))) derivations)))))
(simple-format (current-error-port) (simple-format (current-error-port)
"Deleted ~A derivations\n" "Deleted ~A derivations\n"
deleted-count) deleted-count)
@ -558,10 +560,9 @@ SET CONSTRAINTS derivations_by_output_details_set_derivation_id_fkey DEFERRED")
(let ((batch-deleted-count (delete-batch conn))) (let ((batch-deleted-count (delete-batch conn)))
(if (eq? 0 batch-deleted-count) (if (eq? 0 batch-deleted-count)
(begin (begin
(simple-format (with-time-logging
(current-output-port) "Deleting unused derivation_source_files entries"
"Deleting unused derivation_source_files entries") (delete-unreferenced-derivations-source-files conn))
(delete-unreferenced-derivations-source-files conn)
(simple-format (simple-format
(current-output-port) (current-output-port)
"Finished deleting derivations, deleted ~A in total\n" "Finished deleting derivations, deleted ~A in total\n"