From b28d338de75a6470097b9c12769dcb1ba143f09d Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sun, 3 Oct 2021 14:54:43 +0100 Subject: [PATCH] Insert derivations in chunks To avoid making a very large query when inserting lots of derivations. --- guix-data-service/model/derivation.scm | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/guix-data-service/model/derivation.scm b/guix-data-service/model/derivation.scm index 3178e83..b258803 100644 --- a/guix-data-service/model/derivation.scm +++ b/guix-data-service/model/derivation.scm @@ -1514,7 +1514,7 @@ LIMIT $1" (map read-derivation-from-file missing-derivations-filenames)))))) - (define (insert-into-derivations) + (define (insert-into-derivations dervs) (string-append "INSERT INTO derivations " "(file_name, builder, args, env_vars, system_id) VALUES " @@ -1535,7 +1535,7 @@ LIMIT $1" env-vars) ",") (system->system-id conn system)))) - derivations) + dervs) ",") " RETURNING id" ";")) @@ -1544,9 +1544,12 @@ LIMIT $1" #t "debug: insert-missing-derivations: inserting ~A derivations\n" (length derivations)) (let ((derivation-ids - (map (lambda (result) - (string->number (car result))) - (exec-query conn (insert-into-derivations))))) + (append-map + (lambda (chunk) + (map (lambda (result) + (string->number (car result))) + (exec-query conn (insert-into-derivations chunk)))) + (chunk derivations 5000)))) (simple-format #t "debug: insert-missing-derivations: updating hash table\n")