cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Insert derivations in chunks

Browse source 
To avoid making a very large query when inserting lots of derivations.
This commit is contained in:
Christopher Baines 2021-10-03 14:54:43 +01:00
parent 0796cb3bd3
commit b28d338de7
1 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

13
guix-data-service/model/derivation.scm
View file

@ -1514,7 +1514,7 @@ LIMIT $1"
(map read-derivation-from-file (map read-derivation-from-file
missing-derivations-filenames)))))) missing-derivations-filenames))))))
(define (insert-into-derivations) (define (insert-into-derivations dervs)
(string-append (string-append
"INSERT INTO derivations " "INSERT INTO derivations "
"(file_name, builder, args, env_vars, system_id) VALUES " "(file_name, builder, args, env_vars, system_id) VALUES "
@ -1535,7 +1535,7 @@ LIMIT $1"
env-vars) env-vars)
",") ",")
(system->system-id conn system)))) (system->system-id conn system))))
derivations) dervs)
",") ",")
" RETURNING id" " RETURNING id"
";")) ";"))
@ -1544,9 +1544,12 @@ LIMIT $1"
#t "debug: insert-missing-derivations: inserting ~A derivations\n" #t "debug: insert-missing-derivations: inserting ~A derivations\n"
(length derivations)) (length derivations))
(let ((derivation-ids (let ((derivation-ids
(map (lambda (result) (append-map
(string->number (car result))) (lambda (chunk)
(exec-query conn (insert-into-derivations))))) (map (lambda (result)
(string->number (car result)))
(exec-query conn (insert-into-derivations chunk))))
(chunk derivations 5000))))
(simple-format (simple-format
#t "debug: insert-missing-derivations: updating hash table\n") #t "debug: insert-missing-derivations: updating hash table\n")