Improve chunking when inserting derivation inputs

Chunk the values inserted in the query, rather than the derivations involved,
as this is more consistent.
This commit is contained in:
Christopher Baines 2022-09-17 00:41:14 +02:00
parent 7050ea749f
commit 78a5abee21

View file

@ -1381,51 +1381,48 @@ WHERE derivation_source_files.store_path = $1"
#f)))
(define (insert-derivation-inputs conn derivation-ids derivations)
(define (process-chunk derivation-ids derivations)
(let ((query-parts
(append-map!
(lambda (derivation-id derivation)
(append-map!
(match-lambda
(($ <derivation-input> derivation-or-path sub-derivations)
(let ((path
(match derivation-or-path
((? derivation? d)
;; The first field changed to a derivation (from the file
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
(derivation-file-name d))
((? string? s)
s))))
(map (lambda (sub-derivation)
(string-append "("
(number->string derivation-id)
", '" path
"', '" sub-derivation "')"))
sub-derivations))))
(derivation-inputs derivation)))
derivation-ids
derivations)))
(let ((query-parts
(append-map!
(lambda (derivation-id derivation)
(append-map!
(match-lambda
(($ <derivation-input> derivation-or-path sub-derivations)
(let ((path
(match derivation-or-path
((? derivation? d)
;; The first field changed to a derivation (from the file
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
(derivation-file-name d))
((? string? s)
s))))
(map (lambda (sub-derivation)
(string-append "("
(number->string derivation-id)
", '" path
"', '" sub-derivation "')"))
sub-derivations))))
(derivation-inputs derivation)))
derivation-ids
derivations)))
(unless (null? query-parts)
(exec-query
conn
(string-append
"
(chunk-for-each!
(lambda (query-parts-chunk)
(exec-query
conn
(string-append
"
INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
SELECT vals.derivation_id, derivation_outputs.id
FROM (VALUES "
(string-join query-parts ", ")
") AS vals (derivation_id, file_name, output_name)
(string-join query-parts-chunk ", ")
") AS vals (derivation_id, file_name, output_name)
INNER JOIN derivations
ON derivations.file_name = vals.file_name
INNER JOIN derivation_outputs
ON derivation_outputs.derivation_id = derivations.id
AND vals.output_name = derivation_outputs.name")))))
(chunk-for-each! process-chunk
200
(list-copy derivation-ids)
(list-copy derivations)))
AND vals.output_name = derivation_outputs.name")))
1000
query-parts)))
(define (select-from-derivation-source-files store-paths)
(string-append