Improve chunking when inserting derivation inputs
Chunk the values inserted in the query, rather than the derivations involved, as this is more consistent.
This commit is contained in:
parent
7050ea749f
commit
78a5abee21
1 changed files with 34 additions and 37 deletions
|
|
@ -1381,51 +1381,48 @@ WHERE derivation_source_files.store_path = $1"
|
|||
#f)))
|
||||
|
||||
(define (insert-derivation-inputs conn derivation-ids derivations)
|
||||
(define (process-chunk derivation-ids derivations)
|
||||
(let ((query-parts
|
||||
(append-map!
|
||||
(lambda (derivation-id derivation)
|
||||
(append-map!
|
||||
(match-lambda
|
||||
(($ <derivation-input> derivation-or-path sub-derivations)
|
||||
(let ((path
|
||||
(match derivation-or-path
|
||||
((? derivation? d)
|
||||
;; The first field changed to a derivation (from the file
|
||||
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
|
||||
(derivation-file-name d))
|
||||
((? string? s)
|
||||
s))))
|
||||
(map (lambda (sub-derivation)
|
||||
(string-append "("
|
||||
(number->string derivation-id)
|
||||
", '" path
|
||||
"', '" sub-derivation "')"))
|
||||
sub-derivations))))
|
||||
(derivation-inputs derivation)))
|
||||
derivation-ids
|
||||
derivations)))
|
||||
(let ((query-parts
|
||||
(append-map!
|
||||
(lambda (derivation-id derivation)
|
||||
(append-map!
|
||||
(match-lambda
|
||||
(($ <derivation-input> derivation-or-path sub-derivations)
|
||||
(let ((path
|
||||
(match derivation-or-path
|
||||
((? derivation? d)
|
||||
;; The first field changed to a derivation (from the file
|
||||
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
|
||||
(derivation-file-name d))
|
||||
((? string? s)
|
||||
s))))
|
||||
(map (lambda (sub-derivation)
|
||||
(string-append "("
|
||||
(number->string derivation-id)
|
||||
", '" path
|
||||
"', '" sub-derivation "')"))
|
||||
sub-derivations))))
|
||||
(derivation-inputs derivation)))
|
||||
derivation-ids
|
||||
derivations)))
|
||||
|
||||
(unless (null? query-parts)
|
||||
(exec-query
|
||||
conn
|
||||
(string-append
|
||||
"
|
||||
(chunk-for-each!
|
||||
(lambda (query-parts-chunk)
|
||||
(exec-query
|
||||
conn
|
||||
(string-append
|
||||
"
|
||||
INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
|
||||
SELECT vals.derivation_id, derivation_outputs.id
|
||||
FROM (VALUES "
|
||||
(string-join query-parts ", ")
|
||||
") AS vals (derivation_id, file_name, output_name)
|
||||
(string-join query-parts-chunk ", ")
|
||||
") AS vals (derivation_id, file_name, output_name)
|
||||
INNER JOIN derivations
|
||||
ON derivations.file_name = vals.file_name
|
||||
INNER JOIN derivation_outputs
|
||||
ON derivation_outputs.derivation_id = derivations.id
|
||||
AND vals.output_name = derivation_outputs.name")))))
|
||||
|
||||
(chunk-for-each! process-chunk
|
||||
200
|
||||
(list-copy derivation-ids)
|
||||
(list-copy derivations)))
|
||||
AND vals.output_name = derivation_outputs.name")))
|
||||
1000
|
||||
query-parts)))
|
||||
|
||||
(define (select-from-derivation-source-files store-paths)
|
||||
(string-append
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue