cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve chunking when inserting derivation inputs

Browse source 
Chunk the values inserted in the query, rather than the derivations involved,
as this is more consistent.
This commit is contained in:
Christopher Baines 2022-09-17 00:41:14 +02:00
parent 7050ea749f
commit 78a5abee21
1 changed files with 34 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

71
guix-data-service/model/derivation.scm
View file

@ -1381,51 +1381,48 @@ WHERE derivation_source_files.store_path = $1"
#f))) #f)))
(define (insert-derivation-inputs conn derivation-ids derivations) (define (insert-derivation-inputs conn derivation-ids derivations)
(define (process-chunk derivation-ids derivations) (let ((query-parts
(let ((query-parts (append-map!
(append-map! (lambda (derivation-id derivation)
(lambda (derivation-id derivation) (append-map!
(append-map! (match-lambda
(match-lambda (($ <derivation-input> derivation-or-path sub-derivations)
(($ <derivation-input> derivation-or-path sub-derivations) (let ((path
(let ((path (match derivation-or-path
(match derivation-or-path ((? derivation? d)
((? derivation? d) ;; The first field changed to a derivation (from the file
;; The first field changed to a derivation (from the file ;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55 (derivation-file-name d))
(derivation-file-name d)) ((? string? s)
((? string? s) s))))
s)))) (map (lambda (sub-derivation)
(map (lambda (sub-derivation) (string-append "("
(string-append "(" (number->string derivation-id)
(number->string derivation-id) ", '" path
", '" path "', '" sub-derivation "')"))
"', '" sub-derivation "')")) sub-derivations))))
sub-derivations)))) (derivation-inputs derivation)))
(derivation-inputs derivation))) derivation-ids
derivation-ids derivations)))
derivations)))
(unless (null? query-parts) (chunk-for-each!
(exec-query (lambda (query-parts-chunk)
conn (exec-query
(string-append conn
" (string-append
"
INSERT INTO derivation_inputs (derivation_id, derivation_output_id) INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
SELECT vals.derivation_id, derivation_outputs.id SELECT vals.derivation_id, derivation_outputs.id
FROM (VALUES " FROM (VALUES "
(string-join query-parts ", ") (string-join query-parts-chunk ", ")
") AS vals (derivation_id, file_name, output_name) ") AS vals (derivation_id, file_name, output_name)
INNER JOIN derivations INNER JOIN derivations
ON derivations.file_name = vals.file_name ON derivations.file_name = vals.file_name
INNER JOIN derivation_outputs INNER JOIN derivation_outputs
ON derivation_outputs.derivation_id = derivations.id ON derivation_outputs.derivation_id = derivations.id
AND vals.output_name = derivation_outputs.name"))))) AND vals.output_name = derivation_outputs.name")))
1000
(chunk-for-each! process-chunk query-parts)))
200
(list-copy derivation-ids)
(list-copy derivations)))
(define (select-from-derivation-source-files store-paths) (define (select-from-derivation-source-files store-paths)
(string-append (string-append