cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimise inserting derivation inputs

Browse source 
Rather than querying for the output ids one by one and then running an insert
query for each derivation, perform the task with a single insert query.
This commit is contained in:
Christopher Baines 2021-09-24 17:51:52 +01:00
parent abff41f9ae
commit 3081887b90
1 changed files with 44 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

106
guix-data-service/model/derivation.scm
View file

@ -1082,25 +1082,6 @@ VALUES ($1, $2)"
(vector->list (json-string->scm env_vars))) (vector->list (json-string->scm env_vars)))
system)))) system))))
(define select-derivation-output-id
(mlambda (conn name path)
(match (exec-query
conn
"
SELECT derivation_outputs.id FROM derivation_outputs
INNER JOIN derivations
ON derivation_outputs.derivation_id = derivations.id
WHERE derivations.file_name = $1
AND derivation_outputs.name = $2"
path
name)
(((id))
id)
(()
(error (simple-format
#f "cannot find derivation-output with name ~A and path ~A"
name path))))))
(define (select-derivation-outputs-by-derivation-id conn id) (define (select-derivation-outputs-by-derivation-id conn id)
(define query (define query
(string-append (string-append
@ -1347,40 +1328,46 @@ WHERE derivation_source_files.store_path = $1"
env-vars))))))))) env-vars)))))))))
#f))) #f)))
(define (insert-derivation-inputs conn derivation-id derivation-inputs) (define (insert-derivation-inputs conn derivation-ids derivations)
(define (insert-into-derivation-inputs output-ids) (let ((data
(string-append "INSERT INTO derivation_inputs " (append-map
"(derivation_id, derivation_output_id) VALUES " (lambda (derivation-id derivation)
(string-join (append-map
(map (lambda (output-id) (match-lambda
(simple-format (($ <derivation-input> derivation-or-path sub-derivations)
#f "(~A, ~A)" (let ((path
derivation-id output-id)) (match derivation-or-path
output-ids) ((? derivation? d)
",") ;; The first field changed to a derivation (from the file
";")) ;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55
(derivation-file-name d))
((? string? s)
s))))
(map (lambda (sub-derivation)
(string-append "("
(number->string derivation-id)
", '" path
"', '" sub-derivation "')"))
sub-derivations))))
(derivation-inputs derivation)))
derivation-ids
derivations)))
(unless (null? derivation-inputs) (unless (null? data)
(exec-query (exec-query
conn conn
(insert-into-derivation-inputs (string-append
(append-map "
(match-lambda INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
(($ <derivation-input> derivation-or-path sub-derivations) SELECT vals.derivation_id, derivation_outputs.id
(let ((path FROM (VALUES "
(match derivation-or-path (string-join data ", ")
((? derivation? d) ") AS vals (derivation_id, file_name, output_name)
;; The first field changed to a derivation (from the file INNER JOIN derivations
;; name) in 5cf4b26d52bcea382d98fb4becce89be9ee37b55 ON derivations.file_name = vals.file_name
(derivation-file-name d)) INNER JOIN derivation_outputs
((? string? s) ON derivation_outputs.derivation_id = derivations.id
s)))) AND vals.output_name = derivation_outputs.name")))))
(map (lambda (sub-derivation)
(select-derivation-output-id conn
sub-derivation
path))
sub-derivations))))
derivation-inputs)))))
(define (select-from-derivation-source-files store-paths) (define (select-from-derivation-source-files store-paths)
(string-append (string-append
@ -1598,21 +1585,16 @@ LIMIT $1"
(ensure-input-derivations-exist (deduplicate-strings (ensure-input-derivations-exist (deduplicate-strings
(map derivation-input-path (map derivation-input-path
(append-map (append-map derivation-inputs
derivation-inputs derivations))))
derivations))))
(with-time-logging (with-time-logging
(simple-format (simple-format
#f "insert-missing-derivations: inserting inputs for ~A derivations" #f "insert-missing-derivations: inserting inputs for ~A derivations"
(length derivations)) (length derivations))
(for-each (lambda (derivation-id derivation) (insert-derivation-inputs conn
(insert-derivation-inputs conn derivation-ids
derivation-id derivations))
(derivation-inputs derivation)))
derivation-ids
derivations))
derivation-ids)) derivation-ids))