Optimise inserting derivation inputs

Rather than querying for the output ids one by one and then running an insert
query for each derivation, perform the task with a single insert query.
This commit is contained in:
Christopher Baines 2021-09-24 17:51:52 +01:00
parent abff41f9ae
commit 3081887b90

View file

@ -1082,25 +1082,6 @@ VALUES ($1, $2)"
(vector->list (json-string->scm env_vars))) (vector->list (json-string->scm env_vars)))
system)))) system))))
(define select-derivation-output-id
(mlambda (conn name path)
(match (exec-query
conn
"
SELECT derivation_outputs.id FROM derivation_outputs
INNER JOIN derivations
ON derivation_outputs.derivation_id = derivations.id
WHERE derivations.file_name = $1
AND derivation_outputs.name = $2"
path
name)
(((id))
id)
(()
(error (simple-format
#f "cannot find derivation-output with name ~A and path ~A"
name path))))))
(define (select-derivation-outputs-by-derivation-id conn id) (define (select-derivation-outputs-by-derivation-id conn id)
(define query (define query
(string-append (string-append
@ -1347,23 +1328,10 @@ WHERE derivation_source_files.store_path = $1"
env-vars))))))))) env-vars)))))))))
#f))) #f)))
(define (insert-derivation-inputs conn derivation-id derivation-inputs) (define (insert-derivation-inputs conn derivation-ids derivations)
(define (insert-into-derivation-inputs output-ids) (let ((data
(string-append "INSERT INTO derivation_inputs " (append-map
"(derivation_id, derivation_output_id) VALUES " (lambda (derivation-id derivation)
(string-join
(map (lambda (output-id)
(simple-format
#f "(~A, ~A)"
derivation-id output-id))
output-ids)
",")
";"))
(unless (null? derivation-inputs)
(exec-query
conn
(insert-into-derivation-inputs
(append-map (append-map
(match-lambda (match-lambda
(($ <derivation-input> derivation-or-path sub-derivations) (($ <derivation-input> derivation-or-path sub-derivations)
@ -1376,11 +1344,30 @@ WHERE derivation_source_files.store_path = $1"
((? string? s) ((? string? s)
s)))) s))))
(map (lambda (sub-derivation) (map (lambda (sub-derivation)
(select-derivation-output-id conn (string-append "("
sub-derivation (number->string derivation-id)
path)) ", '" path
"', '" sub-derivation "')"))
sub-derivations)))) sub-derivations))))
derivation-inputs))))) (derivation-inputs derivation)))
derivation-ids
derivations)))
(unless (null? data)
(exec-query
conn
(string-append
"
INSERT INTO derivation_inputs (derivation_id, derivation_output_id)
SELECT vals.derivation_id, derivation_outputs.id
FROM (VALUES "
(string-join data ", ")
") AS vals (derivation_id, file_name, output_name)
INNER JOIN derivations
ON derivations.file_name = vals.file_name
INNER JOIN derivation_outputs
ON derivation_outputs.derivation_id = derivations.id
AND vals.output_name = derivation_outputs.name")))))
(define (select-from-derivation-source-files store-paths) (define (select-from-derivation-source-files store-paths)
(string-append (string-append
@ -1598,19 +1585,14 @@ LIMIT $1"
(ensure-input-derivations-exist (deduplicate-strings (ensure-input-derivations-exist (deduplicate-strings
(map derivation-input-path (map derivation-input-path
(append-map (append-map derivation-inputs
derivation-inputs
derivations)))) derivations))))
(with-time-logging (with-time-logging
(simple-format (simple-format
#f "insert-missing-derivations: inserting inputs for ~A derivations" #f "insert-missing-derivations: inserting inputs for ~A derivations"
(length derivations)) (length derivations))
(for-each (lambda (derivation-id derivation)
(insert-derivation-inputs conn (insert-derivation-inputs conn
derivation-id
(derivation-inputs derivation)))
derivation-ids derivation-ids
derivations)) derivations))