diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index e1700ad..f9b35e9 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -1224,17 +1224,40 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1" '() derivations))) (unless (null? input-derivations) - ;; Ensure all the input derivations exist - (chunk-for-each! - (lambda (chunk) - (insert-missing-derivations - postgresql-connection-pool - call-with-utility-thread - derivation-ids-hash-table - chunk - #:log-tag log-tag)) - 1000 - input-derivations))))) + (let loop ((chunk '()) + (count 0) + (rest input-derivations)) + (if (null? rest) + (unless (null? chunk) + (insert-missing-derivations + postgresql-connection-pool + call-with-utility-thread + derivation-ids-hash-table + chunk + #:log-tag log-tag)) + (if (= count 1000) + (begin + (simple-format #t "debug: inserting ~A input derivations\n" + count) + (insert-missing-derivations + postgresql-connection-pool + call-with-utility-thread + derivation-ids-hash-table + chunk + #:log-tag log-tag) + (loop '() + 0 + rest)) + (let ((drv (car rest))) + (if (hash-ref derivation-ids-hash-table + (derivation-file-name + drv)) + (loop chunk + count + (cdr rest)) + (loop (cons drv chunk) + (+ 1 count) + (cdr rest))))))))))) (let ((derivations derivation-ids @@ -1739,6 +1762,38 @@ WHERE builder != 'builtin:download' (for-each fix-derivation broken-derivations)))) +(define (flatten-derivation-graph derivations + derivation-ids-hash-table) + (define seen-hash-table + (make-hash-table)) + + (define (flatten-inputs derivation result) + (fold + (lambda (input result) + (let ((drv (derivation-input-derivation input))) + (if (or (hash-ref derivation-ids-hash-table + (derivation-file-name drv)) + (hash-ref seen-hash-table + drv)) + result + (begin + (hash-set! seen-hash-table drv #t) + (cons drv + (flatten-inputs drv result)))))) + result + (derivation-inputs derivation))) + + (reverse! + (fold + (lambda (derivation result) + (let ((flat-inputs + (flatten-inputs derivation + result))) + (cons derivation + flat-inputs))) + '() + derivations))) + (define* (derivation-file-names->derivation-ids postgresql-connection-pool call-with-utility-thread read-derivations/serialised @@ -1777,12 +1832,21 @@ WHERE builder != 'builtin:download' (let ((chunk-counter 0)) (chunk-for-each! (lambda (missing-derivation-file-names-chunk) - (let ((missing-derivations-chunk - (read-derivations/serialised - missing-derivation-file-names-chunk))) + (simple-format + #t "debug: derivation-file-names->derivation-ids: processing chunk ~A (~A)\n" + chunk-counter + log-tag) + (let* ((missing-derivations-chunk + (read-derivations/serialised + missing-derivation-file-names-chunk)) + (flat-missing-derivations + (with-time-logging "flattening missing derivations" + (flatten-derivation-graph + missing-derivations-chunk + derivation-ids-hash-table)))) (simple-format - #t "debug: derivation-file-names->derivation-ids: processing chunk ~A (~A)\n" - chunk-counter + #t "debug: derivation-file-names->derivation-ids: processing ~A flat missing derivations (~A)\n" + (length flat-missing-derivations) log-tag) (set! chunk-counter (+ 1 chunk-counter)) (insert-missing-derivations postgresql-connection-pool