cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tweak inserting sources

Browse source 
To try and make it faster.
This commit is contained in:
Christopher Baines 2025-07-09 12:55:14 +01:00
parent a4480ac357
commit 683b375d56
1 changed files with 57 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

95
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -1093,46 +1093,65 @@
derivation-ids derivation-ids
#:key (log-tag "unspecified")) #:key (log-tag "unspecified"))
(with-time-logging (with-time-logging
(string-append "insert-missing-derivations: inserting sources (" log-tag ")") (string-append "insert-missing-derivations: inserting sources for "
(fibers-for-each (number->string (vector-length derivations))
(lambda (derivation-id derivation) " derivations (" log-tag ")")
(let ((sources (derivation-sources derivation))) (let ((sources-ids-vector
(unless (null? sources) (with-resource-from-pool postgresql-connection-pool conn
(let ((sources-ids (with-time-logging
(string-append
"insert-missing-derivations: inserting "
(number->string (vector-length derivations))
" derivation_source_files and derivation_sources"
" (" log-tag ")")
(vector-map
(lambda (_ derivation-id derivation)
(let ((sources (derivation-sources derivation)))
(if (null? sources)
#()
(insert-derivation-sources conn
derivation-id
sources))))
derivation-ids
derivations)))))
(with-time-logging
(string-append
"insert-missing-derivations: inserting "
(number->string (vector-length derivations))
" derivation_source_file_nars"
" (" log-tag ")")
(fibers-for-each
(lambda (derivation source-ids)
(for-each
(lambda (id source-file)
(when
(with-resource-from-pool postgresql-connection-pool conn (with-resource-from-pool postgresql-connection-pool conn
(insert-derivation-sources conn (match
derivation-id (exec-query
sources)))) conn
(fibers-for-each "
(lambda (id source-file)
(when
(with-resource-from-pool postgresql-connection-pool conn
(match
(exec-query
conn
"
SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1" SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(list (number->string id))) (list (number->string id)))
(() (()
;; Insert a placeholder to avoid other fibers ;; Insert a placeholder to avoid other fibers
;; working on this source file ;; working on this source file
(insert-placeholder-derivation-source-file-nar (insert-placeholder-derivation-source-file-nar
conn conn
id) id)
#t) #t)
(_ #f))) (_ #f)))
;; Use a utility thread to control concurrency here, to ;; Use a utility thread to control concurrency here, to
;; avoid using too much memory ;; avoid using too much memory
(call-with-utility-thread (call-with-utility-thread
(lambda () (lambda ()
(compute-and-update-derivation-source-file-nar (compute-and-update-derivation-source-file-nar
postgresql-connection-pool postgresql-connection-pool
id id
source-file))))) source-file)))))
sources-ids (vector->list source-ids)
sources))))) (derivation-sources derivation)))
derivation-ids derivations
derivations))) sources-ids-vector)))))
(define* (insert-missing-derivations postgresql-connection-pool (define* (insert-missing-derivations postgresql-connection-pool
call-with-utility-thread call-with-utility-thread