cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tweak inserting sources

Browse source 
To try and make it faster.
This commit is contained in:
Christopher Baines 2025-07-09 12:55:14 +01:00
parent a4480ac357
commit 683b375d56
1 changed files with 57 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

95
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -1093,46 +1093,65 @@
derivation-ids
#:key (log-tag "unspecified"))
(with-time-logging
(string-append "insert-missing-derivations: inserting sources (" log-tag ")")
(fibers-for-each
(lambda (derivation-id derivation)
(let ((sources (derivation-sources derivation)))
(unless (null? sources)
(let ((sources-ids
(string-append "insert-missing-derivations: inserting sources for "
(number->string (vector-length derivations))
" derivations (" log-tag ")")
(let ((sources-ids-vector
(with-resource-from-pool postgresql-connection-pool conn
(with-time-logging
(string-append
"insert-missing-derivations: inserting "
(number->string (vector-length derivations))
" derivation_source_files and derivation_sources"
" (" log-tag ")")
(vector-map
(lambda (_ derivation-id derivation)
(let ((sources (derivation-sources derivation)))
(if (null? sources)
#()
(insert-derivation-sources conn
derivation-id
sources))))
derivation-ids
derivations)))))
(with-time-logging
(string-append
"insert-missing-derivations: inserting "
(number->string (vector-length derivations))
" derivation_source_file_nars"
" (" log-tag ")")
(fibers-for-each
(lambda (derivation source-ids)
(for-each
(lambda (id source-file)
(when
(with-resource-from-pool postgresql-connection-pool conn
(insert-derivation-sources conn
derivation-id
sources))))
(fibers-for-each
(lambda (id source-file)
(when
(with-resource-from-pool postgresql-connection-pool conn
(match
(exec-query
conn
"
(match
(exec-query
conn
"
SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(list (number->string id)))
(()
;; Insert a placeholder to avoid other fibers
;; working on this source file
(insert-placeholder-derivation-source-file-nar
conn
id)
#t)
(_ #f)))
;; Use a utility thread to control concurrency here, to
;; avoid using too much memory
(call-with-utility-thread
(lambda ()
(compute-and-update-derivation-source-file-nar
postgresql-connection-pool
id
source-file)))))
sources-ids
sources)))))
derivation-ids
derivations)))
(list (number->string id)))
(()
;; Insert a placeholder to avoid other fibers
;; working on this source file
(insert-placeholder-derivation-source-file-nar
conn
id)
#t)
(_ #f)))
;; Use a utility thread to control concurrency here, to
;; avoid using too much memory
(call-with-utility-thread
(lambda ()
(compute-and-update-derivation-source-file-nar
postgresql-connection-pool
id
source-file)))))
(vector->list source-ids)
(derivation-sources derivation)))
derivations
sources-ids-vector)))))
(define* (insert-missing-derivations postgresql-connection-pool
call-with-utility-thread