cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Parallelise inserting derivation counts

Browse source
This commit is contained in:
Christopher Baines 2024-12-21 08:39:36 +00:00
parent 6ce51e8880
commit 50127de12a
1 changed files with 33 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

59
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -2882,38 +2882,45 @@ SKIP LOCKED")
(when result (when result
(fibers-parallel (fibers-parallel
(with-postgresql-connection (let ((revision-id
(simple-format #f "post load-new-guix-revision ~A counts" id) system-ids-and-targets
(lambda (conn) (with-postgresql-connection
(let ((revision-id (simple-format #f "post load-new-guix-revision ~A counts" id)
(guix-revision-id-for-job conn id))) (lambda (conn)
(with-time-logging (let ((revision-id
(simple-format (guix-revision-id-for-job conn id)))
#f "inserting guix_revision_package_derivation_distribution_counts for revision ~A" (values
revision-id) revision-id
(let ((system-ids-and-targets (exec-query
(exec-query conn
conn "
"
SELECT DISTINCT system_id, target SELECT DISTINCT system_id, target
FROM package_derivations FROM package_derivations
INNER JOIN guix_revision_package_derivations INNER JOIN guix_revision_package_derivations
ON package_derivations.id = guix_revision_package_derivations.package_derivation_id ON package_derivations.id = guix_revision_package_derivations.package_derivation_id
WHERE revision_id = $1" WHERE revision_id = $1"
(list revision-id)))) (list revision-id))))))))
(with-postgresql-transaction (with-time-logging
conn (simple-format
(lambda (conn) #f "inserting guix_revision_package_derivation_distribution_counts for revision ~A"
(for-each revision-id)
(match-lambda (fibers-batch-for-each
((system-id target) (match-lambda
(insert-guix-revision-package-derivation-distribution-counts ((system-id target)
conn (with-postgresql-connection
revision-id (simple-format #f "post load-new-guix-revision ~A counts" id)
system-id (lambda (conn)
target))) (with-postgresql-transaction
system-ids-and-targets)))))))) conn
(lambda (conn)
(insert-guix-revision-package-derivation-distribution-counts
conn
revision-id
system-id
target)))))))
parallelism
system-ids-and-targets)))
(with-postgresql-connection (with-postgresql-connection
(simple-format #f "post load-new-guix-revision ~A" id) (simple-format #f "post load-new-guix-revision ~A" id)