cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Insert guix_revision_package_derivation_distribution_counts later

Browse source 
As this should speed up processing revisions.
This commit is contained in:
Christopher Baines 2024-11-26 14:12:17 +00:00
parent 620ac16e26
commit 64aeeffd8e
2 changed files with 49 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

15
guix-data-service/jobs.scm
View file

@ -30,6 +30,8 @@
count-log-parts count-log-parts
combine-log-parts! combine-log-parts!
guix-revision-id-for-job
process-jobs process-jobs
default-max-processes)) default-max-processes))
@ -124,6 +126,19 @@ guix-data-service: error: missing log line: ~A
(with-time-logging "vacuuming log parts" (with-time-logging "vacuuming log parts"
(vacuum-log-parts-table conn))) (vacuum-log-parts-table conn)))
(define (guix-revision-id-for-job conn job-id)
(match (exec-query
conn
"
SELECT guix_revisions.id
FROM guix_revisions
INNER JOIN load_new_guix_revision_jobs
ON guix_revisions.git_repository_id = load_new_guix_revision_jobs.git_repository_id
AND guix_revisions.commit = load_new_guix_revision_jobs.commit
WHERE load_new_guix_revision_jobs.id = $1"
(list (simple-format #f "~A" job-id)))
(((id)) id)))
(define* (process-jobs conn #:key max-processes (define* (process-jobs conn #:key max-processes
latest-branch-revision-max-processes latest-branch-revision-max-processes
skip-system-tests? skip-system-tests?

46
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -53,6 +53,7 @@
#:use-module (guix build utils) #:use-module (guix build utils)
#:use-module ((guix build syscalls) #:use-module ((guix build syscalls)
#:select (set-thread-name free-disk-space)) #:select (set-thread-name free-disk-space))
#:use-module (guix-data-service jobs)
#:use-module (guix-data-service config) #:use-module (guix-data-service config)
#:use-module (guix-data-service database) #:use-module (guix-data-service database)
#:use-module (guix-data-service utils) #:use-module (guix-data-service utils)
@ -2109,18 +2110,6 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
2000 2000
package-derivation-ids))) package-derivation-ids)))
(with-resource-from-pool postgresql-connection-pool conn
(with-time-logging
(simple-format
#f "insert-guix-revision-package-derivation-distribution-counts (~A ~A)"
system target)
(insert-guix-revision-package-derivation-distribution-counts
conn
(fibers-force guix-revision-id-promise)
(number->string
(system->system-id conn system))
(or target ""))))
'finished) 'finished)
(let ((get-derivations/fiberized (let ((get-derivations/fiberized
@ -2876,6 +2865,39 @@ SKIP LOCKED")
(when result (when result
(parallel-via-fibers (parallel-via-fibers
(with-postgresql-connection
(simple-format #f "post load-new-guix-revision ~A counts" id)
(lambda (conn)
(let ((revision-id
(guix-revision-id-for-job conn id)))
(with-time-logging
(simple-format
#f "inserting guix_revision_package_derivation_distribution_counts for revision ~A"
revision-id)
(let ((system-ids-and-targets
(exec-query
conn
"
SELECT DISTINCT system_id, target
FROM package_derivations
INNER JOIN guix_revision_package_derivations
ON package_derivations.id = guix_revision_package_derivations.package_derivation_id
WHERE revision_id = $1"
(list revision-id))))
(with-postgresql-transaction
conn
(lambda (conn)
(for-each
(match-lambda
((system-id target)
(insert-guix-revision-package-derivation-distribution-counts
conn
revision-id
system-id
target)))
system-ids-and-targets))))))))
(with-postgresql-connection (with-postgresql-connection
(simple-format #f "post load-new-guix-revision ~A" id) (simple-format #f "post load-new-guix-revision ~A" id)
(lambda (conn) (lambda (conn)