cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Parallelise inserting package derivation distribution counts

Browse source
This commit is contained in:
Christopher Baines 2024-08-08 13:30:30 +01:00
parent 885a4758d3
commit 7869082816
2 changed files with 51 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

25
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -1856,8 +1856,9 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(loop (catch #t (loop (catch #t
(lambda () (lambda ()
(stat:size (stat "/var/guix/db/db.sqlite-wal"))) (stat:size (stat "/var/guix/db/db.sqlite-wal")))
(lambda _ 0))))) (lambda _ 0))))))
(define (process-system-and-target system target)
(with-time-logging (with-time-logging
(simple-format #f "processing derivations for ~A" (cons system target)) (simple-format #f "processing derivations for ~A" (cons system target))
(let ((derivations-vector (make-vector packages-count))) (let ((derivations-vector (make-vector packages-count)))
@ -1913,7 +1914,18 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
guix-revision-id guix-revision-id
package-derivation-ids-chunk))) package-derivation-ids-chunk)))
2000 2000
package-derivation-ids)))))) package-derivation-ids)))))
(with-resource-from-pool postgresql-connection-pool conn
(with-time-logging
(simple-format
#f "insert-guix-revision-package-derivation-distribution-counts (~A ~A)"
system target)
(insert-guix-revision-package-derivation-distribution-counts
conn
guix-revision-id
system
(or target "")))))
(let ((process-system-and-target/fiberized (let ((process-system-and-target/fiberized
(fiberize process-system-and-target (fiberize process-system-and-target
@ -1925,14 +1937,7 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(with-resource-from-pool inf-and-store-pool res (with-resource-from-pool inf-and-store-pool res
(match res (match res
((inferior . inferior-store) ((inferior . inferior-store)
(inferior-fetch-system-target-pairs inferior)))))) (inferior-fetch-system-target-pairs inferior)))))))
(with-resource-from-pool postgresql-connection-pool conn
(with-time-logging
"insert-guix-revision-package-derivation-distribution-counts"
(insert-guix-revision-package-derivation-distribution-counts
conn
guix-revision-id))))
(define (extract-and-store-system-tests) (define (extract-and-store-system-tests)
(if skip-system-tests? (if skip-system-tests?

63
guix-data-service/model/guix-revision-package-derivation.scm
View file

@ -20,6 +20,7 @@
#:use-module (ice-9 threads) #:use-module (ice-9 threads)
#:use-module (squee) #:use-module (squee)
#:use-module (guix-data-service database) #:use-module (guix-data-service database)
#:use-module (guix-data-service model system)
#:export (insert-guix-revision-package-derivations #:export (insert-guix-revision-package-derivations
insert-guix-revision-package-derivation-distribution-counts insert-guix-revision-package-derivation-distribution-counts
@ -46,17 +47,9 @@
(define (insert-guix-revision-package-derivation-distribution-counts (define (insert-guix-revision-package-derivation-distribution-counts
conn conn
guix-revision-id) guix-revision-id
(define system-ids-and-targets system
(exec-query target)
conn
"
SELECT DISTINCT system_id, target
FROM package_derivations
INNER JOIN guix_revision_package_derivations
ON package_derivations.id = guix_revision_package_derivations.package_derivation_id
WHERE revision_id = $1"
(list guix-revision-id)))
(define (get-count-for-next-level system-id target level-counts) (define (get-count-for-next-level system-id target level-counts)
(define next-level (define next-level
@ -154,17 +147,16 @@ VALUES ($1, $2, $3, $4, $5)"
(number->string level) (number->string level)
(number->string count)))) (number->string count))))
(for-each (define system-id
(match-lambda (number->string
((system-id target) (system->system-id conn system)))
(let loop ((level-counts '())) (let loop ((level-counts '()))
(let ((level (length level-counts)) (let ((level (length level-counts))
(count (get-count-for-next-level system-id target level-counts))) (count (get-count-for-next-level system-id target level-counts)))
(unless (= count 0) (unless (= count 0)
(insert-level-count system-id target level count) (insert-level-count system-id target level count)
(loop (append level-counts (list count)))))))) (loop (append level-counts (list count)))))))
system-ids-and-targets))
(define (backfill-guix-revision-package-derivation-distribution-counts conn) (define (backfill-guix-revision-package-derivation-distribution-counts conn)
(define revision-ids (define revision-ids
@ -183,12 +175,29 @@ ORDER BY id DESC")))
(for-each (for-each
(lambda (revision-id) (lambda (revision-id)
(simple-format #t "backfilling guix_revision_package_derivation_distribution_counts for revision ~A\n" revision-id) (simple-format #t "backfilling guix_revision_package_derivation_distribution_counts for revision ~A\n" revision-id)
(with-postgresql-transaction (let ((system-ids-and-targets
conn (exec-query
(lambda (conn) conn
(insert-guix-revision-package-derivation-distribution-counts "
conn SELECT DISTINCT system_id, target
revision-id)))) FROM package_derivations
INNER JOIN guix_revision_package_derivations
ON package_derivations.id = guix_revision_package_derivations.package_derivation_id
WHERE revision_id = $1"
(list revision-id))))
(with-postgresql-transaction
conn
(lambda (conn)
(for-each
(match-lambda
((system-id target)
(insert-guix-revision-package-derivation-distribution-counts
conn
revision-id
system-id
target)))
system-ids-and-targets)))))
revision-ids)) revision-ids))
(define* (get-sql-to-select-package-and-related-derivations-for-revision (define* (get-sql-to-select-package-and-related-derivations-for-revision