cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move backfilling in to the server module and use the connection pool

Browse source 
To avoid using the old PostgreSQL connection per thread code.
This commit is contained in:
Christopher Baines 2024-04-01 21:51:29 +01:00
parent 5f5b8fc91d
commit b5f59189e1
3 changed files with 19 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

29
guix-data-service/model/guix-revision-package-derivation.scm
View file

@ -166,34 +166,29 @@ VALUES ($1, $2, $3, $4, $5)"
(loop (append level-counts (list count)))))))) (loop (append level-counts (list count))))))))
system-ids-and-targets)) system-ids-and-targets))
(define (backfill-guix-revision-package-derivation-distribution-counts) (define (backfill-guix-revision-package-derivation-distribution-counts conn)
(define revision-ids (define revision-ids
(with-thread-postgresql-connection (map
(lambda (conn) car
(map (exec-query
car conn
(exec-query "
conn
"
SELECT id SELECT id
FROM guix_revisions FROM guix_revisions
EXCEPT EXCEPT
SELECT guix_revision_id SELECT guix_revision_id
FROM guix_revision_package_derivation_distribution_counts FROM guix_revision_package_derivation_distribution_counts
ORDER BY id DESC"))))) ORDER BY id DESC")))
(n-par-for-each (for-each
4
(lambda (revision-id) (lambda (revision-id)
(simple-format #t "backfilling guix_revision_package_derivation_distribution_counts for revision ~A\n" revision-id) (simple-format #t "backfilling guix_revision_package_derivation_distribution_counts for revision ~A\n" revision-id)
(with-thread-postgresql-connection (with-postgresql-transaction
conn
(lambda (conn) (lambda (conn)
(with-postgresql-transaction (insert-guix-revision-package-derivation-distribution-counts
conn conn
(lambda (conn) revision-id))))
(insert-guix-revision-package-derivation-distribution-counts
conn
revision-id))))))
revision-ids)) revision-ids))
(define* (get-sql-to-select-package-and-related-derivations-for-revision (define* (get-sql-to-select-package-and-related-derivations-for-revision

7
guix-data-service/web/server.scm
View file

@ -35,6 +35,7 @@
#:use-module (guix-data-service database) #:use-module (guix-data-service database)
#:use-module (guix-data-service web controller) #:use-module (guix-data-service web controller)
#:use-module (guix-data-service web util) #:use-module (guix-data-service web util)
#:use-module (guix-data-service model guix-revision-package-derivation)
#:export (%guix-data-service-metrics-registry #:export (%guix-data-service-metrics-registry
start-guix-data-service-web-server)) start-guix-data-service-web-server))
@ -120,6 +121,12 @@
(resource-pool-default-timeout 5)) (resource-pool-default-timeout 5))
(spawn-fiber
(lambda ()
(with-resource-from-pool (connection-pool) conn
(backfill-guix-revision-package-derivation-distribution-counts
conn))))
(let ((render-metrics (let ((render-metrics
(make-render-metrics registry))) (make-render-metrics registry)))

7
scripts/guix-data-service.in
View file

@ -197,13 +197,6 @@
(lambda (port) (lambda (port)
(simple-format port "~A\n" (getpid))))) (simple-format port "~A\n" (getpid)))))
(call-with-new-thread
(lambda ()
(with-postgresql-connection-per-thread
"backfill"
(lambda ()
(backfill-guix-revision-package-derivation-distribution-counts)))))
(start-substitute-query-threads) (start-substitute-query-threads)
(call-with-new-thread (call-with-new-thread