cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use a more long lived store connection for loading data

Browse source 
As this will enable registering temporary roots, to avoid store items being
garbage collected.
This commit is contained in:
Christopher Baines 2020-02-13 19:38:20 +00:00
parent 32052c45b3
commit 0ef3349ed8
2 changed files with 212 additions and 214 deletions
Download patch file Download diff file Expand all files Collapse all files

46
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -715,7 +715,7 @@ WHERE job_id = $1"
(build-derivations store (list derivation)))) (build-derivations store (list derivation))))
(derivation->output-path derivation))) (derivation->output-path derivation)))
(define (channel->derivation-file-names-by-system channel) (define (channel->derivation-file-names-by-system store channel)
(define use-container? (defined? (define use-container? (defined?
'open-inferior/container 'open-inferior/container
(resolve-module '(guix inferior)))) (resolve-module '(guix inferior))))
@ -780,9 +780,6 @@ WHERE job_id = $1"
#f)))))))) #f))))))))
(list ,@systems))))) (list ,@systems)))))
(with-store store
(set-build-options store #:fallback? #t)
(let ((inferior (let ((inferior
(if use-container? (if use-container?
(open-inferior/container (open-inferior/container
@ -854,9 +851,9 @@ WHERE job_id = $1"
key parameters)))) key parameters))))
(lambda args (lambda args
(close-inferior inferior) (close-inferior inferior)
#f))))) #f))))
(define (channel->derivations-by-system conn channel) (define (channel->derivations-by-system conn store channel)
(let* ((derivation-file-names-by-system (let* ((derivation-file-names-by-system
(log-time (log-time
"computing the channel derivation" "computing the channel derivation"
@ -867,7 +864,7 @@ WHERE job_id = $1"
conn conn
'channel->manifest-store-item 'channel->manifest-store-item
(lambda () (lambda ()
(channel->derivation-file-names-by-system channel))))))) (channel->derivation-file-names-by-system store channel)))))))
(for-each (for-each
(match-lambda (match-lambda
((system . derivation-file-name) ((system . derivation-file-name)
@ -880,6 +877,7 @@ WHERE job_id = $1"
derivation-file-names-by-system)) derivation-file-names-by-system))
(define (channel-derivations-by-system->guix-store-item (define (channel-derivations-by-system->guix-store-item
store
channel-derivations-by-system) channel-derivations-by-system)
(define (store-item->guix-store-item filename) (define (store-item->guix-store-item filename)
@ -895,13 +893,10 @@ WHERE job_id = $1"
(if derivation-file-name-for-current-system (if derivation-file-name-for-current-system
(let ((derivation-for-current-system (let ((derivation-for-current-system
(read-derivation-from-file derivation-file-name-for-current-system))) (read-derivation-from-file derivation-file-name-for-current-system)))
(with-store store
(set-build-options store #:fallback? #t)
(log-time (log-time
"building the channel derivation" "building the channel derivation"
(lambda () (lambda ()
(build-derivations store (list derivation-for-current-system))))) (build-derivations store (list derivation-for-current-system))))
(store-item->guix-store-item (store-item->guix-store-item
(derivation->output-path derivation-for-current-system))) (derivation->output-path derivation-for-current-system)))
@ -939,13 +934,8 @@ WHERE job_id = $1"
output))) output)))
(define (extract-information-from conn guix-revision-id commit store-path) (define (extract-information-from conn store guix-revision-id commit store-path)
(simple-format (simple-format #t "debug: extract-information-from: ~A\n" store-path)
#t "debug: extract-information-from: ~A\n" store-path)
(with-store store
(set-build-options store
#:fallback? #t)
(let* ((guix-locpath (getenv "GUIX_LOCPATH")) (let* ((guix-locpath (getenv "GUIX_LOCPATH"))
(inf (let ((guix-locpath (inf (let ((guix-locpath
;; Augment the GUIX_LOCPATH to include glibc-locales from ;; Augment the GUIX_LOCPATH to include glibc-locales from
@ -1118,7 +1108,7 @@ WHERE job_id = $1"
" ~A ~A\n\n" key args) " ~A ~A\n\n" key args)
#f) #f)
(lambda (key . args) (lambda (key . args)
(display-backtrace (make-stack #t) (current-error-port))))))) (display-backtrace (make-stack #t) (current-error-port))))))
(define (update-package-versions-table conn git-repository-id commit) (define (update-package-versions-table conn git-repository-id commit)
(log-time (log-time
@ -1186,7 +1176,7 @@ ORDER BY packages.name, packages.version"
#t) #t)
(define (load-new-guix-revision conn git-repository-id commit) (define (load-new-guix-revision conn store git-repository-id commit)
(let* ((channel-for-commit (let* ((channel-for-commit
(channel (name 'guix) (channel (name 'guix)
(url (git-repository-id->url (url (git-repository-id->url
@ -1195,9 +1185,11 @@ ORDER BY packages.name, packages.version"
(commit commit))) (commit commit)))
(channel-derivations-by-system (channel-derivations-by-system
(channel->derivations-by-system conn (channel->derivations-by-system conn
store
channel-for-commit)) channel-for-commit))
(store-item (store-item
(channel-derivations-by-system->guix-store-item (channel-derivations-by-system->guix-store-item
store
channel-derivations-by-system))) channel-derivations-by-system)))
(if store-item (if store-item
(let ((guix-revision-id (let ((guix-revision-id
@ -1205,7 +1197,8 @@ ORDER BY packages.name, packages.version"
commit store-item))) commit store-item)))
(and (and
guix-revision-id guix-revision-id
(extract-information-from conn guix-revision-id (extract-information-from conn store
guix-revision-id
commit store-item) commit store-item)
(insert-channel-instances conn (insert-channel-instances conn
guix-revision-id guix-revision-id
@ -1524,8 +1517,10 @@ SKIP LOCKED")
(simple-format #t "Processing job ~A (commit: ~A, source: ~A)\n\n" (simple-format #t "Processing job ~A (commit: ~A, source: ~A)\n\n"
id commit source) id commit source)
(if (or (guix-revision-exists? conn git-repository-id commit) (if (or
(eq? (log-time (guix-revision-exists? conn git-repository-id commit)
(eq?
(log-time
(string-append "loading revision " commit) (string-append "loading revision " commit)
(lambda () (lambda ()
(let* ((previous-output-port (current-output-port)) (let* ((previous-output-port (current-output-port))
@ -1543,9 +1538,12 @@ SKIP LOCKED")
(real-error-port previous-error-port)) (real-error-port previous-error-port))
(catch #t (catch #t
(lambda () (lambda ()
(with-store store
(set-build-options store #:fallback? #t)
(load-new-guix-revision conn (load-new-guix-revision conn
store
git-repository-id git-repository-id
commit)) commit)))
(lambda (key . args) (lambda (key . args)
(simple-format (simple-format
(current-error-port) (current-error-port)

16
tests/jobs-load-new-guix-revision.scm
View file

@ -25,7 +25,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'((x86_64-linux '((x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -34,13 +34,13 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
"/gnu/store/test")) "/gnu/store/test"))
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
extract-information-from extract-information-from
(lambda (conn guix-revision-id commit store-path) (lambda (conn store guix-revision-id commit store-path)
#t)) #t))
(mock (mock
@ -71,7 +71,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'(x86_64-linux '(x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -80,7 +80,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
#f)) #f))
(match (enqueue-load-new-guix-revision-job (match (enqueue-load-new-guix-revision-job
@ -98,7 +98,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'(x86_64-linux '(x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -107,13 +107,13 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
"/gnu/store/test")) "/gnu/store/test"))
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
extract-information-from extract-information-from
(lambda (conn git-repository-id commit store-path) (lambda (conn store git-repository-id commit store-path)
#f)) #f))
(mock (mock