cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use a more long lived store connection for loading data

Browse source 
As this will enable registering temporary roots, to avoid store items being
garbage collected.
This commit is contained in:
Christopher Baines 2020-02-13 19:38:20 +00:00
parent 32052c45b3
commit 0ef3349ed8
2 changed files with 212 additions and 214 deletions
Download patch file Download diff file Expand all files Collapse all files

410
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -715,7 +715,7 @@ WHERE job_id = $1"
(build-derivations store (list derivation)))) (build-derivations store (list derivation))))
(derivation->output-path derivation))) (derivation->output-path derivation)))
(define (channel->derivation-file-names-by-system channel) (define (channel->derivation-file-names-by-system store channel)
(define use-container? (defined? (define use-container? (defined?
'open-inferior/container 'open-inferior/container
(resolve-module '(guix inferior)))) (resolve-module '(guix inferior))))
@ -780,9 +780,6 @@ WHERE job_id = $1"
#f)))))))) #f))))))))
(list ,@systems))))) (list ,@systems)))))
(with-store store
(set-build-options store #:fallback? #t)
(let ((inferior (let ((inferior
(if use-container? (if use-container?
(open-inferior/container (open-inferior/container
@ -854,9 +851,9 @@ WHERE job_id = $1"
key parameters)))) key parameters))))
(lambda args (lambda args
(close-inferior inferior) (close-inferior inferior)
#f))))) #f))))
(define (channel->derivations-by-system conn channel) (define (channel->derivations-by-system conn store channel)
(let* ((derivation-file-names-by-system (let* ((derivation-file-names-by-system
(log-time (log-time
"computing the channel derivation" "computing the channel derivation"
@ -867,7 +864,7 @@ WHERE job_id = $1"
conn conn
'channel->manifest-store-item 'channel->manifest-store-item
(lambda () (lambda ()
(channel->derivation-file-names-by-system channel))))))) (channel->derivation-file-names-by-system store channel)))))))
(for-each (for-each
(match-lambda (match-lambda
((system . derivation-file-name) ((system . derivation-file-name)
@ -880,6 +877,7 @@ WHERE job_id = $1"
derivation-file-names-by-system)) derivation-file-names-by-system))
(define (channel-derivations-by-system->guix-store-item (define (channel-derivations-by-system->guix-store-item
store
channel-derivations-by-system) channel-derivations-by-system)
(define (store-item->guix-store-item filename) (define (store-item->guix-store-item filename)
@ -895,13 +893,10 @@ WHERE job_id = $1"
(if derivation-file-name-for-current-system (if derivation-file-name-for-current-system
(let ((derivation-for-current-system (let ((derivation-for-current-system
(read-derivation-from-file derivation-file-name-for-current-system))) (read-derivation-from-file derivation-file-name-for-current-system)))
(with-store store (log-time
(set-build-options store #:fallback? #t) "building the channel derivation"
(lambda ()
(log-time (build-derivations store (list derivation-for-current-system))))
"building the channel derivation"
(lambda ()
(build-derivations store (list derivation-for-current-system)))))
(store-item->guix-store-item (store-item->guix-store-item
(derivation->output-path derivation-for-current-system))) (derivation->output-path derivation-for-current-system)))
@ -939,35 +934,30 @@ WHERE job_id = $1"
output))) output)))
(define (extract-information-from conn guix-revision-id commit store-path) (define (extract-information-from conn store guix-revision-id commit store-path)
(simple-format (simple-format #t "debug: extract-information-from: ~A\n" store-path)
#t "debug: extract-information-from: ~A\n" store-path) (let* ((guix-locpath (getenv "GUIX_LOCPATH"))
(with-store store (inf (let ((guix-locpath
(set-build-options store ;; Augment the GUIX_LOCPATH to include glibc-locales from
#:fallback? #t) ;; the Guix at store-path, this should mean that the
;; inferior Guix works, even if it's build using a different
(let* ((guix-locpath (getenv "GUIX_LOCPATH")) ;; glibc version
(inf (let ((guix-locpath (string-append
;; Augment the GUIX_LOCPATH to include glibc-locales from (glibc-locales-for-guix-store-path store store-path)
;; the Guix at store-path, this should mean that the "/lib/locale"
;; inferior Guix works, even if it's build using a different ":" guix-locpath)))
;; glibc version ;; Unset the GUILE_LOAD_PATH and GUILE_LOAD_COMPILED_PATH to
(string-append ;; avoid the values for these being used in the
(glibc-locales-for-guix-store-path store store-path) ;; inferior. Even though the inferior %load-path and
"/lib/locale" ;; %load-compiled-path has the inferior modules first, this
":" guix-locpath))) ;; can cause issues when there are modules present outside
;; Unset the GUILE_LOAD_PATH and GUILE_LOAD_COMPILED_PATH to ;; of the inferior Guix which aren't present in the inferior
;; avoid the values for these being used in the ;; Guix (like the new (guix lint) module
;; inferior. Even though the inferior %load-path and (unsetenv "GUILE_LOAD_PATH")
;; %load-compiled-path has the inferior modules first, this (unsetenv "GUILE_LOAD_COMPILED_PATH")
;; can cause issues when there are modules present outside (simple-format (current-error-port) "debug: set GUIX_LOCPATH to ~A\n"
;; of the inferior Guix which aren't present in the inferior guix-locpath)
;; Guix (like the new (guix lint) module (if (defined?
(unsetenv "GUILE_LOAD_PATH")
(unsetenv "GUILE_LOAD_COMPILED_PATH")
(simple-format (current-error-port) "debug: set GUIX_LOCPATH to ~A\n"
guix-locpath)
(if (defined?
'open-inferior/container 'open-inferior/container
(resolve-module '(guix inferior))) (resolve-module '(guix inferior)))
(open-inferior/container store store-path (open-inferior/container store store-path
@ -982,143 +972,143 @@ WHERE job_id = $1"
(simple-format #t "debug: using open-inferior\n") (simple-format #t "debug: using open-inferior\n")
(open-inferior store-path (open-inferior store-path
#:error-port (real-error-port))))))) #:error-port (real-error-port)))))))
(setenv "GUIX_LOCPATH" guix-locpath) ; restore GUIX_LOCPATH (setenv "GUIX_LOCPATH" guix-locpath) ; restore GUIX_LOCPATH
(when (eq? inf #f) (when (eq? inf #f)
(error "error: inferior is #f")) (error "error: inferior is #f"))
;; Normalise the locale for the inferior process ;; Normalise the locale for the inferior process
(catch (catch
'system-error 'system-error
(lambda () (lambda ()
(inferior-eval '(setlocale LC_ALL "en_US.utf8") inf)) (inferior-eval '(setlocale LC_ALL "en_US.utf8") inf))
(lambda (key . args) (lambda (key . args)
(simple-format (current-error-port) (simple-format (current-error-port)
"warning: failed to set locale to en_US.utf8: ~A ~A\n" "warning: failed to set locale to en_US.utf8: ~A ~A\n"
key args) key args)
(display "trying to setlocale to en_US.UTF-8 instead\n" (display "trying to setlocale to en_US.UTF-8 instead\n"
(current-error-port)) (current-error-port))
(with-exception-handler (with-exception-handler
(lambda (key . args) (lambda (key . args)
(simple-format (simple-format
(current-error-port) (current-error-port)
"warning: failed to set locale to en_US.UTF-8: ~A ~A\n" "warning: failed to set locale to en_US.UTF-8: ~A ~A\n"
key args)) key args))
(lambda () (lambda ()
(inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf))))) (inferior-eval '(setlocale LC_ALL "en_US.UTF-8") inf)))))
(inferior-eval '(use-modules (srfi srfi-1) (inferior-eval '(use-modules (srfi srfi-1)
(srfi srfi-34) (srfi srfi-34)
(guix grafts) (guix grafts)
(guix derivations) (guix derivations)
(gnu tests)) (gnu tests))
inf) inf)
(inferior-eval '(when (defined? '%graft?) (%graft? #f)) inf) (inferior-eval '(when (defined? '%graft?) (%graft? #f)) inf)
(catch (catch
#t #t
(lambda () (lambda ()
(let* ((packages (let* ((packages
(log-time (log-time
"fetching inferior packages" "fetching inferior packages"
(lambda () (lambda ()
(deduplicate-inferior-packages (deduplicate-inferior-packages
(inferior-packages inf))))) (inferior-packages inf)))))
(inferior-lint-warnings (inferior-lint-warnings
(log-time (log-time
"fetching inferior lint warnings" "fetching inferior lint warnings"
(lambda () (lambda ()
(all-inferior-lint-warnings inf store)))) (all-inferior-lint-warnings inf store))))
(inferior-data-4-tuples (inferior-data-4-tuples
(log-time (log-time
"getting inferior derivations" "getting inferior derivations"
(lambda () (lambda ()
(all-inferior-package-derivations store inf packages)))) (all-inferior-package-derivations store inf packages))))
(inferior-system-tests (inferior-system-tests
(log-time (log-time
"getting inferior system tests" "getting inferior system tests"
(lambda () (lambda ()
(all-inferior-system-tests inf store))))) (all-inferior-system-tests inf store)))))
(log-time (log-time
"acquiring advisory transaction lock: load-new-guix-revision-inserts" "acquiring advisory transaction lock: load-new-guix-revision-inserts"
(lambda () (lambda ()
;; Wait until this is the only transaction inserting data, to ;; Wait until this is the only transaction inserting data, to
;; avoid any concurrency issues ;; avoid any concurrency issues
(obtain-advisory-transaction-lock conn (obtain-advisory-transaction-lock conn
'load-new-guix-revision-inserts))) 'load-new-guix-revision-inserts)))
(let* ((package-ids (let* ((package-ids
(insert-packages conn inf packages)) (insert-packages conn inf packages))
(inferior-package-id->package-database-id (inferior-package-id->package-database-id
(let ((lookup-table (let ((lookup-table
(alist->hashq-table (alist->hashq-table
(map (lambda (package package-id) (map (lambda (package package-id)
(cons (inferior-package-id package) (cons (inferior-package-id package)
package-id)) package-id))
packages packages
package-ids)))) package-ids))))
(lambda (inferior-id) (lambda (inferior-id)
(or (or
(hashq-ref lookup-table inferior-id) (hashq-ref lookup-table inferior-id)
(error (error
(simple-format (simple-format
#f #f
"error: inferior-package-id->package-database-id: ~A missing\n" "error: inferior-package-id->package-database-id: ~A missing\n"
inferior-id))))))) inferior-id)))))))
(simple-format
#t "debug: finished loading information from inferior\n")
(close-inferior inf)
(when inferior-lint-warnings
(let* ((lint-checker-ids
(lint-checkers->lint-checker-ids
conn
(map car inferior-lint-warnings)))
(lint-warning-ids
(insert-lint-warnings
conn
inferior-package-id->package-database-id
lint-checker-ids
inferior-lint-warnings)))
(insert-guix-revision-lint-checkers conn
guix-revision-id
lint-checker-ids)
(insert-guix-revision-lint-warnings conn
guix-revision-id
lint-warning-ids)))
(insert-system-tests-for-guix-revision conn
guix-revision-id
inferior-system-tests)
(let ((package-derivation-ids
(log-time
"inferior-data->package-derivation-ids"
(lambda ()
(inferior-data->package-derivation-ids
conn inf inferior-package-id->package-database-id
inferior-data-4-tuples)))))
(update-builds-derivation-output-details-set-id
conn
(map fourth inferior-data-4-tuples))
(insert-guix-revision-package-derivations conn
guix-revision-id
package-derivation-ids)
(simple-format (simple-format
#t "debug: finished loading information from inferior\n") #t "Successfully loaded ~A package/derivation pairs\n"
(close-inferior inf) (length package-derivation-ids)))))
#t)
(when inferior-lint-warnings (lambda (key . args)
(let* ((lint-checker-ids (simple-format (current-error-port)
(lint-checkers->lint-checker-ids "Failed extracting information from commit: ~A\n\n" commit)
conn (simple-format (current-error-port)
(map car inferior-lint-warnings))) " ~A ~A\n\n" key args)
(lint-warning-ids #f)
(insert-lint-warnings (lambda (key . args)
conn (display-backtrace (make-stack #t) (current-error-port))))))
inferior-package-id->package-database-id
lint-checker-ids
inferior-lint-warnings)))
(insert-guix-revision-lint-checkers conn
guix-revision-id
lint-checker-ids)
(insert-guix-revision-lint-warnings conn
guix-revision-id
lint-warning-ids)))
(insert-system-tests-for-guix-revision conn
guix-revision-id
inferior-system-tests)
(let ((package-derivation-ids
(log-time
"inferior-data->package-derivation-ids"
(lambda ()
(inferior-data->package-derivation-ids
conn inf inferior-package-id->package-database-id
inferior-data-4-tuples)))))
(update-builds-derivation-output-details-set-id
conn
(map fourth inferior-data-4-tuples))
(insert-guix-revision-package-derivations conn
guix-revision-id
package-derivation-ids)
(simple-format
#t "Successfully loaded ~A package/derivation pairs\n"
(length package-derivation-ids)))))
#t)
(lambda (key . args)
(simple-format (current-error-port)
"Failed extracting information from commit: ~A\n\n" commit)
(simple-format (current-error-port)
" ~A ~A\n\n" key args)
#f)
(lambda (key . args)
(display-backtrace (make-stack #t) (current-error-port)))))))
(define (update-package-versions-table conn git-repository-id commit) (define (update-package-versions-table conn git-repository-id commit)
(log-time (log-time
@ -1186,7 +1176,7 @@ ORDER BY packages.name, packages.version"
#t) #t)
(define (load-new-guix-revision conn git-repository-id commit) (define (load-new-guix-revision conn store git-repository-id commit)
(let* ((channel-for-commit (let* ((channel-for-commit
(channel (name 'guix) (channel (name 'guix)
(url (git-repository-id->url (url (git-repository-id->url
@ -1195,9 +1185,11 @@ ORDER BY packages.name, packages.version"
(commit commit))) (commit commit)))
(channel-derivations-by-system (channel-derivations-by-system
(channel->derivations-by-system conn (channel->derivations-by-system conn
store
channel-for-commit)) channel-for-commit))
(store-item (store-item
(channel-derivations-by-system->guix-store-item (channel-derivations-by-system->guix-store-item
store
channel-derivations-by-system))) channel-derivations-by-system)))
(if store-item (if store-item
(let ((guix-revision-id (let ((guix-revision-id
@ -1205,7 +1197,8 @@ ORDER BY packages.name, packages.version"
commit store-item))) commit store-item)))
(and (and
guix-revision-id guix-revision-id
(extract-information-from conn guix-revision-id (extract-information-from conn store
guix-revision-id
commit store-item) commit store-item)
(insert-channel-instances conn (insert-channel-instances conn
guix-revision-id guix-revision-id
@ -1524,44 +1517,49 @@ SKIP LOCKED")
(simple-format #t "Processing job ~A (commit: ~A, source: ~A)\n\n" (simple-format #t "Processing job ~A (commit: ~A, source: ~A)\n\n"
id commit source) id commit source)
(if (or (guix-revision-exists? conn git-repository-id commit) (if (or
(eq? (log-time (guix-revision-exists? conn git-repository-id commit)
(string-append "loading revision " commit) (eq?
(lambda () (log-time
(let* ((previous-output-port (current-output-port)) (string-append "loading revision " commit)
(previous-error-port (current-error-port)) (lambda ()
(result (let* ((previous-output-port (current-output-port))
(with-postgresql-connection (previous-error-port (current-error-port))
(simple-format #f "load-new-guix-revision ~A logging" id) (result
(lambda (logging-conn) (with-postgresql-connection
(insert-empty-log-entry logging-conn id) (simple-format #f "load-new-guix-revision ~A logging" id)
(let ((logging-port (log-port id logging-conn))) (lambda (logging-conn)
(set-current-output-port logging-port) (insert-empty-log-entry logging-conn id)
(set-current-error-port logging-port) (let ((logging-port (log-port id logging-conn)))
(let ((result (set-current-output-port logging-port)
(parameterize ((current-build-output-port logging-port) (set-current-error-port logging-port)
(real-error-port previous-error-port)) (let ((result
(catch #t (parameterize ((current-build-output-port logging-port)
(lambda () (real-error-port previous-error-port))
(load-new-guix-revision conn (catch #t
git-repository-id (lambda ()
commit)) (with-store store
(lambda (key . args) (set-build-options store #:fallback? #t)
(simple-format (load-new-guix-revision conn
(current-error-port) store
"error: load-new-guix-revision: ~A ~A\n" git-repository-id
key args) commit)))
#f))))) (lambda (key . args)
(combine-log-parts! logging-conn id) (simple-format
(current-error-port)
"error: load-new-guix-revision: ~A ~A\n"
key args)
#f)))))
(combine-log-parts! logging-conn id)
;; This can happen with GC, so do it explicitly ;; This can happen with GC, so do it explicitly
(close-port logging-port) (close-port logging-port)
result)))))) result))))))
(set-current-output-port previous-output-port) (set-current-output-port previous-output-port)
(set-current-error-port previous-error-port) (set-current-error-port previous-error-port)
result))) result)))
#t)) #t))
(begin (begin
(record-job-succeeded conn id) (record-job-succeeded conn id)
(record-job-event conn id "success") (record-job-event conn id "success")

16
tests/jobs-load-new-guix-revision.scm
View file

@ -25,7 +25,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'((x86_64-linux '((x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -34,13 +34,13 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
"/gnu/store/test")) "/gnu/store/test"))
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
extract-information-from extract-information-from
(lambda (conn guix-revision-id commit store-path) (lambda (conn store guix-revision-id commit store-path)
#t)) #t))
(mock (mock
@ -71,7 +71,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'(x86_64-linux '(x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -80,7 +80,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
#f)) #f))
(match (enqueue-load-new-guix-revision-job (match (enqueue-load-new-guix-revision-job
@ -98,7 +98,7 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel->derivations-by-system channel->derivations-by-system
(lambda (conn channel) (lambda (conn store channel)
'(x86_64-linux '(x86_64-linux
. .
((manifest-entry-item . /gnu/store/foo.drv) ((manifest-entry-item . /gnu/store/foo.drv)
@ -107,13 +107,13 @@
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
channel-derivations-by-system->guix-store-item channel-derivations-by-system->guix-store-item
(lambda (channel-derivations-by-system) (lambda (store channel-derivations-by-system)
"/gnu/store/test")) "/gnu/store/test"))
(mock (mock
((guix-data-service jobs load-new-guix-revision) ((guix-data-service jobs load-new-guix-revision)
extract-information-from extract-information-from
(lambda (conn git-repository-id commit store-path) (lambda (conn store git-repository-id commit store-path)
#f)) #f))
(mock (mock