cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Store channel instance derivations in a separate transaction

Browse source 
This means that these derivations are stored, even if a later part of the
process fails. Having the channel instance derivations stored might help work
out why the failure occurred, or better display information about it.
This commit is contained in:
Christopher Baines 2021-02-02 23:36:56 +00:00
parent 6195d7b18f
commit 7fbcb3a3c2
8 changed files with 129 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

114
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -68,6 +68,7 @@
select-recent-job-events select-recent-job-events
select-unprocessed-jobs-and-events select-unprocessed-jobs-and-events
select-jobs-and-events-for-commit select-jobs-and-events-for-commit
guix-revision-loaded-successfully?
record-job-event record-job-event
enqueue-load-new-guix-revision-job enqueue-load-new-guix-revision-job
most-recent-n-load-new-guix-revision-jobs)) most-recent-n-load-new-guix-revision-jobs))
@ -1314,50 +1315,60 @@ WHERE job_id = $1"
(channel->derivations-by-system conn (channel->derivations-by-system conn
store store
channel-for-commit channel-for-commit
fetch-with-authentication?)) fetch-with-authentication?)))
(store-item (let ((guix-revision-id
(channel-derivations-by-system->guix-store-item (insert-guix-revision conn git-repository-id commit)))
store (insert-channel-instances conn
channel-derivations-by-system))) guix-revision-id
(if store-item (filter-map
(let ((guix-revision-id (match-lambda
(insert-guix-revision conn git-repository-id ((system . derivations)
commit store-item))) (and=>
(and (assoc-ref derivations
guix-revision-id 'manifest-entry-item)
(extract-information-from conn store (lambda (drv)
guix-revision-id (cons system drv)))))
commit store-item) channel-derivations-by-system))
(insert-channel-instances conn
guix-revision-id
(filter-map
(match-lambda
((system . derivations)
(and=>
(assoc-ref derivations
'manifest-entry-item)
(lambda (drv)
(cons system drv)))))
channel-derivations-by-system))
(if (defined? 'channel-news-for-commit
(resolve-module '(guix channels)))
(with-time-logging "inserting channel news entries"
(insert-channel-news-entries-for-guix-revision
conn
guix-revision-id
(channel-news-for-commit channel-for-commit commit)))
(begin
(simple-format #t "debug: importing channel news not supported\n")
#t))
(update-package-derivations-table conn (simple-format
git-repository-id (current-error-port)
guix-revision-id "guix-data-service: saving the channel instance derivations to the database\n")
commit)))
(begin ;; COMMIT so that the channel instances are saved to the database, then
(simple-format #t "Failed to generate store item for ~A\n" ;; start a new transaction for the rest of the processing.
commit) (exec-query conn "COMMIT")
#f)))) (exec-query conn "BEGIN")
(let ((store-item
(channel-derivations-by-system->guix-store-item
store
channel-derivations-by-system)))
(if store-item
(begin
(extract-information-from conn store
guix-revision-id
commit store-item)
(if (defined? 'channel-news-for-commit
(resolve-module '(guix channels)))
(with-time-logging "inserting channel news entries"
(insert-channel-news-entries-for-guix-revision
conn
guix-revision-id
(channel-news-for-commit channel-for-commit commit)))
(begin
(simple-format
#t "debug: importing channel news not supported\n")
#t))
(update-package-derivations-table conn
git-repository-id
guix-revision-id
commit))
(begin
(simple-format #t "Failed to generate store item for ~A\n"
commit)
#f))))))
(define (enqueue-load-new-guix-revision-job conn git-repository-id commit source) (define (enqueue-load-new-guix-revision-job conn git-repository-id commit source)
(define query (define query
@ -1606,6 +1617,23 @@ ORDER BY load_new_guix_revision_jobs.id DESC")
(string=? log-exists? "t")))) (string=? log-exists? "t"))))
(exec-query conn query (list commit)))) (exec-query conn query (list commit))))
(define (guix-revision-loaded-successfully? conn commit)
(define query
"
SELECT EXISTS(
SELECT 1
FROM load_new_guix_revision_jobs
INNER JOIN load_new_guix_revision_job_events
ON job_id = load_new_guix_revision_jobs.id
WHERE commit = $1
AND event = 'success'
)")
(let ((result (caar
(exec-query conn query (list commit)))))
(string=? result "t")))
(define (most-recent-n-load-new-guix-revision-jobs conn n) (define (most-recent-n-load-new-guix-revision-jobs conn n)
(let ((result (let ((result
(exec-query (exec-query

16
guix-data-service/model/channel-instance.scm
View file

@ -24,6 +24,7 @@
#:use-module (guix-data-service model utils) #:use-module (guix-data-service model utils)
#:use-module (guix-data-service model derivation) #:use-module (guix-data-service model derivation)
#:export (insert-channel-instances #:export (insert-channel-instances
channel-instances-exist-for-guix-revision?
select-channel-instances-for-guix-revision)) select-channel-instances-for-guix-revision))
(define (insert-channel-instances conn (define (insert-channel-instances conn
@ -52,6 +53,21 @@ VALUES "
", ")))) ", "))))
#t) #t)
(define (channel-instances-exist-for-guix-revision? conn commit-hash)
(define query
"
SELECT EXISTS(
SELECT 1
FROM channel_instances
INNER JOIN guix_revisions
ON guix_revisions.id = channel_instances.guix_revision_id
WHERE guix_revisions.commit = $1
)")
(let ((result (caar
(exec-query conn query (list commit-hash)))))
(string=? result "t")))
(define (select-channel-instances-for-guix-revision conn (define (select-channel-instances-for-guix-revision conn
commit-hash) commit-hash)
(define query (define query

13
guix-data-service/model/guix-revision.scm
View file

@ -46,16 +46,13 @@
id) id)
(() #f))) (() #f)))
(define (insert-guix-revision conn git-repository-id commit store_path) (define (insert-guix-revision conn git-repository-id commit)
(define insert (define insert
(string-append "INSERT INTO guix_revisions " "
"(git_repository_id, commit, store_path) VALUES " INSERT INTO guix_revisions (git_repository_id, commit)
"(" git-repository-id ", '" VALUES ($1, $2) RETURNING id")
commit "', '"
store_path "') "
"RETURNING id;"))
(match (exec-query conn insert) (match (exec-query conn insert (list git-repository-id commit))
(((id)) id))) (((id)) id)))
(define (guix-commit-exists? conn commit) (define (guix-commit-exists? conn commit)

30
guix-data-service/web/revision/controller.scm
View file

@ -105,7 +105,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(render-view-revision mime-types (render-view-revision mime-types
commit-hash commit-hash
#:path-base path) #:path-base path)
@ -115,7 +115,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(parse-query-parameters (parse-query-parameters
request request
@ -129,7 +129,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters (guard-against-mutually-exclusive-query-parameters
(parse-query-parameters (parse-query-parameters
@ -158,7 +158,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(render-revision-packages-translation-availability mime-types (render-revision-packages-translation-availability mime-types
commit-hash commit-hash
#:path-base path) #:path-base path)
@ -168,7 +168,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(render-revision-package mime-types (render-revision-package mime-types
commit-hash commit-hash
name) name)
@ -178,7 +178,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(parse-query-parameters (parse-query-parameters
request request
@ -194,7 +194,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters (guard-against-mutually-exclusive-query-parameters
(parse-query-parameters (parse-query-parameters
@ -224,7 +224,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters (guard-against-mutually-exclusive-query-parameters
(parse-query-parameters (parse-query-parameters
@ -250,7 +250,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters (guard-against-mutually-exclusive-query-parameters
(parse-query-parameters (parse-query-parameters
@ -281,7 +281,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(parse-query-parameters (parse-query-parameters
request request
@ -296,7 +296,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (channel-instances-exist-for-guix-revision? conn commit-hash))))
(render-revision-channel-instances mime-types (render-revision-channel-instances mime-types
commit-hash commit-hash
#:path-base path) #:path-base path)
@ -306,7 +306,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(render-revision-package-substitute-availability mime-types (render-revision-package-substitute-availability mime-types
commit-hash commit-hash
#:path-base path) #:path-base path)
@ -316,7 +316,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(render-revision-package-reproduciblity mime-types (render-revision-package-reproduciblity mime-types
commit-hash commit-hash
#:path-base path) #:path-base path)
@ -326,7 +326,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(guard-against-mutually-exclusive-query-parameters (guard-against-mutually-exclusive-query-parameters
(parse-query-parameters (parse-query-parameters
@ -351,7 +351,7 @@
(if (parallel-via-thread-pool-channel (if (parallel-via-thread-pool-channel
(with-thread-postgresql-connection (with-thread-postgresql-connection
(lambda (conn) (lambda (conn)
(guix-commit-exists? conn commit-hash)))) (guix-revision-loaded-successfully? conn commit-hash))))
(let ((parsed-query-parameters (let ((parsed-query-parameters
(parse-query-parameters (parse-query-parameters
request request

7
sqitch/deploy/remove_guix_revisions_store_path.sql Normal file
View file

@ -0,0 +1,7 @@
-- Deploy guix-data-service:remove_guix_revisions_store_path to pg
BEGIN;
ALTER TABLE guix_revisions DROP COLUMN store_path;
COMMIT;

7
sqitch/revert/remove_guix_revisions_store_path.sql Normal file
View file

@ -0,0 +1,7 @@
-- Revert guix-data-service:remove_guix_revisions_store_path from pg
BEGIN;
-- XXX Add DDLs here.
COMMIT;

1
sqitch/sqitch.plan
View file

@ -77,3 +77,4 @@ create_latest_build_status 2020-10-13T17:22:39Z Christopher Baines <mail@cbaines
regenerate_latest_build_status 2020-10-21T18:39:03Z Christopher Baines <mail@cbaines.net> # Regenerate the latest_build_status table regenerate_latest_build_status 2020-10-21T18:39:03Z Christopher Baines <mail@cbaines.net> # Regenerate the latest_build_status table
guix_revision_package_derivations_add_package_derivation_index 2020-10-27T16:58:08Z Christopher Baines <mail@cbaines.net> # Add index for guix_revision_package_derivations.package_derivation_id guix_revision_package_derivations_add_package_derivation_index 2020-10-27T16:58:08Z Christopher Baines <mail@cbaines.net> # Add index for guix_revision_package_derivations.package_derivation_id
increase_derivation_inputs_statistics_targets 2020-12-27T10:34:58Z Christopher Baines <mail@cbaines.net> # Increase stats targets on derivation_inputs fields increase_derivation_inputs_statistics_targets 2020-12-27T10:34:58Z Christopher Baines <mail@cbaines.net> # Increase stats targets on derivation_inputs fields
remove_guix_revisions_store_path 2021-02-02T20:06:18Z Christopher Baines <mail@cbaines.net> # Drop guix_revisions.store_path

7
sqitch/verify/remove_guix_revisions_store_path.sql Normal file
View file

@ -0,0 +1,7 @@
-- Verify guix-data-service:remove_guix_revisions_store_path on pg
BEGIN;
-- XXX Add verifications here.
ROLLBACK;