From f7f4e70d28dda011e55013f9a0dbe48f59595f75 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Wed, 25 Jun 2025 12:46:37 +0200 Subject: [PATCH] Use with-postgresql-transaction always in load-new-guix-revision --- .../jobs/load-new-guix-revision.scm | 169 +++++++++--------- 1 file changed, 85 insertions(+), 84 deletions(-) diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index 370e003..6663a95 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -3044,99 +3044,100 @@ SKIP LOCKED") (define finished-channel (make-channel)) + (define job-not-found-exception + (make-exception-with-message + (simple-format #f "job ~A not found to be processed\n" + id))) + (define result - (parameterize - ;; Mimic the behaviour of with-postgresql-transaction - ((%postgresql-in-transaction? #t)) - (with-postgresql-connection - (simple-format #f "load-new-guix-revision ~A" id) - (lambda (conn) - ;; Fix the hash encoding of derivation_output_details. This'll only run - ;; once on any given database, but is kept here just to make sure any - ;; instances have the data updated. - (fix-derivation-output-details-hash-encoding conn) + (with-postgresql-connection + (simple-format #f "load-new-guix-revision ~A" id) + (lambda (conn) + ;; Fix the hash encoding of derivation_output_details. This'll only run + ;; once on any given database, but is kept here just to make sure any + ;; instances have the data updated. + (fix-derivation-output-details-hash-encoding conn) - (add-hook! after-gc-hook - (lambda () - (simple-format (current-error-port) - "after gc\n"))) + (add-hook! after-gc-hook + (lambda () + (simple-format (current-error-port) + "after gc\n"))) - (exec-query conn "BEGIN") + (with-exception-handler + (lambda (exn) + (unless (eq? job-not-found-exception) + ;; Use a new connection in case the outer conn is unusable + (with-postgresql-connection + (simple-format #f "load-new-guix-revision ~A record failure" id) + (lambda (conn) + (record-job-event conn id "failure")))) + #f) + (lambda () + (with-exception-handler + (lambda (exn) + (simple-format (current-error-port) + "error: load-new-guix-revision: ~A\n" + exn) + (print-backtrace-and-exception/knots exn) + (raise-exception exn)) + (lambda () + (with-postgresql-transaction + conn + (lambda (conn) + (spawn-fiber + (lambda () + (while (perform-operation + (choice-operation + (wrap-operation (get-operation finished-channel) + (const #f)) + (wrap-operation (sleep-operation 20) + (const #t)))) - (spawn-fiber - (lambda () - (while (perform-operation - (choice-operation - (wrap-operation (get-operation finished-channel) - (const #f)) - (wrap-operation (sleep-operation 20) - (const #t)))) + (let ((stats (gc-stats))) + (simple-format + (current-error-port) + "process-job heap: ~a MiB used (~a MiB heap)~%" + (round + (/ (- (assoc-ref stats 'heap-size) + (assoc-ref stats 'heap-free-size)) + (expt 2. 20))) + (round + (/ (assoc-ref stats 'heap-size) + (expt 2. 20)))))))) - (let ((stats (gc-stats))) - (simple-format - (current-error-port) - "process-job heap: ~a MiB used (~a MiB heap)~%" - (round - (/ (- (assoc-ref stats 'heap-size) - (assoc-ref stats 'heap-free-size)) - (expt 2. 20))) - (round - (/ (assoc-ref stats 'heap-size) - (expt 2. 20)))))))) + (match (select-job-for-update conn id) + (((id commit source git-repository-id)) - (match (select-job-for-update conn id) - (((id commit source git-repository-id)) + ;; With a separate connection, outside of the transaction so the event + ;; gets persisted regardless. + (with-postgresql-connection + (simple-format #f "load-new-guix-revision ~A start-event" id) + (lambda (start-event-conn) + (record-job-event start-event-conn id "start"))) - ;; With a separate connection, outside of the transaction so the event - ;; gets persisted regardless. - (with-postgresql-connection - (simple-format #f "load-new-guix-revision ~A start-event" id) - (lambda (start-event-conn) - (record-job-event start-event-conn id "start"))) + (simple-format + #t "Processing job ~A (commit: ~A, source: ~A)\n\n" + id commit source) - (simple-format #t "Processing job ~A (commit: ~A, source: ~A)\n\n" - id commit source) + (with-time-logging + (string-append "processing revision " commit) + (load-new-guix-revision + conn + git-repository-id + commit + #:skip-system-tests? skip-system-tests? + #:extra-inferior-environment-variables + extra-inferior-environment-variables + #:ignore-systems ignore-systems + #:ignore-targets ignore-targets + #:parallelism parallelism)) - (if (eq? - (with-time-logging (string-append "processing revision " commit) - (with-exception-handler - (const #f) - (lambda () - (with-exception-handler - (lambda (exn) - (simple-format (current-error-port) - "error: load-new-guix-revision: ~A\n" - exn) - (print-backtrace-and-exception/knots exn) - (raise-exception exn)) - (lambda () - (load-new-guix-revision - conn - git-repository-id - commit - #:skip-system-tests? skip-system-tests? - #:extra-inferior-environment-variables - extra-inferior-environment-variables - #:ignore-systems ignore-systems - #:ignore-targets ignore-targets - #:parallelism parallelism)))) - #:unwind? #t)) - #t) - (begin - (record-job-succeeded conn id) - (record-job-event conn id "success") - (exec-query conn "COMMIT") - - #t) - (begin - (exec-query conn "ROLLBACK") - (record-job-event conn id "failure") - - #f))) - (() - (exec-query conn "ROLLBACK") - (simple-format #t "job ~A not found to be processed\n" - id))))))) + (record-job-succeeded conn id) + (record-job-event conn id "success")) + (() + (raise-exception + job-not-found-exception)))))))) + #:unwind? #t)))) (when result (fibers-parallel