Christopher Baines
f66ff9a3ff
This is the big change needed to allow for parallel revision processing. Previously, a lock was used to prevent this since the parallel transactions could deadlock if each inserted data that the other then went to insert. By defining the order in which inserts happen, both in terms of the order of tables, and the order of rows within the table, this change should guarantee that there won't be deadlocks. I'm also hoping this change will address whatever issue was causing some derivation data to be missing from the database.
132 lines
4.8 KiB
Text
132 lines
4.8 KiB
Text
#!@GUILE@ --no-auto-compile
|
|
-*- scheme -*-
|
|
-*- geiser-scheme-implementation: guile -*-
|
|
!#
|
|
;;; Guix Data Service -- Information about Guix over time
|
|
;;; Copyright © 2016, 2017 Ricardo Wurmus <rekado@elephly.net>
|
|
;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net>
|
|
;;; Copyright © 2019 Christopher Baines <mail@cbaines.net>
|
|
;;;
|
|
;;; This file is part of guix-data-service.
|
|
;;;
|
|
;;; guix-data-service is free software; you can redistribute it and/or modify it
|
|
;;; under the terms of the GNU General Public License as published by
|
|
;;; the Free Software Foundation; either version 3 of the License, or
|
|
;;; (at your option) any later version.
|
|
;;;
|
|
;;; guix-data-service is distributed in the hope that it will be useful, but
|
|
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
;;; General Public License for more details.
|
|
;;;
|
|
;;; You should have received a copy of the GNU General Public License
|
|
;;; along with the guix-data-service. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
(use-modules (srfi srfi-1)
|
|
(srfi srfi-37)
|
|
(ice-9 match)
|
|
(ice-9 suspendable-ports)
|
|
(fibers)
|
|
(guix-data-service database)
|
|
(guix-data-service data-deletion)
|
|
(guix-data-service model package-derivation-by-guix-revision-range)
|
|
(guix-data-service jobs load-new-guix-revision))
|
|
|
|
(setvbuf (current-output-port) 'line)
|
|
(setvbuf (current-error-port) 'line)
|
|
|
|
(setlocale LC_ALL "en_US.UTF-8")
|
|
|
|
;; Make stack traces more useful
|
|
(setenv "COLUMNS" "256")
|
|
|
|
(install-suspendable-ports!)
|
|
|
|
(define %options
|
|
(list (option '("skip-system-tests") #f #f
|
|
(lambda (opt name _ result)
|
|
(alist-cons 'skip-system-tests #t result)))
|
|
(option '("parallelism") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'parallelism
|
|
(string->number arg)
|
|
(alist-delete 'parallelism
|
|
result))))
|
|
(option '("inferior-memory-limit") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'inferior-memory-limit
|
|
(string->number arg)
|
|
(alist-delete 'inferior-memory-limit
|
|
result))))
|
|
(option '("inferior-set-environment-variable") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'inferior-environment-variable
|
|
(string-split arg #\=)
|
|
result)))
|
|
(option '("ignore-systems") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'ignore-systems
|
|
(string-split arg #\,)
|
|
result)))
|
|
(option '("ignore-targets") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'ignore-targets
|
|
(string-split arg #\,)
|
|
result)))))
|
|
|
|
(define %default-options
|
|
'((parallelism . 1)
|
|
(ignore-systems . ())
|
|
(ignore-targets . ())))
|
|
|
|
(define (parse-options args)
|
|
(args-fold
|
|
args %options
|
|
(lambda (opt name arg result)
|
|
(error "unrecognized option" name))
|
|
(lambda (arg result)
|
|
(alist-cons
|
|
'arguments
|
|
(cons arg
|
|
(or (assoc-ref result 'arguments)
|
|
'()))
|
|
(alist-delete 'arguments result)))
|
|
%default-options))
|
|
|
|
(let ((opts (parse-options (cdr (program-arguments)))))
|
|
(match (assq-ref opts 'arguments)
|
|
((job)
|
|
(with-exception-handler
|
|
(lambda (exn)
|
|
(simple-format #t "failed enabling core dumps: ~A\n" exn))
|
|
(lambda ()
|
|
(setrlimit 'core #f #f))
|
|
#:unwind? #t)
|
|
|
|
(fluid-set! %file-port-name-canonicalization 'none)
|
|
|
|
(run-fibers
|
|
(lambda ()
|
|
(with-exception-handler
|
|
(lambda (exn)
|
|
;; Exit if exceptions get this far, as not all fibers are
|
|
;; guaranteed to finish
|
|
(primitive-exit 1))
|
|
(lambda ()
|
|
(process-load-new-guix-revision-job
|
|
job
|
|
#:skip-system-tests? (assq-ref opts 'skip-system-tests)
|
|
#:extra-inferior-environment-variables
|
|
(filter-map
|
|
(match-lambda
|
|
(('inferior-environment-variable key val)
|
|
(cons key val))
|
|
(_ #f))
|
|
opts)
|
|
#:ignore-systems (assq-ref opts 'ignore-systems)
|
|
#:ignore-targets (assq-ref opts 'ignore-targets)
|
|
#:inferior-memory-limit (assq-ref opts 'inferior-memory-limit)
|
|
#:parallelism (assq-ref opts 'parallelism)))
|
|
#:unwind? #t))
|
|
#:hz 0
|
|
#:parallelism (assq-ref opts 'parallelism)))))
|