Christopher Baines
c1d2f3a1b7
Make parallel use of inferiors when computing channel instance derivations, and when extracting information about a revision. This should allow for some horizontal scalability, reducing the impact of additional systems for which derivations need computing. This commit also fixes an apparent issue with package replacements, as previously the wrong id was used, and this hid some issues around deduplication.
88 lines
3.3 KiB
Text
88 lines
3.3 KiB
Text
#!@GUILE@ --no-auto-compile
|
|
-*- scheme -*-
|
|
-*- geiser-scheme-implementation: guile -*-
|
|
!#
|
|
;;; Guix Data Service -- Information about Guix over time
|
|
;;; Copyright © 2016, 2017 Ricardo Wurmus <rekado@elephly.net>
|
|
;;; Copyright © 2018 Arun Isaac <arunisaac@systemreboot.net>
|
|
;;; Copyright © 2019 Christopher Baines <mail@cbaines.net>
|
|
;;;
|
|
;;; This file is part of guix-data-service.
|
|
;;;
|
|
;;; guix-data-service is free software; you can redistribute it and/or modify it
|
|
;;; under the terms of the GNU General Public License as published by
|
|
;;; the Free Software Foundation; either version 3 of the License, or
|
|
;;; (at your option) any later version.
|
|
;;;
|
|
;;; guix-data-service is distributed in the hope that it will be useful, but
|
|
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
;;; General Public License for more details.
|
|
;;;
|
|
;;; You should have received a copy of the GNU General Public License
|
|
;;; along with the guix-data-service. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
(use-modules (srfi srfi-1)
|
|
(srfi srfi-37)
|
|
(guix-data-service database)
|
|
(guix-data-service jobs))
|
|
|
|
(setvbuf (current-output-port) 'line)
|
|
(setvbuf (current-error-port) 'line)
|
|
|
|
(define %options
|
|
;; Specifications of the command-line options
|
|
(list (option '("max-processes") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'max-processes
|
|
(string->number arg)
|
|
result)))
|
|
(option '("latest-branch-revision-max-processes") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'latest-branch-revision-max-processes
|
|
(string->number arg)
|
|
result)))
|
|
(option '("skip-system-tests") #f #f
|
|
(lambda (opt name _ result)
|
|
(alist-cons 'skip-system-tests #t result)))
|
|
(option '("per-job-parallelism") #t #f
|
|
(lambda (opt name arg result)
|
|
(alist-cons 'per-job-parallelism
|
|
(string->number arg)
|
|
result)))))
|
|
|
|
(define %default-options
|
|
;; Alist of default option values
|
|
`((max-processes . ,default-max-processes)
|
|
(per-job-parallelism . 1)))
|
|
|
|
(define (parse-options args)
|
|
(args-fold
|
|
args %options
|
|
(lambda (opt name arg result)
|
|
(error "unrecognized option" name))
|
|
(lambda (arg result)
|
|
(alist-cons
|
|
'revision-commits
|
|
(cons arg
|
|
(or (assoc-ref result 'revision-commits)
|
|
'()))
|
|
(alist-delete 'revision-commits result)))
|
|
%default-options))
|
|
|
|
(let ((opts (parse-options (cdr (program-arguments)))))
|
|
(run-sqitch)
|
|
|
|
(with-postgresql-connection
|
|
"process-jobs"
|
|
(lambda (conn)
|
|
(simple-format #t "Ready to process jobs...\n")
|
|
(process-jobs conn
|
|
#:max-processes (assq-ref opts 'max-processes)
|
|
#:latest-branch-revision-max-processes
|
|
(or (assq-ref opts 'latest-branch-revision-max-processes)
|
|
(* 2 (assq-ref opts 'max-processes)))
|
|
#:skip-system-tests?
|
|
(assq-ref opts 'skip-system-tests)
|
|
#:per-job-parallelism
|
|
(assq-ref opts 'per-job-parallelism)))))
|