cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add meaningful parallelism to processing jobs

Browse source 
Make parallel use of inferiors when computing channel instance derivations,
and when extracting information about a revision. This should allow for some
horizontal scalability, reducing the impact of additional systems for which
derivations need computing.

This commit also fixes an apparent issue with package replacements, as
previously the wrong id was used, and this hid some issues around
deduplication.
This commit is contained in:
Christopher Baines 2024-01-18 14:44:04 +00:00
parent 6842a432d6
commit c1d2f3a1b7
4 changed files with 482 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

5
scripts/guix-data-service-process-job.in
View file

@ -78,7 +78,8 @@
(lambda ()
(process-load-new-guix-revision-job
job
#:skip-system-tests? (assq-ref opts 'skip-system-tests)))
#:skip-system-tests? (assq-ref opts 'skip-system-tests)
#:parallelism (assq-ref opts 'parallelism)))
#:hz 0
#:parallelism (assq-ref opts 'parallelism)
#:parallelism 1
#:drain? #t)))))

14
scripts/guix-data-service-process-jobs.in
View file

@ -44,11 +44,17 @@
result)))
(option '("skip-system-tests") #f #f
(lambda (opt name _ result)
(alist-cons 'skip-system-tests #t result)))))
(alist-cons 'skip-system-tests #t result)))
(option '("per-job-parallelism") #t #f
(lambda (opt name arg result)
(alist-cons 'per-job-parallelism
(string->number arg)
result)))))
(define %default-options
;; Alist of default option values
`((max-processes . ,default-max-processes)))
`((max-processes . ,default-max-processes)
(per-job-parallelism . 1)))
(define (parse-options args)
(args-fold
@ -77,4 +83,6 @@
(or (assq-ref opts 'latest-branch-revision-max-processes)
(* 2 (assq-ref opts 'max-processes)))
#:skip-system-tests?
(assq-ref opts 'skip-system-tests)))))
(assq-ref opts 'skip-system-tests)
#:per-job-parallelism
(assq-ref opts 'per-job-parallelism)))))