cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compute lint warnings for packages in chunks

Browse source 
In an attempt to reduce the peak memory usage, and avoid running in to the:

  Too many heap sections: Increase MAXHINCR or MAX_HEAP_SECTS

issue.
This commit is contained in:
Christopher Baines 2022-09-05 08:40:27 +01:00
parent 1043b51bff
commit aa8c9dbffa
1 changed files with 27 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

44
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -457,7 +457,7 @@ WHERE job_id = $1")
"vi_VN.UTF-8" "vi_VN.UTF-8"
"zh_CN.UTF-8")) "zh_CN.UTF-8"))
(define (lint-warnings-for-checker checker-name) (define (lint-warnings-for-checker packages checker-name)
`(lambda (store) `(lambda (store)
(let* ((checker (find (lambda (checker) (let* ((checker (find (lambda (checker)
(eq? (lint-checker-name checker) (eq? (lint-checker-name checker)
@ -590,25 +590,35 @@ WHERE job_id = $1")
'() '()
(with-time-logging (simple-format #f "getting ~A lint warnings" (with-time-logging (simple-format #f "getting ~A lint warnings"
name) name)
(format (current-error-port) (let loop ((packages-chunks
"inferior heap size: ~a MiB~%" (chunk packages 15000))
(round (warnings '()))
(/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) inf)
(expt 2. 20))))
(let ((warnings (if (null? packages-chunks)
(inferior-eval-with-store warnings
inf (let ((new-warnings
store (inferior-eval-with-store
(lint-warnings-for-checker name)))) inf
store
(lint-warnings-for-checker (car packages-chunks) name))))
;; Clean the cached store connections, as there are caches ;; Clean the cached store connections, as there are caches
;; associated with these that take up lots of memory ;; associated with these that take up lots of memory
(inferior-eval (inferior-eval
'(when (defined? '%store-table) (hash-clear! %store-table)) '(when (defined? '%store-table) (hash-clear! %store-table))
inf) inf)
warnings)))))) (inferior-eval '(gc) inf)
(format (current-error-port)
"inferior heap size: ~a MiB~%"
(round
(/ (inferior-eval '(assoc-ref (gc-stats) 'heap-size) inf)
(expt 2. 20))))
(loop (cdr packages-chunks)
(append! warnings
new-warnings))))))))))
checkers)))) checkers))))
(define (all-inferior-package-derivations store inf packages) (define (all-inferior-package-derivations store inf packages)