cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tweak fetching narinfos

Browse source 
Currently I'm seeing failures due to guile-gnutls not supporting suspendable
ports (write_wait_fd), so batch the requested outputs to try and avoid this.
This commit is contained in:
Christopher Baines 2023-04-28 20:42:31 +02:00
parent 688f4cd79d
commit 8a19bcba13
1 changed files with 30 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

54
guix-data-service/substitutes.scm
View file

@ -22,6 +22,7 @@
#:use-module (ice-9 threads)
#:use-module (guix substitutes)
#:use-module (guix narinfo)
#:use-module (guix-data-service utils)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model build-server)
#:use-module (guix-data-service model git-branch)
@ -85,34 +86,39 @@
(simple-format #t "Querying ~A outputs\n"
(length outputs))
(let ((narinfos
(lookup-narinfos (string-trim-right build-server-url #\/) outputs)))
(chunk-for-each!
(lambda (outputs-chunk)
(let ((narinfos
(lookup-narinfos (string-trim-right build-server-url #\/)
outputs-chunk)))
(simple-format #t "Got ~A narinfo files\n"
(length narinfos))
(simple-format #t "Got ~A narinfo files\n"
(length narinfos))
(let ((filtered-narinfos
(filter-map
(lambda (narinfo)
(if (> (narinfo-size narinfo)
%narinfo-max-size)
(begin
(simple-format (current-error-port)
"narinfo ~A has excessive size ~A\n"
(narinfo-path narinfo)
(narinfo-size narinfo))
#f)
narinfo))
narinfos)))
(let ((filtered-narinfos
(filter-map
(lambda (narinfo)
(if (> (narinfo-size narinfo)
%narinfo-max-size)
(begin
(simple-format (current-error-port)
"narinfo ~A has excessive size ~A\n"
(narinfo-path narinfo)
(narinfo-size narinfo))
#f)
narinfo))
narinfos)))
(unless (null? filtered-narinfos)
(with-postgresql-transaction
conn
(lambda (conn)
(record-narinfo-details-and-return-ids
(unless (null? filtered-narinfos)
(with-postgresql-transaction
conn
build-server-id
filtered-narinfos)))))))
(lambda (conn)
(record-narinfo-details-and-return-ids
conn
build-server-id
filtered-narinfos)))))))
2000
outputs))
(define (start-substitute-query-thread)
(call-with-new-thread