cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use chunk-for-each!

Browse source 
As this might use less memory than chunk!
This commit is contained in:
Christopher Baines 2025-03-10 19:28:30 +00:00
parent 2eb5714829
commit 9fa059af61
1 changed files with 16 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

28
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -994,7 +994,7 @@
derivations-count (length missing-file-names))
(unless (null? missing-file-names)
(for-each
(chunk-for-each!
(lambda (chunk)
(for-each
(match-lambda
@ -1003,7 +1003,8 @@
file-name
(string->number id))))
(exec-query conn (select-existing-derivations chunk))))
(chunk! missing-file-names 1000)))))
1000
missing-file-names))))
(define* (derivations-insert-sources postgresql-connection-pool
call-with-utility-thread
@ -1208,7 +1209,7 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
derivations)))
(unless (null? input-derivations)
;; Ensure all the input derivations exist
(for-each
(chunk-for-each!
(lambda (chunk)
(insert-missing-derivations
postgresql-connection-pool
@ -1216,7 +1217,8 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
derivation-ids-hash-table
chunk
#:log-tag log-tag))
(chunk! input-derivations 1000))))))
1000
input-derivations)))))
(simple-format
(current-error-port)
@ -1285,14 +1287,15 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(derivation-inputs drv))))
(unless (null? input-derivations)
;; Ensure all the input derivations exist
(for-each
(chunk-for-each!
(lambda (chunk)
(insert-missing-derivations
postgresql-connection-pool
call-with-utility-thread
derivation-ids-hash-table
chunk))
(chunk! input-derivations 1000))))))
1000
input-derivations 1000)))))
(fix-derivation-inputs conn drv))))))))
#:hz 0
@ -1333,23 +1336,24 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(length missing-derivation-filenames)
log-tag)
(let ((chunks (chunk! missing-derivation-filenames 1000)))
(for-each
(lambda (i missing-derivation-file-names-chunk)
(let ((chunk-counter 0))
(chunk-for-each!
(lambda (missing-derivation-file-names-chunk)
(let ((missing-derivations-chunk
(read-derivations/fiberized
missing-derivation-file-names-chunk)))
(simple-format
#t "debug: derivation-file-names->derivation-ids: processing chunk ~A (~A)\n"
i
chunk-counter
log-tag)
(set! chunk-counter (+ 1 chunk-counter))
(insert-missing-derivations postgresql-connection-pool
call-with-utility-thread
derivation-ids-hash-table
missing-derivations-chunk
#:log-tag log-tag)))
(iota (length chunks))
chunks))
1000
missing-derivation-filenames))
(let ((all-ids
(vector-map