cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use chunk-for-each!

Browse source 
As this might use less memory than chunk!
This commit is contained in:
Christopher Baines 2025-03-10 19:28:30 +00:00
parent 2eb5714829
commit 9fa059af61
1 changed files with 16 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

28
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -994,7 +994,7 @@
derivations-count (length missing-file-names)) derivations-count (length missing-file-names))
(unless (null? missing-file-names) (unless (null? missing-file-names)
(for-each (chunk-for-each!
(lambda (chunk) (lambda (chunk)
(for-each (for-each
(match-lambda (match-lambda
@ -1003,7 +1003,8 @@
file-name file-name
(string->number id)))) (string->number id))))
(exec-query conn (select-existing-derivations chunk)))) (exec-query conn (select-existing-derivations chunk))))
(chunk! missing-file-names 1000))))) 1000
missing-file-names))))
(define* (derivations-insert-sources postgresql-connection-pool (define* (derivations-insert-sources postgresql-connection-pool
call-with-utility-thread call-with-utility-thread
@ -1208,7 +1209,7 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
derivations))) derivations)))
(unless (null? input-derivations) (unless (null? input-derivations)
;; Ensure all the input derivations exist ;; Ensure all the input derivations exist
(for-each (chunk-for-each!
(lambda (chunk) (lambda (chunk)
(insert-missing-derivations (insert-missing-derivations
postgresql-connection-pool postgresql-connection-pool
@ -1216,7 +1217,8 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
derivation-ids-hash-table derivation-ids-hash-table
chunk chunk
#:log-tag log-tag)) #:log-tag log-tag))
(chunk! input-derivations 1000)))))) 1000
input-derivations)))))
(simple-format (simple-format
(current-error-port) (current-error-port)
@ -1285,14 +1287,15 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(derivation-inputs drv)))) (derivation-inputs drv))))
(unless (null? input-derivations) (unless (null? input-derivations)
;; Ensure all the input derivations exist ;; Ensure all the input derivations exist
(for-each (chunk-for-each!
(lambda (chunk) (lambda (chunk)
(insert-missing-derivations (insert-missing-derivations
postgresql-connection-pool postgresql-connection-pool
call-with-utility-thread call-with-utility-thread
derivation-ids-hash-table derivation-ids-hash-table
chunk)) chunk))
(chunk! input-derivations 1000)))))) 1000
input-derivations 1000)))))
(fix-derivation-inputs conn drv)))))))) (fix-derivation-inputs conn drv))))))))
#:hz 0 #:hz 0
@ -1333,23 +1336,24 @@ SELECT 1 FROM derivation_source_file_nars WHERE derivation_source_file_id = $1"
(length missing-derivation-filenames) (length missing-derivation-filenames)
log-tag) log-tag)
(let ((chunks (chunk! missing-derivation-filenames 1000))) (let ((chunk-counter 0))
(for-each (chunk-for-each!
(lambda (i missing-derivation-file-names-chunk) (lambda (missing-derivation-file-names-chunk)
(let ((missing-derivations-chunk (let ((missing-derivations-chunk
(read-derivations/fiberized (read-derivations/fiberized
missing-derivation-file-names-chunk))) missing-derivation-file-names-chunk)))
(simple-format (simple-format
#t "debug: derivation-file-names->derivation-ids: processing chunk ~A (~A)\n" #t "debug: derivation-file-names->derivation-ids: processing chunk ~A (~A)\n"
i chunk-counter
log-tag) log-tag)
(set! chunk-counter (+ 1 chunk-counter))
(insert-missing-derivations postgresql-connection-pool (insert-missing-derivations postgresql-connection-pool
call-with-utility-thread call-with-utility-thread
derivation-ids-hash-table derivation-ids-hash-table
missing-derivations-chunk missing-derivations-chunk
#:log-tag log-tag))) #:log-tag log-tag)))
(iota (length chunks)) 1000
chunks)) missing-derivation-filenames))
(let ((all-ids (let ((all-ids
(vector-map (vector-map