Further tweak fetching narinfos

Move the batching to the database, which should reduce memory usage while
removing the limit on the number of fetched narinfos.

guix-data-service/model/nar.scm

@@ -381,7 +381,9 @@ ORDER BY COUNT(*) DESC")
           build-server-id
           guix-revision-commits
           #:key
-          build-success-after)
+          build-success-after
+          after-id
+          (limit 2000))
   (define query
     (string-append
      "
@@ -440,16 +442,25 @@ WHERE derivation_output_details.path NOT IN (
                        ",")
           ")
   )"))
+     (if after-id
+         (string-append
           "
-ORDER BY derivation_output_details.id DESC
-LIMIT 100000"))
+  AND derivation_output_details.id > " after-id)
+         "")
+     "
+ORDER BY derivation_output_details.id ASC"
+     (if limit
+         (string-append
+          "
+LIMIT " (number->string limit))
+         "")))
 
-  (map car (exec-query conn
+  (exec-query conn
               query
               `(,(number->string build-server-id)
                 ,@(if build-success-after
                       (list (date->string build-success-after "~1 ~3"))
-                               '())))))
+                      '()))))
 
 (define (select-nars-for-output conn output-file-name)
   (define query

guix-data-service/substitutes.scm

@@ -50,7 +50,6 @@
               (simple-format #t "\nQuerying ~A\n" url)
               (catch #t
                 (lambda ()
-                  (simple-format #t "\nFetching narinfo files\n")
                   (fetch-narinfo-files conn id url revision-commits
                                        #:specific-outputs
                                        outputs))
@@ -69,7 +68,10 @@
 (define* (fetch-narinfo-files conn build-server-id build-server-url
                               revision-commits
                               #:key specific-outputs)
-  (define outputs
+  (let loop ((last-id #f)
+             (requests 0)
+             (success-responses 0))
+    (let ((outputs-chunk
            (or specific-outputs
                (select-outputs-without-known-nar-entries
                 conn
@@ -81,19 +83,25 @@
                      (subtract-duration (current-time)
                                         (make-time time-duration 0 (* 60 5)))
                      0)                        ; tz-offset
-             #f))))
+                    #f)
+                #:after-id last-id))))
 
-  (simple-format #t "Querying ~A outputs\n"
-                 (length outputs))
-
-  (chunk-for-each!
-   (lambda (outputs-chunk)
-     (let ((narinfos
+      (unless (null? outputs-chunk)
+        (let* ((narinfos
                 (lookup-narinfos (string-trim-right build-server-url #\/)
-                             outputs-chunk)))
-
-       (simple-format #t "Got ~A narinfo files\n"
+                                 (map car outputs-chunk)))
+               (narinfos-count
                 (length narinfos))
+               (total-requested
+                (+ requests (length outputs-chunk)))
+               (total-narinfos
+                (+ success-responses narinfos-count)))
+
+          (simple-format #t "Fetched ~A narinfos from ~A (total requested: ~A, total narinfos: ~A)\n"
+                         (length narinfos)
+                         build-server-url
+                         total-requested
+                         total-narinfos)
 
           (let ((filtered-narinfos
                  (filter-map
@@ -116,9 +124,11 @@
                  (record-narinfo-details-and-return-ids
                   conn
                   build-server-id
-               filtered-narinfos)))))))
-   2000
-   outputs))
+                  filtered-narinfos)))))
+
+          (loop (second (last outputs-chunk))
+                total-requested
+                total-narinfos))))))
 
 (define (start-substitute-query-thread)
   (call-with-new-thread