cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stop inserting missing source file nars

Browse source 
This was more an issue several years ago, so this code is not really needed
now.
This commit is contained in:
Christopher Baines 2024-07-16 16:06:46 +01:00
parent a61c4baccd
commit 1754d1a321
1 changed files with 63 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

137
guix-data-service/model/derivation.scm
View file

@ -1806,46 +1806,42 @@ WHERE " criteria ";"))
(exec-query conn (select-existing-derivations chunk))))
(chunk! missing-file-names 1000)))))
(define (derivation-file-names->derivation-ids conn derivation-file-names)
(define derivations-count
(vector-length derivation-file-names))
(define (insert-source-files-missing-nars derivation-ids)
(define (derivation-ids->next-related-derivation-ids! ids seen-ids)
(delete-duplicates/sort!
(append-map!
(lambda (ids-chunk)
(let ((query
(string-append
"
(define (insert-source-files-missing-nars conn derivation-ids)
(define (derivation-ids->next-related-derivation-ids! ids seen-ids)
(delete-duplicates/sort!
(append-map!
(lambda (ids-chunk)
(let ((query
(string-append
"
SELECT derivation_outputs.derivation_id
FROM derivation_inputs
INNER JOIN derivation_outputs
ON derivation_outputs.id = derivation_inputs.derivation_output_id
WHERE derivation_inputs.derivation_id IN ("
(string-join (map number->string ids) ",")
")")))
(string-join (map number->string ids) ",")
")")))
(filter-map
(lambda (row)
(let ((number
(string->number
(car row))))
(if (hash-ref seen-ids number)
#f
(begin
(hash-set! seen-ids number #t)
(filter-map
(lambda (row)
(let ((number
(string->number
(car row))))
(if (hash-ref seen-ids number)
#f
(begin
(hash-set! seen-ids number #t)
number))))
(exec-query conn query))))
(chunk! ids 500))
<
=))
number))))
(exec-query conn query))))
(chunk! ids 500))
<
=))
(define (derivation-ids->missing-sources ids)
(define query
(string-append
"
(define (derivation-ids->missing-sources ids)
(define query
(string-append
"
SELECT derivation_sources.derivation_source_file_id, derivation_source_files.store_path
FROM derivation_sources
LEFT JOIN derivation_source_file_nars
@ -1855,42 +1851,46 @@ INNER JOIN derivation_source_files
ON derivation_sources.derivation_source_file_id =
derivation_source_files.id
WHERE derivation_sources.derivation_id IN ("
(string-join (map number->string ids) ", ")
")
(string-join (map number->string ids) ", ")
")
AND derivation_source_file_nars.derivation_source_file_id IS NULL"))
(map (lambda (row)
(list (string->number (first row))
(second row)))
(exec-query conn query)))
(map (lambda (row)
(list (string->number (first row))
(second row)))
(exec-query conn query)))
(let ((seen-ids (make-hash-table)))
(let loop ((next-related-derivation-ids
(derivation-ids->next-related-derivation-ids!
(list-copy derivation-ids)
seen-ids)))
(unless (null? next-related-derivation-ids)
(let ((missing-sources
(append-map! derivation-ids->missing-sources
(chunk next-related-derivation-ids
10000))))
(let ((seen-ids (make-hash-table)))
(let loop ((next-related-derivation-ids
(derivation-ids->next-related-derivation-ids!
(list-copy derivation-ids)
seen-ids)))
(unless (null? next-related-derivation-ids)
(let ((missing-sources
(append-map! derivation-ids->missing-sources
(chunk next-related-derivation-ids
10000))))
(unless (null? missing-sources)
(with-time-logging
(simple-format #f "inserting ~A missing source files"
(length missing-sources))
(for-each (match-lambda
((derivation-source-file-id store-path)
(insert-derivation-source-file-nar
conn
derivation-source-file-id
store-path)))
missing-sources))))
(unless (null? missing-sources)
(with-time-logging
(simple-format #f "inserting ~A missing source files"
(length missing-sources))
(for-each (match-lambda
((derivation-source-file-id store-path)
(insert-derivation-source-file-nar
conn
derivation-source-file-id
store-path)))
missing-sources))))
(loop
(derivation-ids->next-related-derivation-ids!
next-related-derivation-ids
seen-ids))))))
(loop
(derivation-ids->next-related-derivation-ids!
next-related-derivation-ids
seen-ids))))))
(define (derivation-file-names->derivation-ids conn derivation-file-names)
(define derivations-count
(vector-length derivation-file-names))
(if (= 0 derivations-count)
#()
@ -1951,17 +1951,6 @@ INNER JOIN derivation_source_files
#f))
derivation-file-names)))
(with-time-logging "insert-source-files-missing-nars"
(insert-source-files-missing-nars
;; TODO Avoid this conversion
(vector-fold
(lambda (_ result x)
(if x
(cons x result)
result))
'()
all-ids)))
all-ids)))))
(define (update-derivation-inputs-statistics conn)