cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stop inserting missing source file nars

Browse source 
This was more an issue several years ago, so this code is not really needed
now.
This commit is contained in:
Christopher Baines 2024-07-16 16:06:46 +01:00
parent a61c4baccd
commit 1754d1a321
1 changed files with 63 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

137
guix-data-service/model/derivation.scm
View file

@ -1806,46 +1806,42 @@ WHERE " criteria ";"))
(exec-query conn (select-existing-derivations chunk)))) (exec-query conn (select-existing-derivations chunk))))
(chunk! missing-file-names 1000))))) (chunk! missing-file-names 1000)))))
(define (derivation-file-names->derivation-ids conn derivation-file-names) (define (insert-source-files-missing-nars conn derivation-ids)
(define derivations-count (define (derivation-ids->next-related-derivation-ids! ids seen-ids)
(vector-length derivation-file-names)) (delete-duplicates/sort!
(append-map!
(define (insert-source-files-missing-nars derivation-ids) (lambda (ids-chunk)
(define (derivation-ids->next-related-derivation-ids! ids seen-ids) (let ((query
(delete-duplicates/sort! (string-append
(append-map! "
(lambda (ids-chunk)
(let ((query
(string-append
"
SELECT derivation_outputs.derivation_id SELECT derivation_outputs.derivation_id
FROM derivation_inputs FROM derivation_inputs
INNER JOIN derivation_outputs INNER JOIN derivation_outputs
ON derivation_outputs.id = derivation_inputs.derivation_output_id ON derivation_outputs.id = derivation_inputs.derivation_output_id
WHERE derivation_inputs.derivation_id IN (" WHERE derivation_inputs.derivation_id IN ("
(string-join (map number->string ids) ",") (string-join (map number->string ids) ",")
")"))) ")")))
(filter-map (filter-map
(lambda (row) (lambda (row)
(let ((number (let ((number
(string->number (string->number
(car row)))) (car row))))
(if (hash-ref seen-ids number) (if (hash-ref seen-ids number)
#f #f
(begin (begin
(hash-set! seen-ids number #t) (hash-set! seen-ids number #t)
number)))) number))))
(exec-query conn query)))) (exec-query conn query))))
(chunk! ids 500)) (chunk! ids 500))
< <
=)) =))
(define (derivation-ids->missing-sources ids) (define (derivation-ids->missing-sources ids)
(define query (define query
(string-append (string-append
" "
SELECT derivation_sources.derivation_source_file_id, derivation_source_files.store_path SELECT derivation_sources.derivation_source_file_id, derivation_source_files.store_path
FROM derivation_sources FROM derivation_sources
LEFT JOIN derivation_source_file_nars LEFT JOIN derivation_source_file_nars
@ -1855,42 +1851,46 @@ INNER JOIN derivation_source_files
ON derivation_sources.derivation_source_file_id = ON derivation_sources.derivation_source_file_id =
derivation_source_files.id derivation_source_files.id
WHERE derivation_sources.derivation_id IN (" WHERE derivation_sources.derivation_id IN ("
(string-join (map number->string ids) ", ") (string-join (map number->string ids) ", ")
") ")
AND derivation_source_file_nars.derivation_source_file_id IS NULL")) AND derivation_source_file_nars.derivation_source_file_id IS NULL"))
(map (lambda (row) (map (lambda (row)
(list (string->number (first row)) (list (string->number (first row))
(second row))) (second row)))
(exec-query conn query))) (exec-query conn query)))
(let ((seen-ids (make-hash-table))) (let ((seen-ids (make-hash-table)))
(let loop ((next-related-derivation-ids (let loop ((next-related-derivation-ids
(derivation-ids->next-related-derivation-ids! (derivation-ids->next-related-derivation-ids!
(list-copy derivation-ids) (list-copy derivation-ids)
seen-ids))) seen-ids)))
(unless (null? next-related-derivation-ids) (unless (null? next-related-derivation-ids)
(let ((missing-sources (let ((missing-sources
(append-map! derivation-ids->missing-sources (append-map! derivation-ids->missing-sources
(chunk next-related-derivation-ids (chunk next-related-derivation-ids
10000)))) 10000))))
(unless (null? missing-sources) (unless (null? missing-sources)
(with-time-logging (with-time-logging
(simple-format #f "inserting ~A missing source files" (simple-format #f "inserting ~A missing source files"
(length missing-sources)) (length missing-sources))
(for-each (match-lambda (for-each (match-lambda
((derivation-source-file-id store-path) ((derivation-source-file-id store-path)
(insert-derivation-source-file-nar (insert-derivation-source-file-nar
conn conn
derivation-source-file-id derivation-source-file-id
store-path))) store-path)))
missing-sources)))) missing-sources))))
(loop (loop
(derivation-ids->next-related-derivation-ids! (derivation-ids->next-related-derivation-ids!
next-related-derivation-ids next-related-derivation-ids
seen-ids)))))) seen-ids))))))
(define (derivation-file-names->derivation-ids conn derivation-file-names)
(define derivations-count
(vector-length derivation-file-names))
(if (= 0 derivations-count) (if (= 0 derivations-count)
#() #()
@ -1951,17 +1951,6 @@ INNER JOIN derivation_source_files
#f)) #f))
derivation-file-names))) derivation-file-names)))
(with-time-logging "insert-source-files-missing-nars"
(insert-source-files-missing-nars
;; TODO Avoid this conversion
(vector-fold
(lambda (_ result x)
(if x
(cons x result)
result))
'()
all-ids)))
all-ids))))) all-ids)))))
(define (update-derivation-inputs-statistics conn) (define (update-derivation-inputs-statistics conn)