cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Avoid loading in all derivations

Browse source 
Just load in the ones that are not already in the database, as this
will hopefully be faster and require less memory.
This commit is contained in:
Christopher Baines 2019-03-14 08:18:01 +00:00
parent a0dd298239
commit d2697d8dd2
Signed by: cbaines
GPG key ID: 5E28A33B0B84F577
2 changed files with 14 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

7
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -82,12 +82,9 @@
(close-inferior inf) (close-inferior inf)
(let ((derivation-ids (let ((derivation-ids
(derivations->derivation-ids (derivation-file-names->derivation-ids
conn conn
(map (lambda (tuple) (map fourth inferior-data-4-tuples)))
(read-derivation-from-file
(fourth tuple)))
inferior-data-4-tuples)))
(flat-package-ids-systems-and-targets (flat-package-ids-systems-and-targets
(map (map
(match-lambda (match-lambda

24
guix-data-service/model/derivation.scm
View file

@ -18,7 +18,7 @@
select-derivations-by-id select-derivations-by-id
select-derivations-and-build-status-by-file-name select-derivations-and-build-status-by-file-name
insert-into-derivations insert-into-derivations
derivations->derivation-ids)) derivation-file-names->derivation-ids))
(define (select-existing-derivations file-names) (define (select-existing-derivations file-names)
(string-append "SELECT id, file_name " (string-append "SELECT id, file_name "
@ -558,28 +558,26 @@ ORDER BY derivations.system DESC,
result result
result-for-missing-file-names))))) result-for-missing-file-names)))))
(define (derivations->derivation-ids conn derivations) (define (derivation-file-names->derivation-ids conn derivation-file-names)
(if (null? derivations) (if (null? derivation-file-names)
'() '()
(let* ((derivations-count (length derivations)) (let* ((derivations-count (length derivation-file-names))
(derivation-ids-hash-table (make-hash-table derivations-count))) (derivation-ids-hash-table (make-hash-table derivations-count)))
(simple-format (simple-format
#t "debug: derivations->derivation-ids: processing ~A derivations\n" #t "debug: derivation-file-names->derivation-ids: processing ~A derivations\n"
derivations-count) derivations-count)
(let* ((derivation-file-names (map derivation-file-name (let* ((existing-derivation-entries
derivations))
(existing-derivation-entries
(derivation-file-names->vhash conn (derivation-file-names->vhash conn
derivation-ids-hash-table derivation-ids-hash-table
derivation-file-names)) derivation-file-names))
(missing-derivations (missing-derivations
(deduplicate-derivations (map read-derivation-from-file
(filter (lambda (derivation) (deduplicate-strings
(not (vhash-assoc (derivation-file-name derivation) (filter (lambda (derivation-file-name)
(not (vhash-assoc derivation-file-name
existing-derivation-entries))) existing-derivation-entries)))
derivations))) derivation-file-names))))
(new-derivation-entries (new-derivation-entries
(if (null? missing-derivations) (if (null? missing-derivations)