cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Avoid loading in all derivations

Browse source 
Just load in the ones that are not already in the database, as this
will hopefully be faster and require less memory.
This commit is contained in:
Christopher Baines 2019-03-14 08:18:01 +00:00
parent a0dd298239
commit d2697d8dd2
Signed by: cbaines
GPG key ID: 5E28A33B0B84F577
2 changed files with 14 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

7
guix-data-service/jobs/load-new-guix-revision.scm
View file

@ -82,12 +82,9 @@
(close-inferior inf)
(let ((derivation-ids
(derivations->derivation-ids
(derivation-file-names->derivation-ids
conn
(map (lambda (tuple)
(read-derivation-from-file
(fourth tuple)))
inferior-data-4-tuples)))
(map fourth inferior-data-4-tuples)))
(flat-package-ids-systems-and-targets
(map
(match-lambda

26
guix-data-service/model/derivation.scm
View file

@ -18,7 +18,7 @@
select-derivations-by-id
select-derivations-and-build-status-by-file-name
insert-into-derivations
derivations->derivation-ids))
derivation-file-names->derivation-ids))
(define (select-existing-derivations file-names)
(string-append "SELECT id, file_name "
@ -558,28 +558,26 @@ ORDER BY derivations.system DESC,
result
result-for-missing-file-names)))))
(define (derivations->derivation-ids conn derivations)
(if (null? derivations)
(define (derivation-file-names->derivation-ids conn derivation-file-names)
(if (null? derivation-file-names)
'()
(let* ((derivations-count (length derivations))
(let* ((derivations-count (length derivation-file-names))
(derivation-ids-hash-table (make-hash-table derivations-count)))
(simple-format
#t "debug: derivations->derivation-ids: processing ~A derivations\n"
#t "debug: derivation-file-names->derivation-ids: processing ~A derivations\n"
derivations-count)
(let* ((derivation-file-names (map derivation-file-name
derivations))
(existing-derivation-entries
(let* ((existing-derivation-entries
(derivation-file-names->vhash conn
derivation-ids-hash-table
derivation-file-names))
(missing-derivations
(deduplicate-derivations
(filter (lambda (derivation)
(not (vhash-assoc (derivation-file-name derivation)
existing-derivation-entries)))
derivations)))
(map read-derivation-from-file
(deduplicate-strings
(filter (lambda (derivation-file-name)
(not (vhash-assoc derivation-file-name
existing-derivation-entries)))
derivation-file-names))))
(new-derivation-entries
(if (null? missing-derivations)