cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Speed up loading package metadata

Browse source 
By batching the SQL queries.
This commit is contained in:
Christopher Baines 2024-02-01 13:15:30 +01:00
parent 215a67238a
commit 0468bc4321
1 changed files with 40 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

64
guix-data-service/model/package-metadata.scm
View file

@ -292,18 +292,6 @@ WHERE packages.id IN (
(prevent-inlining-for-tests inferior-packages->translated-package-descriptions-and-synopsis) (prevent-inlining-for-tests inferior-packages->translated-package-descriptions-and-synopsis)
(define (package-synopsis-data->package-synopsis-ids
conn synopsis-by-locale)
(insert-missing-data-and-return-all-ids
conn
"package_synopsis"
'(locale synopsis)
(map (match-lambda
((locale . synopsis)
(list locale synopsis)))
synopsis-by-locale)
#:delete-duplicates? #t))
(define (insert-package-synopsis-set conn package-synopsis-ids) (define (insert-package-synopsis-set conn package-synopsis-ids)
(let ((query (let ((query
(string-append (string-append
@ -320,12 +308,8 @@ WHERE packages.id IN (
(((id)) id)))) (((id)) id))))
(define (package-synopsis-data->package-synopsis-set-id (define (package-synopsis-data->package-synopsis-set-id
conn synopsis-by-locale) conn package-synopsis-ids)
(let* ((package-synopsis-ids (let ((package-synopsis-set-id
(package-synopsis-data->package-synopsis-ids
conn
synopsis-by-locale))
(package-synopsis-set-id
(exec-query (exec-query
conn conn
(string-append (string-append
@ -340,18 +324,6 @@ WHERE packages.id IN (
(() (()
(insert-package-synopsis-set conn package-synopsis-ids)))))) (insert-package-synopsis-set conn package-synopsis-ids))))))
(define (package-description-data->package-description-ids
conn descriptions-by-locale)
(insert-missing-data-and-return-all-ids
conn
"package_descriptions"
'(locale description)
(map (match-lambda
((locale . description)
(list locale description)))
descriptions-by-locale)
#:delete-duplicates? #t))
(define (insert-package-description-set conn package-description-ids) (define (insert-package-description-set conn package-description-ids)
(let ((query (let ((query
(string-append (string-append
@ -419,19 +391,37 @@ WHERE packages.id IN (
conn conn
package-description-ids)) package-description-ids))
(with-time-logging "preparing package description ids" (with-time-logging "preparing package description ids"
(insert-missing-data-and-return-all-ids
conn
"package_descriptions"
'(locale description)
(map (match-lambda (map (match-lambda
((_ _ package-description-data _) ((_ _ package-description-data _)
(package-description-data->package-description-ids
conn
package-description-data)))
package-metadata))))
(with-time-logging "preparing package synopsis set ids"
(map (match-lambda (map (match-lambda
((_ _ _ package-synopsis-data) ((locale . description)
(list locale description)))
package-description-data)))
package-metadata)
#:delete-duplicates? #t
#:sets-of-data? #t))))
(with-time-logging "preparing package synopsis set ids"
(map (lambda (package-synopsis-ids)
(package-synopsis-data->package-synopsis-set-id (package-synopsis-data->package-synopsis-set-id
conn conn
package-synopsis-ids))
(insert-missing-data-and-return-all-ids
conn
"package_synopsis"
'(locale synopsis)
(map (match-lambda
((_ _ _ package-synopsis-data)
(map (match-lambda
((locale . synopsis)
(list locale synopsis)))
package-synopsis-data))) package-synopsis-data)))
package-metadata))) package-metadata)
#:delete-duplicates? #t
#:sets-of-data? #t))))
;; There can be duplicated entires in package-metadata, for example where ;; There can be duplicated entires in package-metadata, for example where
;; you have one package definition which interits from another, and just ;; you have one package definition which interits from another, and just
;; overrides the version and the source, the package_metadata entries for ;; overrides the version and the source, the package_metadata entries for