cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Speed up loading package metadata

Browse source 
By batching the SQL queries.
This commit is contained in:
Christopher Baines 2024-02-01 13:15:30 +01:00
parent 215a67238a
commit 0468bc4321
1 changed files with 40 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

90
guix-data-service/model/package-metadata.scm
View file

@ -292,18 +292,6 @@ WHERE packages.id IN (
(prevent-inlining-for-tests inferior-packages->translated-package-descriptions-and-synopsis) (prevent-inlining-for-tests inferior-packages->translated-package-descriptions-and-synopsis)
(define (package-synopsis-data->package-synopsis-ids
conn synopsis-by-locale)
(insert-missing-data-and-return-all-ids
conn
"package_synopsis"
'(locale synopsis)
(map (match-lambda
((locale . synopsis)
(list locale synopsis)))
synopsis-by-locale)
#:delete-duplicates? #t))
(define (insert-package-synopsis-set conn package-synopsis-ids) (define (insert-package-synopsis-set conn package-synopsis-ids)
(let ((query (let ((query
(string-append (string-append
@ -320,38 +308,22 @@ WHERE packages.id IN (
(((id)) id)))) (((id)) id))))
(define (package-synopsis-data->package-synopsis-set-id (define (package-synopsis-data->package-synopsis-set-id
conn synopsis-by-locale) conn package-synopsis-ids)
(let* ((package-synopsis-ids (let ((package-synopsis-set-id
(package-synopsis-data->package-synopsis-ids (exec-query
conn conn
synopsis-by-locale)) (string-append
(package-synopsis-set-id "SELECT id FROM package_synopsis_sets"
(exec-query " WHERE synopsis_ids = ARRAY["
conn (string-join (map number->string
(string-append (sort package-synopsis-ids <)) ", ")
"SELECT id FROM package_synopsis_sets" "]"))))
" WHERE synopsis_ids = ARRAY["
(string-join (map number->string
(sort package-synopsis-ids <)) ", ")
"]"))))
(string->number (string->number
(match package-synopsis-set-id (match package-synopsis-set-id
(((id)) id) (((id)) id)
(() (()
(insert-package-synopsis-set conn package-synopsis-ids)))))) (insert-package-synopsis-set conn package-synopsis-ids))))))
(define (package-description-data->package-description-ids
conn descriptions-by-locale)
(insert-missing-data-and-return-all-ids
conn
"package_descriptions"
'(locale description)
(map (match-lambda
((locale . description)
(list locale description)))
descriptions-by-locale)
#:delete-duplicates? #t))
(define (insert-package-description-set conn package-description-ids) (define (insert-package-description-set conn package-description-ids)
(let ((query (let ((query
(string-append (string-append
@ -419,19 +391,37 @@ WHERE packages.id IN (
conn conn
package-description-ids)) package-description-ids))
(with-time-logging "preparing package description ids" (with-time-logging "preparing package description ids"
(map (match-lambda (insert-missing-data-and-return-all-ids
((_ _ package-description-data _) conn
(package-description-data->package-description-ids "package_descriptions"
conn '(locale description)
package-description-data))) (map (match-lambda
package-metadata)))) ((_ _ package-description-data _)
(map (match-lambda
((locale . description)
(list locale description)))
package-description-data)))
package-metadata)
#:delete-duplicates? #t
#:sets-of-data? #t))))
(with-time-logging "preparing package synopsis set ids" (with-time-logging "preparing package synopsis set ids"
(map (match-lambda (map (lambda (package-synopsis-ids)
((_ _ _ package-synopsis-data) (package-synopsis-data->package-synopsis-set-id
(package-synopsis-data->package-synopsis-set-id conn
conn package-synopsis-ids))
package-synopsis-data))) (insert-missing-data-and-return-all-ids
package-metadata))) conn
"package_synopsis"
'(locale synopsis)
(map (match-lambda
((_ _ _ package-synopsis-data)
(map (match-lambda
((locale . synopsis)
(list locale synopsis)))
package-synopsis-data)))
package-metadata)
#:delete-duplicates? #t
#:sets-of-data? #t))))
;; There can be duplicated entires in package-metadata, for example where ;; There can be duplicated entires in package-metadata, for example where
;; you have one package definition which interits from another, and just ;; you have one package definition which interits from another, and just
;; overrides the version and the source, the package_metadata entries for ;; overrides the version and the source, the package_metadata entries for