cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Chunk the data for some queries in insert-missing-data-and-return-all-ids

Browse source 
This helps to avoid queries getting logged as slow just because of the amount
of data.
This commit is contained in:
Christopher Baines 2022-09-15 09:18:19 +01:00
parent d791c2bd8a
commit 37e8af60fb
1 changed files with 27 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

42
guix-data-service/model/utils.scm
View file

@ -53,12 +53,13 @@
s) s)
#f)) #f))
(define (exec-query->vhash conn query field-function value-function) (define* (exec-query->vhash conn query field-function value-function
#:key (vhash vlist-null))
(fold (lambda (row result) (fold (lambda (row result)
(vhash-cons (field-function row) (vhash-cons (field-function row)
(value-function row) (value-function row)
result)) result))
vlist-null vhash
(exec-query-with-null-handling conn query))) (exec-query-with-null-handling conn query)))
(define (two-lists->vhash l1 l2) (define (two-lists->vhash l1 l2)
@ -194,7 +195,7 @@ WHERE table_name = $1"
schema-details) schema-details)
(error "error: field-can-be-null?")))) (error "error: field-can-be-null?"))))
(define select-query (define (select-query data)
(string-append (string-append
"SELECT id,\n" "SELECT id,\n"
(string-join (map (lambda (field) (string-join (map (lambda (field)
@ -210,10 +211,7 @@ WHERE table_name = $1"
"(" "("
(string-join (map value->sql field-values) ",") (string-join (map value->sql field-values) ",")
")")) ")"))
(if sets-of-data? data)
(delete-duplicates
(concatenate data))
data))
", ") ", ")
")\n AS vals (" (string-join field-strings ", ") ") " ")\n AS vals (" (string-join field-strings ", ") ") "
"ON " "ON "
@ -355,11 +353,20 @@ WHERE table_name = $1"
;; If not using a temporary table, just do a single SELECT query ;; If not using a temporary table, just do a single SELECT query
(if (null? data) (if (null? data)
'() '()
(exec-query->vhash conn (fold
select-query (lambda (data-chunk result)
cdr (exec-query->vhash conn
(lambda (result) (select-query data-chunk)
(string->number (first result))))))) cdr
(lambda (result)
(string->number (first result)))
#:vhash result))
vlist-null
(chunk (if sets-of-data?
(delete-duplicates
(concatenate data))
data)
5000)))))
(missing-entries (missing-entries
(filter (lambda (field-values) (filter (lambda (field-values)
(not (vhash-assoc (not (vhash-assoc
@ -375,9 +382,14 @@ WHERE table_name = $1"
(new-entries (new-entries
(if (null? missing-entries) (if (null? missing-entries)
'() '()
(map (lambda (result) (append-map!
(string->number (first result))) (lambda (missing-entries-chunk)
(exec-query conn (insert-sql missing-entries))))) (map (lambda (result)
(string->number (first result)))
(exec-query conn
(insert-sql missing-entries-chunk))))
(chunk missing-entries 5000))))
(new-entries-lookup-vhash (new-entries-lookup-vhash
(two-lists->vhash missing-entries (two-lists->vhash missing-entries
new-entries)) new-entries))