cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Handle conflicts in insert-missing-data-and-return-all-ids

Browse source
This commit is contained in:
Christopher Baines 2024-11-29 13:49:01 +00:00
parent c949deb9d4
commit bb84e45c42
1 changed files with 25 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

43
guix-data-service/model/utils.scm
View file

@ -323,7 +323,7 @@ WHERE table_name = $1"
")"))
missing-data)
", ")
" RETURNING id"))
" ON CONFLICT DO NOTHING"))
(define (format-json json)
;; PostgreSQL formats JSON strings differently to guile-json, so use
@ -355,16 +355,17 @@ WHERE table_name = $1"
(error (simple-format #f "normalise-values: error: ~A\n" unknown))))
data))
(let* ((existing-entries
(if use-temporary-table?
(let ((temp-table-name
(string-append "temp_" table-name))
(data
(let* ((flattened-deduplicated-data
(if sets-of-data?
(delete-duplicates* (concatenate data))
(delete-duplicates*
(concatenate data))
(if delete-duplicates?
(delete-duplicates* data)
data))))
data)))
(existing-entries
(if use-temporary-table?
(let ((temp-table-name
(string-append "temp_" table-name)))
;; Create a temporary table to store the data
(exec-query
conn
@ -379,11 +380,10 @@ WHERE table_name = $1"
"ANALYZE " temp-table-name))
;; Populate the temporary table
(if (null? data)
'()
(unless (null? flattened-deduplicated-data)
(with-time-logging (string-append "populating " temp-table-name)
(exec-query conn
(insert-sql data
(insert-sql flattened-deduplicated-data
#:table-name temp-table-name))))
;; Use the temporary table to find the existing values
(let ((result
@ -400,7 +400,7 @@ WHERE table_name = $1"
result))
;; If not using a temporary table, just do a single SELECT query
(if (null? data)
(if (null? flattened-deduplicated-data)
'()
(fold
(lambda (data-chunk result)
@ -411,15 +411,10 @@ WHERE table_name = $1"
(string->number (first result)))
#:vhash result))
vlist-null
(chunk (if sets-of-data?
(delete-duplicates*
(concatenate data))
data)
(chunk flattened-deduplicated-data
3000)))))
(missing-entries
(let loop ((lst (if sets-of-data?
(concatenate data)
data))
(let loop ((lst flattened-deduplicated-data)
(result '()))
(if (null? lst)
(if delete-duplicates?
@ -440,10 +435,12 @@ WHERE table_name = $1"
'()
(append-map!
(lambda (missing-entries-chunk)
(map (lambda (result)
(string->number (first result)))
(exec-query conn
(insert-sql missing-entries-chunk))))
(insert-sql missing-entries-chunk))
(map (lambda (row)
(string->number (first row)))
(exec-query conn (select-query missing-entries-chunk))))
(chunk missing-entries 3000))))
(new-entries-lookup-vhash
@ -474,4 +471,4 @@ WHERE table_name = $1"
(error "missing entry" field-values))))
data))))
(values all-ids
new-entries)))
(delete-duplicates/sort! new-entries <))))