cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use delete-duplicates/sort! in insert-missing-data-and-return-all-ids

Browse source 
As it's faster than delete-duplicates for large amounts of data.
This commit is contained in:
Christopher Baines 2024-01-15 11:18:39 +00:00
parent 241d7e4889
commit 49b4841c4e
1 changed files with 43 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

48
guix-data-service/model/utils.scm
View file

@ -178,6 +178,44 @@ WHERE table_name = $1"
(error (error
(simple-format #f "error: unknown type for value: ~A" v))))) (simple-format #f "error: unknown type for value: ~A" v)))))
(define (delete-duplicates* data)
(delete-duplicates/sort!
(list-copy data)
(lambda (full-a full-b)
(let loop ((a full-a)
(b full-b))
(if (null? a)
#f
(let ((a-val (match (car a)
((_ . val) val)
((? symbol? val) (symbol->string val))
(val val)))
(b-val (match (car b)
((_ . val) val)
((? symbol? val) (symbol->string val))
(val val))))
(cond
((null? a-val)
(if (null? b-val)
(loop (cdr a) (cdr b))
#t))
((null? b-val)
#f)
(else
(match a-val
((? string? v)
(if (string=? a-val b-val)
(loop (cdr a) (cdr b))
(string<? a-val b-val)))
((? number? v)
(if (= a-val b-val)
(loop (cdr a) (cdr b))
(< a-val b-val)))
((? boolean? v)
(if (eq? a-val b-val)
(loop (cdr a) (cdr b))
a-val)))))))))))
(define schema-details (define schema-details
(table-schema conn table-name)) (table-schema conn table-name))
@ -312,9 +350,9 @@ WHERE table_name = $1"
(string-append "temp_" table-name)) (string-append "temp_" table-name))
(data (data
(if sets-of-data? (if sets-of-data?
(delete-duplicates (concatenate data)) (delete-duplicates* (concatenate data))
(if delete-duplicates? (if delete-duplicates?
(delete-duplicates data) (delete-duplicates* data)
data)))) data))))
;; Create a temporary table to store the data ;; Create a temporary table to store the data
(exec-query (exec-query
@ -363,7 +401,7 @@ WHERE table_name = $1"
#:vhash result)) #:vhash result))
vlist-null vlist-null
(chunk (if sets-of-data? (chunk (if sets-of-data?
(delete-duplicates (delete-duplicates*
(concatenate data)) (concatenate data))
data) data)
3000))))) 3000)))))
@ -375,9 +413,9 @@ WHERE table_name = $1"
(normalise-values field-values) (normalise-values field-values)
existing-entries))) existing-entries)))
(if sets-of-data? (if sets-of-data?
(delete-duplicates (concatenate data)) (delete-duplicates* (concatenate data))
(if delete-duplicates? (if delete-duplicates?
(delete-duplicates data) (delete-duplicates* data)
data)))) data))))
(new-entries (new-entries
(if (null? missing-entries) (if (null? missing-entries)