cbaines/guix-data-service
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rework how data is inserted

Browse source 
This is the big change needed to allow for parallel revision
processing. Previously, a lock was used to prevent this since the parallel
transactions could deadlock if each inserted data that the other then went to
insert.

By defining the order in which inserts happen, both in terms of the order of
tables, and the order of rows within the table, this change should guarantee
that there won't be deadlocks.

I'm also hoping this change will address whatever issue was causing some
derivation data to be missing from the database.
This commit is contained in:
Christopher Baines 2025-12-28 19:01:39 +00:00
parent 72fe3b4e47
commit f66ff9a3ff
25 changed files with 3383 additions and 1825 deletions
Download patch file Download diff file Expand all files Collapse all files

26
tests/jobs-load-new-guix-revision.scm
View file

@ -3,11 +3,13 @@
#:use-module (ice-9 match)
#:use-module (squee)
#:use-module (fibers)
#:use-module (knots)
#:use-module (guix utils)
#:use-module (guix store)
#:use-module (guix tests)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model git-repository)
#:use-module (guix-data-service model guix-revision)
#:use-module (guix-data-service jobs load-new-guix-revision))
(test-begin "jobs-load-new-guix-revision")
@ -65,13 +67,16 @@
((guix-data-service jobs load-new-guix-revision)
extract-information-from
(lambda _
#t))
'()))
(mock
((guix-data-service model channel-instance)
insert-channel-instances
(lambda (conn guix-revision-id derivations-by-system)
#t))
((guix-data-service jobs load-new-guix-revision)
load-channel-instances
(lambda (call-with-utility-thread
read-derivations/serialised
git-repository-id commit
channel-derivations-by-system)
(insert-guix-revision conn git-repository-id commit)))
(mock
((guix channels)
@ -81,7 +86,7 @@
(mock
((guix-data-service jobs load-new-guix-revision)
derivation-file-names->derivation-ids
insert-derivations-with-table-managers
(lambda _
#(1)))
@ -103,8 +108,13 @@
((id)
(run-fibers
(lambda ()
(process-load-new-guix-revision-job
id #:parallelism 1))
(with-exception-handler
(lambda (exn)
(print-backtrace-and-exception/knots exn)
(raise-exception exn))
(lambda ()
(process-load-new-guix-revision-job
id #:parallelism 1))))
#:hz 0
#:parallelism 1
#:drain? #t))))))))))))))

17
tests/model-derivation.scm
View file

@ -1,17 +0,0 @@
(define-module (test-model-derivation)
#:use-module (srfi srfi-64)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model derivation))
(test-begin "test-model-derivation")
(with-postgresql-connection
"test-model-derivation"
(lambda (conn)
(check-test-database! conn)
(test-equal "count-derivations"
'("0")
(count-derivations conn))))
(test-end)

47
tests/model-license-set.scm
View file

@ -1,47 +0,0 @@
(define-module (tests model-license-set)
#:use-module (srfi srfi-64)
#:use-module (guix utils)
#:use-module (guix tests)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model license)
#:use-module (guix-data-service model license-set))
(test-begin "test-model-license-set")
(define license-data
'#((("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))
(("License 1"
"https://gnu.org/licenses/test-1.html"
#f)
("License 2"
#f
#f))
()))
(with-postgresql-connection
"test-model-license-set"
(lambda (conn)
(check-test-database! conn)
(with-postgresql-transaction
conn
(lambda (conn)
(test-assert "works"
(inferior-packages->license-set-ids
conn
(inferior-packages->license-id-lists conn license-data))))
#:always-rollback? #t)
(with-postgresql-transaction
conn
(lambda (conn)
(let ((license-id-lists
(inferior-packages->license-id-lists conn license-data)))
(test-equal "works repeatedly"
(inferior-packages->license-set-ids conn license-id-lists)
(inferior-packages->license-set-ids conn license-id-lists))))
#:always-rollback? #t)))
(test-end)

44
tests/model-license.scm
View file

@ -1,44 +0,0 @@
(define-module (tests model-license)
#:use-module (srfi srfi-64)
#:use-module (guix utils)
#:use-module (guix tests)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model license))
(test-begin "test-model-license")
(define license-data
'#((("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))
(("License 1"
"https://gnu.org/licenses/test-1.html"
#f)
("License 2"
"https://gnu.org/licenses/test-2.html"
#f)
("License 3"
#f
#f))))
(with-postgresql-connection
"test-model-license"
(lambda (conn)
(check-test-database! conn)
(with-postgresql-transaction
conn
(lambda (conn)
(test-assert "works"
(inferior-packages->license-id-lists conn license-data)))
#:always-rollback? #t)
(with-postgresql-transaction
conn
(lambda (conn)
(test-equal "works repeatedly"
(inferior-packages->license-id-lists conn license-data)
(inferior-packages->license-id-lists conn license-data)))
#:always-rollback? #t)))
(test-end)

38
tests/model-lint-checker.scm
View file

@ -1,38 +0,0 @@
(define-module (tests model-lint-checker)
#:use-module (srfi srfi-64)
#:use-module (ice-9 match)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model lint-checker))
(test-begin "test-model-lint-checker")
(with-postgresql-connection
"test-model-lint-checker"
(lambda (conn)
(check-test-database! conn)
(test-assert "single insert"
(with-postgresql-transaction
conn
(lambda (conn)
(define data
`#((name-1
#t
,(lint-checker-description-data->lint-checker-description-set-id
conn
'(("en_US" . "foo"))))
(name-2
#f
,(lint-checker-description-data->lint-checker-description-set-id
conn
'(("en_US" . "bar"))))))
(match (lint-checkers->lint-checker-ids conn data)
(#((? number? id1) (? number? id2))
(match (lint-checkers->lint-checker-ids conn data)
(#((? number? second-id1) (? number? second-id2))
(and (= id1 second-id1)
(= id2 second-id2)))))))
#:always-rollback? #t))))
(test-end)

59
tests/model-lint-warning-message.scm
View file

@ -1,59 +0,0 @@
(define-module (tests model-lint-warning-message)
#:use-module (srfi srfi-64)
#:use-module (ice-9 match)
#:use-module (guix-data-service database)
#:use-module (guix-data-service model lint-warning-message))
(test-begin "test-model-lint-warning-message")
(define data
'(("en" . "Test message")
("es" . "Test message in Spanish")))
(with-postgresql-connection
"test-model-lint-checker"
(lambda (conn)
(check-test-database! conn)
(test-assert "single insert"
(with-postgresql-transaction
conn
(lambda (conn)
(match (lint-warning-message-data->lint-warning-message-ids conn data)
(#((? number? id1) (? number? id2))
#t)))
#:always-rollback? #t))
(test-assert "double insert"
(with-postgresql-transaction
conn
(lambda (conn)
(match (lint-warning-message-data->lint-warning-message-ids conn data)
(#((? number? id1) (? number? id2))
(match (lint-warning-message-data->lint-warning-message-ids conn data)
(#((? number? second-id1) (? number? second-id2))
(and (= id1 second-id1)
(= id2 second-id2)))))))
#:always-rollback? #t))
(test-assert "single set insert"
(with-postgresql-transaction
conn
(lambda (conn)
(match (lint-warning-message-data->lint-warning-message-set-id conn data)
((? number? id1)
#t)))
#:always-rollback? #t))
(test-assert "double set insert"
(with-postgresql-transaction
conn
(lambda (conn)
(match (lint-warning-message-data->lint-warning-message-set-id conn data)
((? number? id)
(match (lint-warning-message-data->lint-warning-message-set-id conn data)
((? number? second-id)
(= id second-id))))))
#:always-rollback? #t))))
(test-end)

98
tests/model-package-metadata.scm
View file

@ -1,98 +0,0 @@
(define-module (test-model-package-metadata)
#:use-module (ice-9 match)
#:use-module (srfi srfi-64)
#:use-module (guix utils)
#:use-module (guix tests)
#:use-module (tests mock-inferior)
#:use-module (guix-data-service model license)
#:use-module (guix-data-service model license-set)
#:use-module (guix-data-service model package-metadata)
#:use-module (guix-data-service database))
(test-begin "test-model-package-metadata")
(define mock-inferior-package-foo
(mock-inferior-package
(name "foo")
(version "2")
(synopsis "Foo")
(description "Foo description")
(home-page "https://example.com")
(location (location "file.scm" 5 0))))
(define mock-inferior-package-foo-2
(mock-inferior-package
(name "foo")
(version "2")
(synopsis "Foo")
(description "Foo description")
(home-page #f)
(location #f)))
(define mock-inferior-packages
(list mock-inferior-package-foo
mock-inferior-package-foo-2))
(define mock-package-metadata
(list->vector
(map (lambda (mock-inf-pkg)
(list
(mock-inferior-package-home-page mock-inf-pkg)
(mock-inferior-package-location mock-inf-pkg)
`(("en_US.UTF-8" . "Fake synopsis"))
`(("en_US.UTF-8" . "Fake description"))))
mock-inferior-packages)))
(define (test-license-set-ids conn)
(let ((license-id-lists
(inferior-packages->license-id-lists
conn
'#((("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))
(("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))))))
(inferior-packages->license-set-ids conn license-id-lists)))
(with-mock-inferior-packages
(lambda ()
(use-modules (guix-data-service model package)
(guix-data-service model git-repository)
(guix-data-service model guix-revision)
(guix-data-service model package-metadata))
(with-postgresql-connection
"test-model-package-metadata"
(lambda (conn)
(check-test-database! conn)
(test-assert "inferior-packages->package-metadata-ids"
(with-postgresql-transaction
conn
(lambda (conn)
(match
(inferior-packages->package-metadata-ids
conn
mock-package-metadata
(test-license-set-ids conn))
(#(x y) (and (number? x)
(number? y)))))
#:always-rollback? #t))
(with-postgresql-transaction
conn
(lambda (conn)
(test-equal "inferior-packages->package-metadata-ids"
(inferior-packages->package-metadata-ids
conn
mock-package-metadata
(test-license-set-ids conn))
(inferior-packages->package-metadata-ids
conn
mock-package-metadata
(test-license-set-ids conn)))
#:always-rollback? #t))))))
(test-end)

125
tests/model-package.scm
View file

@ -1,125 +0,0 @@
(define-module (test-model-package)
#:use-module (ice-9 match)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-64)
#:use-module (guix utils)
#:use-module (guix tests)
#:use-module (tests mock-inferior)
#:use-module (guix-data-service model utils)
#:use-module (guix-data-service model license)
#:use-module (guix-data-service model license-set)
#:use-module (guix-data-service model package)
#:use-module (guix-data-service model package-metadata)
#:use-module (guix-data-service database))
(test-begin "test-model-package")
(define mock-inferior-package-foo
(mock-inferior-package
(name "foo")
(version "2")
(synopsis "Foo")
(description "Foo description")
(home-page "https://example.com")
(location (location "file.scm" 5 0))))
(define mock-inferior-package-foo-2
(mock-inferior-package
(name "foo")
(version "2")
(synopsis "Foo")
(description "Foo description")
(home-page #f)
(location #f)))
(define (test-license-set-ids conn)
(let ((license-id-lists
(inferior-packages->license-id-lists
conn
'#((("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))
(("License 1"
"https://gnu.org/licenses/test-1.html"
"https://example.com/why-license-1"))))))
(inferior-packages->license-set-ids conn license-id-lists)))
(define mock-inferior-packages
(list mock-inferior-package-foo
mock-inferior-package-foo-2))
(define mock-package-metadata
(list->vector
(map (lambda (mock-inf-pkg)
(list
(mock-inferior-package-home-page mock-inf-pkg)
(mock-inferior-package-location mock-inf-pkg)
`(("en_US.UTF-8" . "Fake synopsis"))
`(("en_US.UTF-8" . "Fake description"))))
mock-inferior-packages)))
(with-mock-inferior-packages
(lambda ()
(use-modules (guix-data-service model package)
(guix-data-service model git-repository)
(guix-data-service model guix-revision)
(guix-data-service model package-metadata))
(with-postgresql-connection
"test-model-package"
(lambda (conn)
(check-test-database! conn)
(with-postgresql-transaction
conn
(lambda (conn)
(test-assert "inferior-packages->package-ids works once"
(let ((package-metadata-ids
(inferior-packages->package-metadata-ids
conn
mock-package-metadata
(test-license-set-ids conn)))
(package-replacement-package-ids
(make-list (length mock-inferior-packages)
(cons "integer" NULL))))
(match (inferior-packages->package-ids
conn
(list->vector
(zip (map mock-inferior-package-name mock-inferior-packages)
(map mock-inferior-package-version mock-inferior-packages)
(vector->list package-metadata-ids)
package-replacement-package-ids)))
(#(x y) (and (number? x)
(number? y)))))))
#:always-rollback? #t)
(with-postgresql-transaction
conn
(lambda (conn)
(let ((package-metadata-ids
(inferior-packages->package-metadata-ids
conn
mock-package-metadata
(test-license-set-ids conn)))
(package-replacement-package-ids
(make-list (length mock-inferior-packages)
(cons "integer" NULL))))
(test-equal "inferior-packages->package-ids is idempotent"
(inferior-packages->package-ids
conn
(list->vector
(zip (map mock-inferior-package-name mock-inferior-packages)
(map mock-inferior-package-version mock-inferior-packages)
(vector->list package-metadata-ids)
package-replacement-package-ids)))
(inferior-packages->package-ids
conn
(list->vector
(zip (map mock-inferior-package-name mock-inferior-packages)
(map mock-inferior-package-version mock-inferior-packages)
(vector->list package-metadata-ids)
package-replacement-package-ids))))))
#:always-rollback? #t)))))
(test-end)