From 639f24bb2260d979ed97df1bb4db589fd82b97a9 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sun, 22 Dec 2019 16:09:32 +0000 Subject: [PATCH] Extract management of package_derivations_by_guix_revision_range ... entries in to a separate module, to split the code up a little further. --- Makefile.am | 101 +++++++-------- .../jobs/load-new-guix-revision.scm | 89 +------------ ...kage-derivation-by-guix-revision-range.scm | 118 ++++++++++++++++++ 3 files changed, 170 insertions(+), 138 deletions(-) create mode 100644 guix-data-service/model/package-derivation-by-guix-revision-range.scm diff --git a/Makefile.am b/Makefile.am index 954b4b9..cded23a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -64,56 +64,57 @@ check-with-tmp-database: GUIX_DATA_SERVICE_DATABASE_URI="$(GUIX_DATA_SERVICE_DATABASE_URI)" make check pg_tmp stop -w 1 -d "$(TMP_DATABASE)" -SOURCES = \ - guix-data-service/branch-updated-emails.scm \ - guix-data-service/builds.scm \ - guix-data-service/comparison.scm \ - guix-data-service/config.scm \ - guix-data-service/database.scm \ - guix-data-service/jobs.scm \ - guix-data-service/jobs/load-new-guix-revision.scm \ - guix-data-service/model/build-server.scm \ - guix-data-service/model/build-server-token-seed.scm \ - guix-data-service/model/build-status.scm \ - guix-data-service/model/build.scm \ - guix-data-service/model/channel-news.scm \ - guix-data-service/model/derivation.scm \ - guix-data-service/model/git-branch.scm \ - guix-data-service/model/git-repository.scm \ - guix-data-service/model/guix-revision-package-derivation.scm \ - guix-data-service/model/guix-revision.scm \ - guix-data-service/model/license-set.scm \ - guix-data-service/model/license.scm \ - guix-data-service/model/lint-checker.scm \ - guix-data-service/model/lint-warning-message.scm \ - guix-data-service/model/lint-warning.scm \ - guix-data-service/model/location.scm \ - guix-data-service/model/nar.scm \ - guix-data-service/model/package-derivation.scm \ - guix-data-service/model/package-metadata.scm \ - guix-data-service/model/package.scm \ - guix-data-service/model/utils.scm \ - guix-data-service/web/html-utils.scm \ - guix-data-service/web/compare/controller.scm \ - guix-data-service/web/compare/html.scm \ - guix-data-service/web/controller.scm \ - guix-data-service/web/build/controller.scm \ - guix-data-service/web/build/html.scm \ - guix-data-service/web/build-server/controller.scm \ - guix-data-service/web/build-server/html.scm \ - guix-data-service/web/jobs/controller.scm \ - guix-data-service/web/jobs/html.scm \ - guix-data-service/web/query-parameters.scm \ - guix-data-service/web/render.scm \ - guix-data-service/web/repository/controller.scm \ - guix-data-service/web/repository/html.scm \ - guix-data-service/web/revision/controller.scm \ - guix-data-service/web/revision/html.scm \ - guix-data-service/web/server.scm \ - guix-data-service/web/sxml.scm \ - guix-data-service/web/util.scm \ - guix-data-service/web/view/html.scm \ - tests/driver.scm \ +SOURCES = \ + guix-data-service/branch-updated-emails.scm \ + guix-data-service/builds.scm \ + guix-data-service/comparison.scm \ + guix-data-service/config.scm \ + guix-data-service/database.scm \ + guix-data-service/jobs.scm \ + guix-data-service/jobs/load-new-guix-revision.scm \ + guix-data-service/model/build-server.scm \ + guix-data-service/model/build-server-token-seed.scm \ + guix-data-service/model/build-status.scm \ + guix-data-service/model/build.scm \ + guix-data-service/model/channel-news.scm \ + guix-data-service/model/derivation.scm \ + guix-data-service/model/git-branch.scm \ + guix-data-service/model/git-repository.scm \ + guix-data-service/model/guix-revision-package-derivation.scm \ + guix-data-service/model/guix-revision.scm \ + guix-data-service/model/license-set.scm \ + guix-data-service/model/license.scm \ + guix-data-service/model/lint-checker.scm \ + guix-data-service/model/lint-warning-message.scm \ + guix-data-service/model/lint-warning.scm \ + guix-data-service/model/location.scm \ + guix-data-service/model/nar.scm \ + guix-data-service/model/package-derivation-by-guix-revision-range.scm \ + guix-data-service/model/package-derivation.scm \ + guix-data-service/model/package-metadata.scm \ + guix-data-service/model/package.scm \ + guix-data-service/model/utils.scm \ + guix-data-service/web/html-utils.scm \ + guix-data-service/web/compare/controller.scm \ + guix-data-service/web/compare/html.scm \ + guix-data-service/web/controller.scm \ + guix-data-service/web/build/controller.scm \ + guix-data-service/web/build/html.scm \ + guix-data-service/web/build-server/controller.scm \ + guix-data-service/web/build-server/html.scm \ + guix-data-service/web/jobs/controller.scm \ + guix-data-service/web/jobs/html.scm \ + guix-data-service/web/query-parameters.scm \ + guix-data-service/web/render.scm \ + guix-data-service/web/repository/controller.scm \ + guix-data-service/web/repository/html.scm \ + guix-data-service/web/revision/controller.scm \ + guix-data-service/web/revision/html.scm \ + guix-data-service/web/server.scm \ + guix-data-service/web/sxml.scm \ + guix-data-service/web/util.scm \ + guix-data-service/web/view/html.scm \ + tests/driver.scm \ tests/mock-inferior.scm TEST_EXTENSIONS = .scm diff --git a/guix-data-service/jobs/load-new-guix-revision.scm b/guix-data-service/jobs/load-new-guix-revision.scm index 7ed9032..81b385a 100644 --- a/guix-data-service/jobs/load-new-guix-revision.scm +++ b/guix-data-service/jobs/load-new-guix-revision.scm @@ -20,6 +20,7 @@ #:use-module (guix-data-service model build) #:use-module (guix-data-service model channel-news) #:use-module (guix-data-service model package) + #:use-module (guix-data-service model package-derivation-by-guix-revision-range) #:use-module (guix-data-service model git-repository) #:use-module (guix-data-service model guix-revision) #:use-module (guix-data-service model package-derivation) @@ -961,94 +962,6 @@ ORDER BY packages.name, packages.version" #t) -(define (update-package-derivations-table conn - git-repository-id - guix-revision-id - commit) - ;; Lock the table to wait for other transactions to commit before updating - ;; the table - (exec-query - conn - " -LOCK TABLE ONLY package_derivations_by_guix_revision_range - IN SHARE ROW EXCLUSIVE MODE") - - (for-each - (match-lambda - ((branch-name) - (log-time - (simple-format #f "deleting package derivation entries for ~A" branch-name) - (lambda () - (exec-query - conn - " -DELETE FROM package_derivations_by_guix_revision_range -WHERE git_repository_id = $1 AND - branch_name = $2 AND - derivation_id IN ( - SELECT package_derivations.derivation_id - FROM package_derivations - INNER JOIN guix_revision_package_derivations - ON package_derivations.id = guix_revision_package_derivations.package_derivation_id - WHERE revision_id = $3 - )" - (list git-repository-id - branch-name - guix-revision-id)))) - (log-time - (simple-format #f "inserting package derivation entries for ~A" branch-name) - (lambda () - (exec-query - conn - " -INSERT INTO package_derivations_by_guix_revision_range -SELECT DISTINCT - $1::integer AS git_repository_id, - $2 AS branch_name, - packages.name AS package_name, - packages.version AS package_version, - revision_packages.derivation_id AS derivation_id, - revision_packages.system AS system, - revision_packages.target AS target, - first_value(guix_revisions.id) - OVER package_version AS first_guix_revision_id, - last_value(guix_revisions.id) - OVER package_version AS last_guix_revision_id -FROM packages -INNER JOIN ( - SELECT DISTINCT package_derivations.package_id, - package_derivations.derivation_id, - package_derivations.system, - package_derivations.target, - guix_revision_package_derivations.revision_id - FROM package_derivations - INNER JOIN guix_revision_package_derivations - ON package_derivations.id = guix_revision_package_derivations.package_derivation_id -) AS revision_packages ON packages.id = revision_packages.package_id -INNER JOIN guix_revisions ON revision_packages.revision_id = guix_revisions.id -INNER JOIN git_branches ON guix_revisions.commit = git_branches.commit -WHERE git_branches.name = $2 AND - revision_packages.derivation_id IN ( - SELECT package_derivations.derivation_id - FROM package_derivations - INNER JOIN guix_revision_package_derivations - ON package_derivations.id = guix_revision_package_derivations.package_derivation_id - WHERE revision_id = $3 - ) -WINDOW package_version AS ( - PARTITION BY packages.name, packages.version, revision_packages.derivation_id - ORDER BY git_branches.datetime - RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING -) -ORDER BY packages.name, packages.version" - (list git-repository-id branch-name guix-revision-id)))))) - (exec-query - conn - "SELECT name FROM git_branches WHERE commit = $1 AND git_repository_id = $2" - (list commit git-repository-id))) - - #t) - (define (store-item-for-channel conn channel) (with-store store (set-build-options store #:fallback? #t) diff --git a/guix-data-service/model/package-derivation-by-guix-revision-range.scm b/guix-data-service/model/package-derivation-by-guix-revision-range.scm new file mode 100644 index 0000000..2e48389 --- /dev/null +++ b/guix-data-service/model/package-derivation-by-guix-revision-range.scm @@ -0,0 +1,118 @@ +;;; Guix Data Service -- Information about Guix over time +;;; Copyright © 2019 Christopher Baines +;;; +;;; This program is free software: you can redistribute it and/or +;;; modify it under the terms of the GNU Affero General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; This program is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Affero General Public License for more details. +;;; +;;; You should have received a copy of the GNU Affero General Public +;;; License along with this program. If not, see +;;; . + +(define-module (guix-data-service model package-derivation-by-guix-revision-range) + #:use-module (ice-9 match) + #:use-module (squee) + #:export (update-package-derivations-table)) + +(define (log-time action f) + (simple-format #t "debug: Starting ~A\n" action) + (let* ((start-time (current-time)) + (result (f)) + (time-taken (- (current-time) start-time))) + (simple-format #t "debug: Finished ~A, took ~A seconds\n" + action time-taken) + result)) + +(define (update-package-derivations-table conn + git-repository-id + guix-revision-id + commit) + ;; Lock the table to wait for other transactions to commit before updating + ;; the table + (exec-query + conn + " +LOCK TABLE ONLY package_derivations_by_guix_revision_range + IN SHARE ROW EXCLUSIVE MODE") + + (for-each + (match-lambda + ((branch-name) + (log-time + (simple-format #f "deleting package derivation entries for ~A" branch-name) + (lambda () + (exec-query + conn + " +DELETE FROM package_derivations_by_guix_revision_range +WHERE git_repository_id = $1 AND + branch_name = $2 AND + derivation_id IN ( + SELECT package_derivations.derivation_id + FROM package_derivations + INNER JOIN guix_revision_package_derivations + ON package_derivations.id = guix_revision_package_derivations.package_derivation_id + WHERE revision_id = $3 + )" + (list git-repository-id + branch-name + guix-revision-id)))) + (log-time + (simple-format #f "inserting package derivation entries for ~A" branch-name) + (lambda () + (exec-query + conn + " +INSERT INTO package_derivations_by_guix_revision_range +SELECT DISTINCT + $1::integer AS git_repository_id, + $2 AS branch_name, + packages.name AS package_name, + packages.version AS package_version, + revision_packages.derivation_id AS derivation_id, + revision_packages.system AS system, + revision_packages.target AS target, + first_value(guix_revisions.id) + OVER package_version AS first_guix_revision_id, + last_value(guix_revisions.id) + OVER package_version AS last_guix_revision_id +FROM packages +INNER JOIN ( + SELECT DISTINCT package_derivations.package_id, + package_derivations.derivation_id, + package_derivations.system, + package_derivations.target, + guix_revision_package_derivations.revision_id + FROM package_derivations + INNER JOIN guix_revision_package_derivations + ON package_derivations.id = guix_revision_package_derivations.package_derivation_id +) AS revision_packages ON packages.id = revision_packages.package_id +INNER JOIN guix_revisions ON revision_packages.revision_id = guix_revisions.id +INNER JOIN git_branches ON guix_revisions.commit = git_branches.commit +WHERE git_branches.name = $2 AND + revision_packages.derivation_id IN ( + SELECT package_derivations.derivation_id + FROM package_derivations + INNER JOIN guix_revision_package_derivations + ON package_derivations.id = guix_revision_package_derivations.package_derivation_id + WHERE revision_id = $3 + ) +WINDOW package_version AS ( + PARTITION BY packages.name, packages.version, revision_packages.derivation_id + ORDER BY git_branches.datetime + RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) +ORDER BY packages.name, packages.version" + (list git-repository-id branch-name guix-revision-id)))))) + (exec-query + conn + "SELECT name FROM git_branches WHERE commit = $1 AND git_repository_id = $2" + (list commit git-repository-id))) + + #t)