Prioritise processing the latest revisions for branches

Reserve some capacity to process revisions which are the tip of a branch. This
should reduce the time between new revisions appearing, and then being
processed.

guix-data-service/jobs.scm

@@ -129,12 +129,17 @@
        #f)
       ((jobs ...)
        (for-each
-        (lambda (job-args)
+        (match-lambda
+          ((job-id priority?)
            (let ((current-processes
                   (hash-count (const #t) processes)))
              (when (< current-processes
-                     max-processes)
+                      (if priority?
-              (fork-and-process-job job-args))))
+                          ;; For priority jobs, burst up to twice the number
+                          ;; of max processes
+                          (* 2 max-processes)
+                          max-processes))
+               (fork-and-process-job (list job-id))))))
         jobs)))
     (unless (eq? 0 (sleep 15))
       (exit 0))))

guix-data-service/jobs/load-new-guix-revision.scm

@@ -719,10 +719,21 @@ ORDER BY load_new_guix_revision_jobs.id DESC")
    (list id)))
 
 (define (fetch-unlocked-jobs conn)
-  (exec-query
+  (define query "
-   conn
+SELECT
-   "
+  id,
-SELECT id FROM load_new_guix_revision_jobs
+  commit IN (
+    SELECT commit FROM (
+      SELECT DISTINCT ON (name)
+        name, git_branches.commit
+      FROM git_branches
+      WHERE
+        git_branches.git_repository_id = load_new_guix_revision_jobs.git_repository_id AND
+        git_branches.commit IS NOT NULL
+      ORDER BY name, datetime DESC
+    ) branches_and_latest_commits
+  ) AS latest_branch_commit
+FROM load_new_guix_revision_jobs
 WHERE
   succeeded_at IS NULL AND
   NOT EXISTS (
@@ -731,8 +742,16 @@ WHERE
     -- Skip jobs that have failed, to avoid trying them over and over again
     WHERE job_id = load_new_guix_revision_jobs.id AND event = 'failure'
   )
-ORDER BY id DESC
+ORDER BY latest_branch_commit DESC, id DESC
-FOR NO KEY UPDATE SKIP LOCKED"))
+FOR NO KEY UPDATE OF load_new_guix_revision_jobs
+SKIP LOCKED")
+
+  (map
+   (match-lambda
+     ((id priority)
+      (list id
+            (string=? priority "t"))))
+   (exec-query conn query)))
 
 (define (process-load-new-guix-revision-job id)
   (with-postgresql-connection