Add buffering to the log handling for jobs

To increase the log output speed, avoiding inserting one character at a time
in to the database.
This commit is contained in:
Christopher Baines 2019-08-31 12:10:54 +01:00
parent 8c741c569b
commit bf469504eb

View file

@ -40,6 +40,7 @@
(current-output-port)) (current-output-port))
(define id 0) (define id 0)
(define buffer "")
(define (insert job_id s) (define (insert job_id s)
(exec-query (exec-query
@ -50,9 +51,13 @@
(list (number->string id) job_id s))) (list (number->string id) job_id s)))
(define (log-string s) (define (log-string s)
(set! id (+ 1 id)) ; increment id (if (string-contains s "\n")
(insert job-id s) (let ((output (string-append buffer s)))
(display s output-port)) (set! id (+ 1 id)) ; increment id
(set! buffer "") ; clear the buffer
(insert job-id output)
(display output output-port))
(set! buffer (string-append buffer s))))
;; TODO, this is useful when re-running jobs, but I'm not sure that should ;; TODO, this is useful when re-running jobs, but I'm not sure that should
;; be a thing, jobs should probably be only attempted once. ;; be a thing, jobs should probably be only attempted once.
@ -61,18 +66,21 @@
"DELETE FROM load_new_guix_revision_job_log_parts WHERE job_id = $1" "DELETE FROM load_new_guix_revision_job_log_parts WHERE job_id = $1"
(list job-id)) (list job-id))
(make-soft-port (let ((port
(vector (lambda (c) (make-soft-port
(log-string (string c))) (vector (lambda (c)
log-string (set! buffer (string-append buffer (string c))))
(lambda () log-string
(force-output output-port)) (lambda ()
#f ; fetch one character (force-output output-port))
(lambda () #f ; fetch one character
;; close port (lambda ()
#f) ;; close port
#f) ; number of characters that can be read #f)
"w")) #f) ; number of characters that can be read
"w")))
(setvbuf port 'line)
port))
(define* (log-for-job conn job-id (define* (log-for-job conn job-id
#:key #:key