Make resource pool changes and add parallelism limiter
Some checks failed
/ test (push) Failing after 13s
Some checks failed
/ test (push) Failing after 13s
This was motivated by trying to allow for completely cleaning up resource pools, which involved removing their use of fiberize which currently has no destroy mechanism. As part of this, there's a new parallelism limiter mechanism using resource pools rather than fibers, and also a fixed size resource pool. The tests now drain? and destroy the resource pools to check cleaning up.
This commit is contained in:
parent
edf62414ee
commit
d3e2352ac0
5 changed files with 614 additions and 98 deletions
|
@ -20,6 +20,8 @@
|
|||
(define-module (knots parallelism)
|
||||
#:use-module (srfi srfi-1)
|
||||
#:use-module (srfi srfi-71)
|
||||
#:use-module (srfi srfi-9)
|
||||
#:use-module (srfi srfi-9 gnu)
|
||||
#:use-module (ice-9 match)
|
||||
#:use-module (ice-9 control)
|
||||
#:use-module (ice-9 exceptions)
|
||||
|
@ -27,6 +29,7 @@
|
|||
#:use-module (fibers channels)
|
||||
#:use-module (fibers operations)
|
||||
#:use-module (knots)
|
||||
#:use-module (knots resource-pool)
|
||||
#:export (fibers-batch-map
|
||||
fibers-map
|
||||
|
||||
|
@ -38,7 +41,13 @@
|
|||
fibers-parallel
|
||||
fibers-let
|
||||
|
||||
fiberize))
|
||||
fiberize
|
||||
|
||||
make-parallelism-limiter
|
||||
parallelism-limiter?
|
||||
destroy-parallelism-limiter
|
||||
call-with-parallelism-limiter
|
||||
with-parallelism-limiter))
|
||||
|
||||
(define (defer-to-parallel-fiber thunk)
|
||||
(let ((reply (make-channel)))
|
||||
|
@ -287,3 +296,30 @@
|
|||
(('result . vals) (apply values vals))
|
||||
(('exception exn)
|
||||
(raise-exception exn))))))
|
||||
|
||||
(define-record-type <parallelism-limiter>
|
||||
(make-parallelism-limiter-record resource-pool)
|
||||
parallelism-limiter?
|
||||
(resource-pool parallelism-limiter-resource-pool))
|
||||
|
||||
(define* (make-parallelism-limiter limit #:key (name "unnamed"))
|
||||
(make-parallelism-limiter-record
|
||||
(make-fixed-size-resource-pool
|
||||
(iota limit)
|
||||
#:name name)))
|
||||
|
||||
(define (destroy-parallelism-limiter parallelism-limiter)
|
||||
(destroy-resource-pool
|
||||
(parallelism-limiter-resource-pool
|
||||
parallelism-limiter)))
|
||||
|
||||
(define* (call-with-parallelism-limiter parallelism-limiter thunk)
|
||||
(call-with-resource-from-pool
|
||||
(parallelism-limiter-resource-pool parallelism-limiter)
|
||||
(lambda _
|
||||
(thunk))))
|
||||
|
||||
(define-syntax-rule (with-parallelism-limiter parallelism-limiter exp ...)
|
||||
(call-with-parallelism-limiter
|
||||
parallelism-limiter
|
||||
(lambda () exp ...)))
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue