diff options
Diffstat (limited to 'src/eris-parallel.lisp')
| -rw-r--r-- | src/eris-parallel.lisp | 94 |
1 files changed, 94 insertions, 0 deletions
diff --git a/src/eris-parallel.lisp b/src/eris-parallel.lisp new file mode 100644 index 0000000..7c2f702 --- /dev/null +++ b/src/eris-parallel.lisp @@ -0,0 +1,94 @@ +;; This file is part of eris-cl. +;; Copyright (C) 2023 Piotr Szarmański + +;; eris-cl is free software: you can redistribute it and/or modify it under the +;; terms of the GNU Lesser General Public License as published by the Free +;; Software Foundation, either version 3 of the License, or (at your option) any +;; later version. + +;; eris-cl is distributed in the hope that it will be useful, but WITHOUT ANY +;; WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +;; A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License along with +;; eris-cl. If not, see <https://www.gnu.org/licenses/>. + +(in-package :eris) + + +(defun split-into-n (i parts size) + "Splits an integer I into PARTS parts. Returns a vector of length 1- PARTS with +those parts.." + (declare (type integer i parts size)) + (let* ((k (* size (truncate (/ (/ i size) parts)))) + (vec (make-array + parts + :element-type 'integer + :initial-element k))) + (nlet rec ((remaining (- i (* k parts))) + (i 0)) + (if (< remaining size) + (setf (aref vec (1- parts)) (+ remaining (aref vec (1- parts)))) + (progn (setf (aref vec i) (+ size (aref vec i))) + (rec (- remaining size) (1+ i))))) + vec)) + +(defgeneric p/eris-encode (input block-size output-function &key secret &allow-other-keys) + (:documentation "Encode INPUT in parallel using LPARALLEL. Requires lparallel:*kernel* to be bound.")) + +;; eris-create-tree is not parallelized because it is insignificant on any large +;; input; for 32kib blocks, there's about 512 level 0 blocks for any higher +;; level block, almost three orders of magnitude. +(defmethod p/eris-encode ((input vector) block-size output-function + &key (secret null-secret) &allow-other-keys) + (let ((v (split-into-n (length input) (lparallel:kernel-worker-count) block-size))) + (eris-create-tree + (apply #'concatenate + 'simple-vector + (map + 'list + #'lparallel:force + ;; Recursion to avoid demons + (nreverse + (nlet rec ((i 0) (k 0) (l nil)) + (if (eql i (length v)) l + (rec + (1+ i) + (+ k (aref v i)) + (cons (lparallel:future + (chunk-array + input block-size output-function secret + :start k :end (+ k (aref v i)) + :pad (if (eq i (1- (length v))) t nil))) + l))))))) + block-size output-function))) + +(defmethod p/eris-encode ((input pathname) block-size output-function + &key (secret null-secret) &allow-other-keys) + (let ((v (split-into-n (file-size input) (lparallel:kernel-worker-count) block-size))) + (eris-create-tree + (apply #'concatenate + 'simple-vector + (map + 'list + #'lparallel:force + (nreverse + (nlet rec ((i 0) (k 0) (l nil)) + (if (eql i (length v)) l + (rec + (1+ i) + (+ k (aref v i)) + (cons (lparallel:future + (with-open-file (f input :element-type 'octet) + (file-position f k) + (chunk-stream + f block-size output-function (aref v i) secret + :pad (if (eq i (1- (length v))) t nil)))) + l))))))) + block-size output-function))) + +;; No methods for streams. A stream implementation could be done by allocating n +;; buffers, reading sequentially into each buffer and chunk-array'ing them until +;; eof, but it may be quite unoptimal, depending on the stream, buffer size, +;; etc. + |