From da0e1aa69defa7cbc87209966c751918f523f1fb Mon Sep 17 00:00:00 2001 From: Piotr Szarmanski Date: Sat, 5 Aug 2023 22:57:30 +0200 Subject: Encoder refactor, new tests and proper non-file stream handling --- tests/backend-tests.lisp | 9 ---- tests/common.lisp | 15 ++++++ tests/decode-tests.lisp | 18 +++---- tests/encode-tests.lisp | 128 ++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 140 insertions(+), 30 deletions(-) (limited to 'tests') diff --git a/tests/backend-tests.lisp b/tests/backend-tests.lisp index dc411d5..0f60267 100644 --- a/tests/backend-tests.lisp +++ b/tests/backend-tests.lisp @@ -34,15 +34,6 @@ (test-hash-backend (make-octets 16834 :element 5) 32kib) (test-hash-backend (make-octets 96000 :element 5) 32kib)) -(defun make-temporary-dir () - (let* ((tmpdir (uiop:temporary-directory)) - (tmp-tmpdir (make-pathname :directory (serapeum:append1 - (pathname-directory tmpdir) - (ironclad:byte-array-to-hex-string (ironclad:random-data 10))) - :defaults tmpdir))) - (ensure-directories-exist tmp-tmpdir) - tmp-tmpdir)) - (defmacro test-file-backend (array &optional (secret null-secret)) `(let ((tmpdir (make-temporary-dir))) (unwind-protect diff --git a/tests/common.lisp b/tests/common.lisp index 99a85c9..7a9309f 100644 --- a/tests/common.lisp +++ b/tests/common.lisp @@ -21,3 +21,18 @@ (defmacro make-octet-array-with-loop (loop) `(let ((seq ,loop)) (make-array (length seq) :element-type '(unsigned-byte 8) :initial-contents seq) )) + +(defun make-temporary-dir () + (let* ((tmpdir (uiop:temporary-directory)) + (tmp-tmpdir (make-pathname :directory (serapeum:append1 + (pathname-directory tmpdir) + (ironclad:byte-array-to-hex-string (ironclad:random-data 10))) + :defaults tmpdir))) + (ensure-directories-exist tmp-tmpdir) + tmp-tmpdir)) + +(defmacro with-temporary-dir (sym &body expr) + `(let ((,sym (make-temporary-dir))) + (unwind-protect + (progn ,@expr) + (uiop:delete-directory-tree ,sym :validate t)))) diff --git a/tests/decode-tests.lisp b/tests/decode-tests.lisp index 5053d11..918cd82 100644 --- a/tests/decode-tests.lisp +++ b/tests/decode-tests.lisp @@ -96,7 +96,9 @@ (assert-array-decode (make-octets 16385 :element 8) 1024) (assert-array-decode (make-octets 32767 :element 9) 1024) (assert-array-decode (make-octets 32768 :element 10) 1024) - (assert-array-decode (make-octets 131072 :element 11) 1024)) + (assert-array-decode (make-octets 131072 :element 11) 1024) + (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 40000)))) + (assert-array-decode buffer 1024))) (test simple-decoding-32kib (assert-array-decode (make-octets 1 :element 2) 32kib) @@ -104,7 +106,9 @@ (assert-array-decode (make-octets 32768 :element 2) 32kib) (assert-array-decode (make-octets 32769 :element 2) 32kib) (assert-array-decode (make-octets 32768 :element 2) 32kib) - (assert-array-decode (make-octets 16777216 :element 2) 32kib)) + (assert-array-decode (make-octets 16777216 :element 2) 32kib) + (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 70000)))) + (assert-array-decode buffer 32kib))) (test proper-return-values (assert-bytes-read (make-octets 1 :element 3) 1024 (1)) @@ -127,12 +131,6 @@ (stream (eris-decode read-capability #'hashtable-decode))) (setf (stream-file-position stream) ,pos) (stream-read-sequence stream buf 0 (length buf)) - ;; (print (pos (buffer stream))) - ;; (print (+ 24 ,buffer-pos)) - ;; (print (pos stream)) - ;; (print (+ 24 ,pos)) - ;; (print buf) - ;; (print ,array-at-pos) (is (and (eql (eris::pos (eris::buffer stream)) (+ 24 ,buffer-pos)) @@ -200,7 +198,9 @@ (assert-length (make-array 1024 :element-type '(unsigned-byte 8) :initial-element 2) 1024) (assert-length (make-array 2048 :element-type '(unsigned-byte 8) :initial-element 2) 1024) (assert-length (make-array 16383 :element-type '(unsigned-byte 8) :initial-element 2) 1024) - (assert-length (make-array 16384 :element-type '(unsigned-byte 8) :initial-element 2) 1024)) + (assert-length (make-array 16384 :element-type '(unsigned-byte 8) :initial-element 2) 1024) + (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 40000)))) + (assert-length buffer 1024))) (defmacro assert-read-byte (array block-size) diff --git a/tests/encode-tests.lisp b/tests/encode-tests.lisp index abbeb0d..b4c6892 100644 --- a/tests/encode-tests.lisp +++ b/tests/encode-tests.lisp @@ -16,6 +16,7 @@ (in-package :eris/test) (def-suite* encoding-tests :in eris-tests) + (defmacro check-urn (data block-size urn &key (secret null-secret)) `(let ((urn ,urn) (vector-encode (read-capability-to-urn @@ -71,17 +72,120 @@ ,urn))))) (test 100MiB - (large-content-test (make-array 24 :element-type '(unsigned-byte 8) - :initial-contents - #(49 48 48 77 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 49 75 105 66 41)) - 1024 - "urn:eris:BIC6F5EKY2PMXS2VNOKPD3AJGKTQBD3EXSCSLZIENXAXBM7PCTH2TCMF5OKJWAN36N4DFO6JPFZBR3MS7ECOGDYDERIJJ4N5KAQSZS67YY" - 104857600)) + (large-content-test + (make-array 24 :element-type '(unsigned-byte 8) + :initial-contents + #(49 48 48 77 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 49 75 105 66 41)) + 1024 + "urn:eris:BIC6F5EKY2PMXS2VNOKPD3AJGKTQBD3EXSCSLZIENXAXBM7PCTH2TCMF5OKJWAN36N4DFO6JPFZBR3MS7ECOGDYDERIJJ4N5KAQSZS67YY" + 104857600)) (test 1GiB - (large-content-test (make-array 23 :element-type '(unsigned-byte 8) - :initial-contents - #(49 71 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 51 50 75 105 66 41)) - 32kib - "urn:eris:B4BL4DKSEOPGMYS2CU2OFNYCH4BGQT774GXKGURLFO5FDXAQQPJGJ35AZR3PEK6CVCV74FVTAXHRSWLUUNYYA46ZPOPDOV2M5NVLBETWVI" - 1073741824)) + (large-content-test + (make-array 23 :element-type '(unsigned-byte 8) + :initial-contents + #(49 71 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 51 50 75 105 66 41)) + 32kib + "urn:eris:B4BL4DKSEOPGMYS2CU2OFNYCH4BGQT774GXKGURLFO5FDXAQQPJGJ35AZR3PEK6CVCV74FVTAXHRSWLUUNYYA46ZPOPDOV2M5NVLBETWVI" + 1073741824)) + + +(defmacro encode-consensus-test (tmpdir data block-size &key (secret (random-data 32))) + "Test if all the eris-encode methods give the same results." + `(let ((pathname-encode + (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16)) + ,tmpdir))) + (with-open-file (f pathname + :direction :output + :element-type 'serapeum:octet + :if-does-not-exist :create) + (write-sequence ,data f)) + (read-capability-to-urn + (eris-encode pathname + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret)))) + (file-stream-encode + (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16)) + ,tmpdir))) + (with-open-file (f pathname + :direction :output + :element-type 'serapeum:octet + :if-does-not-exist :create) + (write-sequence ,data f)) + (read-capability-to-urn + (with-open-file (f pathname :direction :input + :element-type 'serapeum:octet) + (eris-encode f + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret))))) + (vector-encode + (read-capability-to-urn + (eris-encode ,data + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret))) + (stream-encode + (read-capability-to-urn + (with-octet-input-stream (stream ,data) + (eris-encode stream + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret))))) + (is (serapeum:equalp* vector-encode stream-encode pathname-encode file-stream-encode)))) + +(test encoding-consensus-tests + (with-temporary-dir tdir + (encode-consensus-test tdir (make-octets 1 :element 2) 1024) + (encode-consensus-test tdir (make-octets 512 :element 2) 1024) + (encode-consensus-test tdir (make-octets 1023 :element 2) 1024) + (encode-consensus-test tdir (make-octets 1024 :element 2) 1024) + (encode-consensus-test tdir (make-octets 16383 :element 2) 1024) + (encode-consensus-test tdir (make-octets 16384 :element 2) 1024) + (encode-consensus-test tdir (make-octets 1024 :element 2) 32kib) + (encode-consensus-test tdir (make-octets 32767 :element 2) 32kib) + (encode-consensus-test tdir (make-octets 32768 :element 2) 32kib) + (encode-consensus-test tdir (make-octets 64000 :element 2) 32kib) + (for-all ((buffer (gen-buffer :length (gen-integer :min 1 :max 70000)))) + (encode-consensus-test tdir buffer 1024) + (encode-consensus-test tdir buffer 32kib)))) + + +(test encoding-nothing + (with-temporary-dir tdir + (encode-consensus-test tdir (make-octets 0) 1024) + (encode-consensus-test tdir (make-octets 0) 32kib))) + + +(defmacro encoding-file-pos (tmpdir data pos block-size &key (secret null-secret)) + `(let ((vector-encode (read-capability-to-urn + (eris-encode (subseq ,data ,pos) + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret))) + (file-stream-encode + (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16)) + ,tmpdir))) + (with-open-file (f pathname + :direction :output + :element-type 'serapeum:octet + :if-does-not-exist :create) + (write-sequence ,data f)) + (read-capability-to-urn + (with-open-file (f pathname :direction :input + :element-type 'serapeum:octet) + (file-position f ,pos) + (eris-encode f + ,block-size + (lambda (block ref) (declare (ignore ref)) block) + :secret ,secret)))))) + (is (equalp vector-encode file-stream-encode)))) + +(test encoding-file-position-tests + (with-temporary-dir tdir + (encoding-file-pos tdir (make-octets 1024 :element 2) 512 1024) + (encoding-file-pos tdir (make-octets 1024 :element 2) 1023 1024) + (encoding-file-pos tdir (make-octets 1024 :element 2) 1 1024) + (encoding-file-pos tdir (make-octets 32000 :element 2) 1673 32kib) + (encoding-file-pos tdir (make-octets 32000 :element 2) 31999 32kib))) -- cgit v1.2.3