Encoder refactor, new tests and proper non-file stream handling

4 files changed, 140 insertions, 30 deletions

diff --git a/tests/backend-tests.lisp b/tests/backend-tests.lisp
index dc411d5..0f60267 100644
--- a/tests/backend-tests.lisp
+++ b/tests/backend-tests.lisp
@@ -34,15 +34,6 @@
   (test-hash-backend (make-octets 16834 :element 5) 32kib)
   (test-hash-backend (make-octets 96000 :element 5) 32kib))
 
-(defun make-temporary-dir ()
-  (let* ((tmpdir (uiop:temporary-directory))
-         (tmp-tmpdir (make-pathname :directory (serapeum:append1
-                                     (pathname-directory tmpdir)
-                                     (ironclad:byte-array-to-hex-string (ironclad:random-data 10)))
-                         :defaults tmpdir)))
-    (ensure-directories-exist tmp-tmpdir)
-    tmp-tmpdir))
-
 (defmacro test-file-backend (array &optional (secret null-secret))
   `(let ((tmpdir (make-temporary-dir)))
      (unwind-protect
diff --git a/tests/common.lisp b/tests/common.lisp
index 99a85c9..7a9309f 100644
--- a/tests/common.lisp
+++ b/tests/common.lisp
@@ -21,3 +21,18 @@
 (defmacro make-octet-array-with-loop (loop)
   `(let ((seq ,loop))
      (make-array (length seq) :element-type '(unsigned-byte 8) :initial-contents seq) ))
+
+(defun make-temporary-dir ()
+  (let* ((tmpdir (uiop:temporary-directory))
+         (tmp-tmpdir (make-pathname :directory (serapeum:append1
+                                     (pathname-directory tmpdir)
+                                     (ironclad:byte-array-to-hex-string (ironclad:random-data 10)))
+                         :defaults tmpdir)))
+    (ensure-directories-exist tmp-tmpdir)
+    tmp-tmpdir))
+
+(defmacro with-temporary-dir (sym &body expr)
+  `(let ((,sym (make-temporary-dir)))
+     (unwind-protect
+          (progn ,@expr)
+       (uiop:delete-directory-tree ,sym :validate t))))
diff --git a/tests/decode-tests.lisp b/tests/decode-tests.lisp
index 5053d11..918cd82 100644
--- a/tests/decode-tests.lisp
+++ b/tests/decode-tests.lisp
@@ -96,7 +96,9 @@
   (assert-array-decode (make-octets 16385 :element 8) 1024)
   (assert-array-decode (make-octets 32767 :element 9) 1024)
   (assert-array-decode (make-octets 32768 :element 10) 1024)
-  (assert-array-decode (make-octets 131072 :element 11) 1024))
+  (assert-array-decode (make-octets 131072 :element 11) 1024)
+  (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 40000))))
+    (assert-array-decode buffer 1024)))
 
 (test simple-decoding-32kib
   (assert-array-decode (make-octets 1 :element 2) 32kib)
@@ -104,7 +106,9 @@
   (assert-array-decode (make-octets 32768 :element 2) 32kib)
   (assert-array-decode (make-octets 32769 :element 2) 32kib)
   (assert-array-decode (make-octets 32768 :element 2) 32kib)
-  (assert-array-decode (make-octets 16777216 :element 2) 32kib))
+  (assert-array-decode (make-octets 16777216 :element 2) 32kib)
+  (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 70000))))
+    (assert-array-decode buffer 32kib)))
 
 (test proper-return-values
   (assert-bytes-read (make-octets 1 :element 3) 1024 (1))
@@ -127,12 +131,6 @@
           (stream (eris-decode read-capability #'hashtable-decode)))
      (setf (stream-file-position stream) ,pos)
      (stream-read-sequence stream buf 0 (length buf))
-     ;; (print (pos (buffer stream)))
-     ;; (print (+ 24 ,buffer-pos))
-     ;; (print (pos stream))
-     ;; (print (+ 24 ,pos))
-     ;; (print buf)
-     ;; (print ,array-at-pos)
      (is (and
           (eql (eris::pos (eris::buffer stream))
                (+ 24 ,buffer-pos))
@@ -200,7 +198,9 @@
   (assert-length (make-array 1024 :element-type '(unsigned-byte 8) :initial-element 2) 1024)
   (assert-length (make-array 2048 :element-type '(unsigned-byte 8) :initial-element 2) 1024)
   (assert-length (make-array 16383 :element-type '(unsigned-byte 8) :initial-element 2) 1024)
-  (assert-length (make-array 16384 :element-type '(unsigned-byte 8) :initial-element 2) 1024))
+  (assert-length (make-array 16384 :element-type '(unsigned-byte 8) :initial-element 2) 1024)
+  (for-all ((buffer (gen-buffer :length (gen-integer :min 0 :max 40000))))
+    (assert-length buffer 1024)))
 
 
 (defmacro assert-read-byte (array block-size)
diff --git a/tests/encode-tests.lisp b/tests/encode-tests.lisp
index abbeb0d..b4c6892 100644
--- a/tests/encode-tests.lisp
+++ b/tests/encode-tests.lisp
@@ -16,6 +16,7 @@
 (in-package :eris/test)
 (def-suite* encoding-tests :in eris-tests)
 
+
 (defmacro check-urn (data block-size urn &key (secret null-secret))
   `(let ((urn ,urn)
          (vector-encode (read-capability-to-urn
@@ -71,17 +72,120 @@
                    ,urn)))))
 
 (test 100MiB
-  (large-content-test (make-array 24 :element-type '(unsigned-byte 8)
-                                     :initial-contents
-                                     #(49 48 48 77 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 49 75 105 66 41))
-                      1024
-                      "urn:eris:BIC6F5EKY2PMXS2VNOKPD3AJGKTQBD3EXSCSLZIENXAXBM7PCTH2TCMF5OKJWAN36N4DFO6JPFZBR3MS7ECOGDYDERIJJ4N5KAQSZS67YY"
-                      104857600))
+  (large-content-test
+   (make-array 24 :element-type '(unsigned-byte 8)
+                  :initial-contents
+                  #(49 48 48 77 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 49 75 105 66 41))
+   1024
+   "urn:eris:BIC6F5EKY2PMXS2VNOKPD3AJGKTQBD3EXSCSLZIENXAXBM7PCTH2TCMF5OKJWAN36N4DFO6JPFZBR3MS7ECOGDYDERIJJ4N5KAQSZS67YY"
+   104857600))
 
 (test 1GiB
-  (large-content-test (make-array 23 :element-type '(unsigned-byte 8)
-                                     :initial-contents
-                                     #(49 71 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 51 50 75 105 66 41))
-                      32kib
-                      "urn:eris:B4BL4DKSEOPGMYS2CU2OFNYCH4BGQT774GXKGURLFO5FDXAQQPJGJ35AZR3PEK6CVCV74FVTAXHRSWLUUNYYA46ZPOPDOV2M5NVLBETWVI"
-                      1073741824))
+  (large-content-test
+   (make-array 23 :element-type '(unsigned-byte 8)
+                  :initial-contents
+                  #(49 71 105 66 32 40 98 108 111 99 107 32 115 105 122 101 32 51 50 75 105 66 41))
+   32kib
+   "urn:eris:B4BL4DKSEOPGMYS2CU2OFNYCH4BGQT774GXKGURLFO5FDXAQQPJGJ35AZR3PEK6CVCV74FVTAXHRSWLUUNYYA46ZPOPDOV2M5NVLBETWVI"
+   1073741824))
+
+
+(defmacro encode-consensus-test (tmpdir data block-size &key (secret (random-data 32)))
+  "Test if all the eris-encode methods give the same results."
+  `(let ((pathname-encode
+           (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16))
+                                            ,tmpdir)))
+             (with-open-file (f pathname
+                                :direction :output
+                                :element-type 'serapeum:octet
+                                :if-does-not-exist :create)
+               (write-sequence ,data f))
+             (read-capability-to-urn
+              (eris-encode pathname
+                           ,block-size
+                           (lambda (block ref) (declare (ignore ref)) block)
+                           :secret ,secret))))
+         (file-stream-encode
+           (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16))
+                                            ,tmpdir)))
+             (with-open-file (f pathname
+                                :direction :output
+                                :element-type 'serapeum:octet
+                                :if-does-not-exist :create)
+               (write-sequence ,data f))
+             (read-capability-to-urn
+              (with-open-file (f pathname :direction :input
+                                          :element-type 'serapeum:octet)
+                (eris-encode f
+                             ,block-size
+                             (lambda (block ref) (declare (ignore ref)) block)
+                             :secret ,secret)))))
+         (vector-encode
+           (read-capability-to-urn
+            (eris-encode ,data
+                         ,block-size
+                         (lambda (block ref) (declare (ignore ref)) block)
+                         :secret ,secret)))
+         (stream-encode
+           (read-capability-to-urn
+            (with-octet-input-stream (stream ,data)
+              (eris-encode stream
+                           ,block-size
+                           (lambda (block ref) (declare (ignore ref)) block)
+                           :secret ,secret)))))
+     (is (serapeum:equalp* vector-encode stream-encode pathname-encode file-stream-encode))))
+
+(test encoding-consensus-tests
+  (with-temporary-dir tdir
+    (encode-consensus-test tdir (make-octets 1 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 512 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 1023 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 1024 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 16383 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 16384 :element 2) 1024)
+    (encode-consensus-test tdir (make-octets 1024 :element 2) 32kib)
+    (encode-consensus-test tdir (make-octets 32767 :element 2) 32kib)
+    (encode-consensus-test tdir (make-octets 32768 :element 2) 32kib)
+    (encode-consensus-test tdir (make-octets 64000 :element 2) 32kib)
+    (for-all ((buffer (gen-buffer :length (gen-integer :min 1 :max 70000))))
+      (encode-consensus-test tdir buffer 1024)
+      (encode-consensus-test tdir buffer 32kib))))
+
+
+(test encoding-nothing
+  (with-temporary-dir tdir
+    (encode-consensus-test tdir (make-octets 0) 1024)
+    (encode-consensus-test tdir (make-octets 0) 32kib)))
+
+
+(defmacro encoding-file-pos (tmpdir data pos block-size &key (secret null-secret))
+  `(let ((vector-encode (read-capability-to-urn
+                         (eris-encode (subseq ,data ,pos)
+                                      ,block-size
+                                      (lambda (block ref) (declare (ignore ref)) block)
+                                      :secret ,secret)))
+         (file-stream-encode
+           (let ((pathname (merge-pathnames (crypto:byte-array-to-hex-string (crypto:random-data 16))
+                                            ,tmpdir)))
+             (with-open-file (f pathname
+                                :direction :output
+                                :element-type 'serapeum:octet
+                                :if-does-not-exist :create)
+               (write-sequence ,data f))
+             (read-capability-to-urn
+              (with-open-file (f pathname :direction :input
+                                          :element-type 'serapeum:octet)
+                (file-position f ,pos)
+                (eris-encode f
+                             ,block-size
+                             (lambda (block ref) (declare (ignore ref)) block)
+                             :secret ,secret))))))
+     (is (equalp vector-encode file-stream-encode))))
+
+(test encoding-file-position-tests
+  (with-temporary-dir tdir
+   (encoding-file-pos tdir (make-octets 1024 :element 2) 512 1024)
+   (encoding-file-pos tdir (make-octets 1024 :element 2) 1023 1024)
+   (encoding-file-pos tdir (make-octets 1024 :element 2) 1 1024)
+   (encoding-file-pos tdir (make-octets 32000 :element 2) 1673 32kib)
+   (encoding-file-pos tdir (make-octets 32000 :element 2) 31999 32kib)))