Commits

Anonymous committed 68571e4

added fully integrated smoke tests and some unit tests for console reporter

Comments (0)

Files changed (6)

reporters/console.scm

 
   (define (report-pending-verifications)
     (unless (null? pending-verifications)
-      (fmt #t (fmt-bold "Pending:") nl)
+      (fmt #t ((colorize fmt-bold) "Pending:") nl)
       (for-each report-pending-verification pending-verifications)))
 
   (define (report-pending-verification result)
     (let* ((reason (meta-data-get (verification-result-subject result) 'pending)))
-      (fmt #t (space-to 4) ((colorize fmt-yellow) (cat (extract-description result) " is pending") nl))
-      (if reason
-          (fmt #t (space-to 4) ((colorize fmt-yellow) (cat "REASON: " reason)) nl))))
+      (fmt #t (space-to 4) ((colorize fmt-yellow) (cat (extract-description result) " is pending")) nl)
+      (when (string? reason)
+          (fmt #t (space-to 4) ((colorize fmt-yellow) (cat "REASON: " reason)) nl))
+      (fmt #t nl)))
 
   (define (extract-description result)
     (or (meta-data-get (verification-result-subject result) 'description)
 
   (define (report-failed-verifications)
     (unless (null? failed-verifications)
-      (fmt #t (fmt-bold "Failed:") nl)
+      (fmt #t ((colorize fmt-bold) "Failed:") nl)
       (for-each report-failed-verification (reverse failed-verifications))))
 
   (define (report-failed-verification entry)
                (newline)
                (report-details)
                (newline)
-               (report-summary)
-               (_exit (if (failures?) (current-failure-exit-code) (current-success-exit-code))))))
+               (report-summary))
+             (_exit (if (failures?) (current-failure-exit-code) (current-success-exit-code)))))
 
   )

tests/console-integration-colors.scm

+(use veritas veritas-base-reporter veritas-console-reporter)
+
+(reporter-use-colors? #t)
+
+(verify #t)
+
+(verify #t "It succeeds")
+
+(verify #f)
+
+(verify #f "It fails")
+
+(pending
+ (verify #t "It's pending"))
+
+(pending "I'm pending for a reason"
+  (verify #t))
+
+(group "Group1"
+       (verify #t)
+       (group "Group2"
+              (verify #f "This failed")))

tests/console-integration-no-colors.scm

+(use veritas veritas-base-reporter veritas-console-reporter)
+
+(reporter-use-colors? #f)
+
+(verify #t)
+
+(verify #t "It succeeds")
+
+(verify #f)
+
+(verify #f "It fails")
+
+(pending
+ (verify #t "It's pending"))
+
+(pending "I'm pending for a reason"
+  (verify #t))
+
+(group "Group1"
+       (verify #t)
+       (group "Group2"
+              (verify #f "This failed")))

tests/console-reporter-test.scm

 (module veritas-console-reporter-test
   ()
-  (import chicken scheme)
-  (use test veritas veritas-console-reporter)
+  (import chicken scheme irregex ports data-structures posix)
+  (use test veritas veritas-base-reporter veritas-console-reporter advice utils)
 
   (report-status-on-exit #f)
 
+  ;; the console reporter might call exit
+  ;; we don't want this
+
+  (define-syntax without-exit
+    (syntax-rules ()
+      ((_ ?body ...)
+       (let ((id (advise 'around _exit (lambda (proc args) #t))))
+         (handle-exceptions exn
+                            (begin
+                              (unadvise _exit id)
+                              (signal exn))
+
+          (begin ?body ...)
+          (unadvise _exit id))))))
+
   (define-syntax test-contains?
     (syntax-rules ()
       ((_ ?args ...)
        (parameterize ((current-test-comparator irregex-search))
          (test ?args ...)))))
 
-  )
+  (define-syntax capture
+    (syntax-rules ()
+      ((_ ?body ...)
+       (with-output-to-string
+         (lambda () ?body ...)))))
+
+  (define-syntax with-colors
+    (syntax-rules ()
+      ((_  ?body ...)
+       (without-exit
+        (parameterize ((reporter-use-colors? #t))
+          ?body ...)))))
+
+  (define-syntax without-colors
+    (syntax-rules ()
+      ((_ ?body ...)
+       (without-exit
+        (parameterize ((reporter-use-colors? #f))
+          ?body ...)))))
+
+  (define example-output-with-colors #<<EOF
+test
+EOF
+    )
+
+  (define example-output-with-colors
+    "\x1b[32m.\x1b[0m\x1b[32m.\x1b[0m\x1b[31mF\x1b[0m\x1b[31mF\x1b[0m\x1b[33mP\x1b[0m\x1b[33mP\x1b[0m\x1b[32m.\x1b[0m\x1b[31mF\x1b[0m\n\n\x1b[1mPending:\x1b[0m\x1b[0m\n    \x1b[33m(verify #t) is pending\x1b[0m\n    \x1b[33mREASON: I'm pending for a reason\x1b[0m\n\n    \x1b[33mIt's pending is pending\x1b[0m\n\n\n\x1b[1mFailed:\x1b[0m\x1b[0m\n    1) (verify #f)\n\n        \x1b[31mExpected #f to hold\x1b[0m\n\n    2) It fails\n\n        \x1b[31mExpected #f to hold\x1b[0m\n\n    3) This failed\n\n        \x1b[31mExpected #f to hold\x1b[0m\n\n\n\x1b[1mTotal: 8\x1b[0m\x1b[0m \x1b[32m\x1b[1mPassed: 3\x1b[0m\x1b[32m\x1b[0m \x1b[33m\x1b[1mPending: 2\x1b[0m\x1b[33m\x1b[0m \x1b[31m\x1b[1mFailed: 3\x1b[0m\x1b[31m\x1b[0m\n"
+    )
+
+  (define example-output-without-colors
+    #<<EOF
+..FFPP.F
+
+Pending:
+    (verify #t) is pending
+    REASON: I'm pending for a reason
+
+    It's pending is pending
+
+
+Failed:
+    1) (verify #f)
+
+        Expected #f to hold
+
+    2) It fails
+
+        Expected #f to hold
+
+    3) This failed
+
+        Expected #f to hold
+
+
+Total: 8 Passed: 3 Pending: 2 Failed: 3
+
+EOF
+    )
+
+  (define (run-veritas-file path)
+    (receive (i o pid e) (process* "csi" (list "-s" path))
+      (read-all i)))
+
+  (test "Integration with colors"
+        example-output-with-colors
+        (run-veritas-file "console-integration-colors.scm"))
+
+  (test "Integration without colors"
+        example-output-without-colors
+        (run-veritas-file "console-integration-no-colors.scm"))
+
+  ;; unit level
+  (test-group "short reporter"
+              (use-short-formatter)
+              (test-group "successful tests"
+                          (test-group "with colors"
+                                      (test "It prints a green dot"
+                                            "\x1b[32m.\x1b[0m"
+                                            (capture (with-colors  (verify #t "it is true")))))
+
+                          (test-group "without colors"
+                                      (test "It prints a dot"
+                                            "." (capture (without-colors (verify #t "it is true"))))))
+
+
+              (test-group "failung tests"
+                          (test-group "with colors"
+                                      (test "It prints a red F"
+                                            "\x1b[31mF\x1b[0m"
+                                            (capture (with-colors  (verify #f "it is false")))))
+                          (test-group "without colores"
+                                      (test "It prints an F"
+                                            "F"
+                                            (capture (without-colors (verify #f "it is false"))))))
+
+              (test-group "pending tests"
+                          (test-group "with colors"
+                                      (test "It prints a yellow P"
+                                            "\x1b[33mP\x1b[0m"
+                                            (capture (with-colors (pending (verify #t))))))
+                          (test-group "without colors"
+                                      (test "It prints a P"
+                                            "P"
+                                            (capture (without-colors (pending (verify #t))))))))
+
+
+  (test-group "documentation reporter"
+              (use-documentation-formatter)
+              (test-group "successful tests"
+                          (test-group "with colors"
+                                      (test "It prints the expression in green"
+                                            "\x1b[32m ✔ (verify #t)\n\x1b[0m"
+                                            (capture (with-colors (verify #t))))
+                                      (test "It prints the description in green"
+                                             "\x1b[32m ✔ It succeeds\n\x1b[0m"
+                                            (capture (with-colors (verify #t "It succeeds")))))
+
+                          (test-group "without colors"
+                                      (test "it prints the expression"
+                                            " ✔ (verify #t)\n"
+                                            (capture (without-colors (verify #t))))
+                                      (test "it prints the description"
+                                            " ✔ It succeeds\n"
+                                            (capture (without-colors (verify #t "It succeeds"))))))
+
+
+
+              )
+
+)
 
-* FEATURE handle errors in verifications
+* DONE handle errors in verifications
   Catch conditions that accure during verification
   and generate a verification result that has nice information about what accured
 
-* FIX is verifier with negative contexts
-  Add all the messages for the negative context
-* FIX is verifier with predicate
-  (verify 2 (is positive?)) doesn't currently work
-* FEATURE integrate quickcheck-like verification (in progress)
+* DONE integrate quickcheck-like verification
   Establish a means to run verifications against values generated by a generator stream
   It can probably look something like this:
 
   There should be two ways to configure filters
   1) via a paramater
   2) environment variables
+* TESTS add tests for console reporter
+* REFACTOR meta-data used by veritas itself to be prefixed with veritas-
+* FEATURE quickcheck output
+  Add information about how much iterations it took to find a falsification
+* FEATURE add FAILURE
+  Instead of ID for failures write FAILURE. This is easily grepable
  (license "MIT")
  (doc-from-wiki)
  (needs fmt matchable kvlists dissector)
- (test-depends test))
+ (test-depends test advice))