pchiusano avatar pchiusano committed 073c2c3

fixed bugs with takeUntil and added additional tests

Comments (0)

Files changed (2)

src/main/scala/nomo/Parser.scala

 
     /** Apply this `Parser` to the given sequence of tokens, then feed `EOF` to extract
       * a result. */
-    def apply(i: Iterable[I])(implicit user: U): Result[X,E,U,A] =
-      feedChunked(i, annotator(user)) match { case (p2, a) => p2.result(a) }
+    def apply(i: Iterable[I], chunkSize: Int = 5000)(implicit user: U): Result[X,E,U,A] =
+      feedChunked(i, annotator(user), chunkSize) match { case (p2, a) => p2.result(a) }
 
     /** Extract a result from this parser, by feeding it EOF. */
     def result(ann: Accumulator[I,X,U]): Result[X,E,U,A] = 
     go(MonotypicW(container(List()), container), s0)
   }
 
+  def take(n: Int): Parser[MonotypicW[F,I]] = 
+    takeUnfold(n)((i,n) => (n > 0, n-1)).map(_._1)
+
   /** Reads until the given string is a prefix of the remaining input, or until EOF.
-    * This parser does not consume the given string. */
-  def takeUntil[S](f: F): Parser[MonotypicW[F,I]] = {
-    val s = MonotypicW(f,container)
+    * This parser does not consume the given string; use `takeThrough` to consume
+    * the delimiter as well. */
+  def takeUntil(delimiter: F): Parser[MonotypicW[F,I]] = {
+    val K = container.size(delimiter)
     def go(acc: MonotypicW[F,I], lookback: MonotypicW[F,I]): Parser[MonotypicW[F,I]] = 
     Cont((i,a) => 
       i match {
         }
         case ChunkInput(chunk) => {
           val text = lookback ++ chunk 
-          lazy val notFound = chunk.slice(0, chunk.size - s.size)
-          lazy val possible = chunk.slice(chunk.size - s.size)
-          text.indexOf(f) match {
-            case None => suspendS { (go(acc ++ notFound, possible), a(notFound)) }
-            case Some(i) => suspendS { 
-              val (h,t) = (text.slice(0,i), text.slice(i))
-              val ah = a(h)
-              (Done(
-                Success(acc ++ h), 
-                IndexedSeq(ChunkInput(t)),
-                ah 
-              ), ah)
+          if (text.size < K)
+            suspendS { (go(acc, text), a) }
+          else {
+            lazy val consumed = text.slice(0, text.size - K)
+            lazy val lookback2 = text.slice(text.size - K)
+            text.indexOf(delimiter) match {
+              case None => suspendS { 
+                (go(acc ++ consumed, lookback2), a(consumed)) }
+              case Some(i) => suspendS { 
+                val (h,t) = (text.slice(0,i), text.slice(i))
+                val ah = a(h)
+                (Done(
+                  Success(acc ++ h), 
+                  IndexedSeq(ChunkInput(t)),
+                  ah 
+                ), ah)
+              }
             }
           }
         }
     val empty = MonotypicW(container(List()), container)
     go(empty, empty)
   }
+  
+  /** Like `takeUntil`, but consumes the delimiter if encountered. */
+  def takeThrough(delimiter: F): Parser[MonotypicW[F,I]] = 
+    takeUntil(delimiter) << take(container.size(delimiter)) 
 
   /** Parser which consumes no input but returns the current accumulated position. */
   def position: Parser[X] = Cont((i,a) => suspendS { (Done(Success(a.get), IndexedSeq(i), a), a) })

src/test/scala/nomo/ParserSpec.scala

 
   property("takeUntil") = secure {
     val s = "ooga booga end"
-    //P.takeUntil("end")(s).get.get == "ooga booga " &&
-    //P.takeUntil("end")(s).position == P.takeWhile(_ != 'e')(s).position &&
-    P.takeUntil("@#$%")(s).position ?= P.takeWhile(_ => true)(s).position //&&
-    //true
-    //P.takeUntil("end")((0 to 10000).mkString + "end ").isSuccess
+    P.takeUntil("end")(s, chunkSize = 2).get.get == "ooga booga " &&
+    P.takeUntil("end")(s).position == P.takeWhile(_ != 'e')(s).position &&
+    P.takeUntil("@#$%")(s).position == P.takeWhile(_ => true)(s).position &&
+    P.takeUntil("end")((0 to 10000).mkString + "end ").isSuccess
   }
 
+  property("takeThrough") = secure {
+    val s = "ooga booga end"
+    P.takeThrough("end")(s, chunkSize = 2).position == 
+    P.takeWhile(_ => true)(s).position
+  }
+
+  property("take") = forAll((n: Size) => 
+    P.take(n.get)(List.fill(n.get*2)("a").mkString, chunkSize = 4).get.size == 
+      n.get &&
+    P.take(n.get)(List.fill((n.get-1) max 0)("a").mkString).get.size == 
+      ((n.get-1) max 0)
+  )
+
   property("map") = forAll((s: String) => 
     P.word(s).map(_.get.reverse)(s).get == s.reverse)
 
     true 
   }
 
-
   property("many-eof") = secure {
     val p = (word("hello") << takeWhile(_.isWhitespace)).many1 << eof
     p("hello   hello hello").isSuccess
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.