Source

ocaml-indent / tokenstr.ml

Full commit
open Pos
open Xparser
open LazyList

type 'a info = {
  token : 'a;
  region : Region.t;
  substr : string;
  space : Region.t * string;
}

let of_channel ic = 
  try
    let reader = Reader.create_from_channel ic in
    let rec loop last_region = 
      let token = 
        try
          Reader.lex reader Xlexer.token 
        with
        | Xlexer.Error (e, _loc) ->
            Format.eprintf "%a@." Xlexer.report_error e;
            assert false
      in
      let region = Reader.region reader in
      (* token's string *)
      let substr = Reader.current_substring reader in

      let space_between = 
        let last_end = (snd last_region).Position.pos_cnum in
        Reader.substring 
          reader last_end ((fst region).Position.pos_cnum - last_end)
      in
      let space_between_region = (snd last_region, fst region) in

      Cons ({ token; region; substr; space = space_between_region, space_between },
            begin match token with
            | EOF -> Lazy.lazy_from_val Null
            | _ -> lazy (loop region)
            end)
    in
    lazy (loop Region.zero)
  with
  | e -> raise e