Source

ocaml-indent / tokenstr.ml

open Pos
open Reader
module Parser = Xparser
open Parser
open Planck.Lazylist

type info = {
  token        : Parser.token;
  region       : Region.t;
  substr       : string;
  space        : Region.t * string;
}

type t = (info, unit) zlist * in_channel

let stream = fst
let close (_, ic) = close_in ic

let of_channel ic = 
  let reader = LexReader.create_from_channel ic in
  let rec loop last_region = 
    let token = 
      try
        LexReader.lex reader Lexer.token 
      with
      | Lexer.Error (e, _loc) ->
          Format.eprintf "%a@." Lexer.report_error e;
          assert false
    in
    let region = LexReader.region reader in
    (* token's string *)
    let substr = LexReader.current_substring reader in

    let space_between = 
      let last_end = (snd last_region).Position.pos_cnum in
      LexReader.substring 
        reader last_end ((fst region).Position.pos_cnum - last_end)
    in
    let space_between_region = (snd last_region, fst region) in

    Cons ({ token; region; substr; space = space_between_region, space_between }, (),
          match token with
          | EOF -> null ()
          | _ -> lazy (loop region))
  in
  lazy (loop Region.zero), ic

let of_path path =
  let ic = open_in path in
  of_channel ic