Source

ocaml-indent / tokenstr.ml

open Pos
open Reader
open Parser

type 'a desc = 
  | Cons of 'a * 'a t * in_channel
  | Null

and 'a t = 'a desc lazy_t

type 'a info = {
  token : 'a option; (* None means white space *)
  region : Region.t;
  substr : string;
}

let of_channel ic = 
  try
    let reader = LexReader.create_from_channel ic in
    let rec loop last_region = 
      let token = 
        try
          LexReader.lex reader Lexer.token 
        with
        | Lexer.Error (e, _loc) ->
            Format.eprintf "%a@." Lexer.report_error e;
            assert false
      in
      let region = LexReader.region reader in
      (* token's string *)
      let substr = LexReader.current_substring reader in

      let space =
        let space_between = 
          let last_end = (snd last_region).Position.pos_cnum in
          LexReader.substring 
            reader last_end ((fst region).Position.pos_cnum - last_end)
        in
        if space_between = "" then None
        else Some { token = None; 
                    region = (snd last_region, fst region);
                    substr = space_between }
      in

      let l = 
        Cons ({ token = Some token ; region; substr },
               lazy (match token with
               | EOF -> Null
               | _ -> loop region),
               ic)
      in
      match space with
      | None -> l
      | Some space ->
          Cons ( space, Lazy.lazy_from_val l, ic )
    in
    lazy (loop Region.zero)
  with
  | e -> raise e

let of_path path =
  let ic = open_in path in
  of_channel ic

let close = function
  | lazy Null -> ()
  | lazy (Cons (_, _, ic)) -> close_in ic

let destr = function
  | lazy Null -> None
  | lazy (Cons (car, cdr, _ic)) -> Some (car, cdr)