Source

ocaml-indent / tokenstr.ml

Full commit
open Spotlib.Spot

open Pos
open Reader
module Parser = Xparser
open Parser

module Elem = struct
  type t = Parser.token
  let show = Sexplib.Sexp.to_string_hum @@ Parser.sexp_of_token
  let format ppf= Sexplib.Sexp.pp_hum ppf @@ Parser.sexp_of_token
  let equal = (=)
  let compare = compare
end

module Attr = struct
  type t = {
    region : Region.t;
    substr : string;
    space  : Region.t * string;
  }

  let position t = t.region

end

module Stream = Planck.Stream.Make(struct
  module Elem = Elem
  module Pos = Pos.Region
  module Attr = Attr
end)

type t = Stream.t * in_channel

let stream = fst
let close (_, ic) = close_in ic

let of_channel ic = 
  let reader = LexReader.create_from_channel ic in
  let rec loop last_region = 
    let token = 
      try
        LexReader.lex reader Lexer.token 
      with
      | Lexer.Error (e, _loc) ->
          Format.eprintf "%a@." Lexer.report_error e;
          assert false
    in
    let region = LexReader.region reader in
    (* token's string *)
    let substr = LexReader.current_substring reader in

    let space_between = 
      let last_end = (snd last_region).Position.pos_cnum in
      LexReader.substring 
        reader last_end ((fst region).Position.pos_cnum - last_end)
    in
    let space_between_region = (snd last_region, fst region) in

    let attr = { Attr.region; substr; space = space_between_region, space_between } in
    match token with
    | EOF -> Stream.null_desc attr
    | _ -> Stream.cons_desc token attr (lazy (loop region))
  in
  lazy (loop & Region.top ""), ic

let of_path = of_channel @@ open_in