Commits

Sébastien Ferré committed 9f556cc

EOL is no more a token.
pp_print_token now depends on previous token (to avoid clash of idents, for instance).

  • Participants
  • Parent commits 8cfc39f

Comments (0)

Files changed (1)

   fun ch ->
     let lexbuf = Lexing.from_channel ch in
     Stream.from (fun _ ->
-	match Lexer.token lexbuf with
+      try Some (Lexer.token lexbuf)
+      with Eof -> None)
+(*
+	try match Lexer.token lexbuf with
 	  EOL -> None
-	| tok -> Some tok)
+	| tok -> Some tok
+	with Eof -> None)
+*)
 
 let from_string : string -> t_stream =
   fun s ->
     let lexbuf = Lexing.from_string s in
     Stream.from (fun _ ->
-	match Lexer.token lexbuf with
-	  EOL -> None
-	| tok -> Some tok)
+      try Some (Lexer.token lexbuf)
+      with Eof -> None)
 
 let rec list_of_stream : t_stream -> t_list = parser
     [<'tok; toks = list_of_stream>] -> tok::toks
   fun toks -> Stream.of_list
       (List.filter
 	 (function
-	     PP_tilda | PP_space | PP_cut | PP_break (_,_) -> false
+	     PP_tilda | PP_space | PP_cut | PP_break (_,_) | PP_newline -> false
 	   | _ -> true
 	 ) toks
       )
 
 open Format
 
-let pp_print_token : formatter -> Token.t -> unit =
-  fun ff -> function
-    | EOL -> pp_print_newline ff ()
+let pp_print_token : formatter -> Token.t -> Token.t -> unit =
+  fun ff pred -> function
     | BackQuote -> pp_print_string ff "`"
     | Tilda -> pp_print_string ff "~"
     | Exclam -> pp_print_string ff "!"
     | Star -> pp_print_string ff "*"
     | LeftPar -> pp_print_string ff "("
     | RightPar -> pp_print_string ff ")"
-    | Minus -> pp_print_string ff "-"
-    | Plus -> pp_print_string ff "+"
+    | Minus ->
+	( match pred with
+	| Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_string ff "-"
+    | Plus ->
+	( match pred with
+	| Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_string ff "+"
     | Equal -> pp_print_string ff "="
-    | LeftAcc ->pp_print_string ff "{"
-    | RightAcc ->pp_print_string ff "}"
-    | LeftBra ->pp_print_string ff "["
-    | RightBra ->pp_print_string ff "]"
-    | Pipe ->pp_print_string ff "|"
-    | BackSlash ->pp_print_string ff "\\"
-    | Slash ->pp_print_string ff "/"
-    | Interro ->pp_print_string ff "?"
-    | LT ->pp_print_string ff "<"
-    | GT ->pp_print_string ff ">"
-    | Comma ->pp_print_string ff ","
-    | DotDot ->pp_print_string ff ".."
-    | Dot ->pp_print_string ff "."
-    | Colon ->pp_print_string ff ":"
-    | SemiColon ->pp_print_string ff ";"
-    | DoubleQuote ->pp_print_string ff "\""
-    | Quote ->pp_print_string ff "'"
-    | Ident s -> pp_print_string ff s
-    | Int n -> pp_print_int ff n
-    | Float f -> pp_print_float ff f
+    | LeftAcc -> pp_print_string ff "{"
+    | RightAcc -> pp_print_string ff "}"
+    | LeftBra -> pp_print_string ff "["
+    | RightBra -> pp_print_string ff "]"
+    | Pipe -> pp_print_string ff "|"
+    | BackSlash -> pp_print_string ff "\\"
+    | Slash -> pp_print_string ff "/"
+    | Interro -> pp_print_string ff "?"
+    | LT -> pp_print_string ff "<"
+    | GT -> pp_print_string ff ">"
+    | Comma -> pp_print_string ff ","
+    | DotDot -> pp_print_string ff ".."
+    | Dot ->
+	( match pred with
+	| Int _ | Float _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_string ff "."
+    | Colon -> pp_print_string ff ":"
+    | SemiColon -> pp_print_string ff ";"
+    | DoubleQuote -> pp_print_string ff "\""
+    | Quote -> pp_print_string ff "'"
+    | Ident s ->
+	( match pred with
+	| Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_string ff s
+    | Int n ->
+	( match pred with
+	| Plus | Minus | Int _ | Float _ | Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_int ff n
+    | Float f ->
+	( match pred with
+	| Plus | Minus | Int _ | Float _ | Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
+	pp_print_float ff f
     | String s -> pp_print_string ff ("\"" ^ String.escaped s ^ "\"")
     | Term s ->
+	( match pred with
+	| Ident _ | Term _ -> pp_print_string ff " "
+	| _ -> ());
 	let b = ref true in
 	(match s.[0] with 'A'..'Z' | '_' -> () | _ -> b:= false);
 	if !b then String.iter (function 'a'..'z' | 'A'..'Z' | '_' | '0'..'9' -> () | _ -> b:= false) s;
     | PP_space -> pp_print_space ff ()
     | PP_cut -> pp_print_cut ff ()
     | PP_break (spaces,offset) -> pp_print_break ff spaces offset
+    | PP_newline -> pp_print_newline ff ()
 
 
 let rec pp_print_tokens : formatter -> t_list -> unit =
-  fun ff tokens -> List.iter (pp_print_token ff) tokens
+  fun ff toks -> pp_print_tokens2 ff PP_newline toks
+and pp_print_tokens2 ff pred =
+  function
+    | [] -> ()
+    | tok::toks ->
+	pp_print_token ff pred tok;
+	pp_print_tokens2 ff tok toks
+
 (*
   fun ff -> function
       [] -> ()