Commits

Tim Hatch  committed bec55c8 Merge

Merged in megajoule/pygments-main (pull request #372)

Update ElixirLexer and ElixirConsoleLexer

  • Participants
  • Parent commits 944a0b5, bbb99ac

Comments (0)

Files changed (3)

File pygments/lexers/functional.py

         return rv
 
 
+def gen_elixir_string_rules(name, symbol, token):
+    states = {}
+    states['string_' + name] = [
+        (r'[^#%s\\]+' % (symbol,), token),
+        include('escapes'),
+        (r'\\.', token),
+        (r'(%s)(:?)' % (symbol,), bygroups(token, Punctuation), "#pop"),
+        include('interpol')
+    ]
+    return states
+
+def gen_elixir_sigstr_rules(term, token, interpol=True):
+    if interpol:
+        return [
+            (r'[^#%s\\]+' % (term,), token),
+            include('escapes'),
+            (r'\\.', token),
+            (r'%s[a-zA-Z]*' % (term,), token, '#pop'),
+            include('interpol')
+        ]
+    else:
+        return [
+            (r'[^%s\\]+' % (term,), token),
+            (r'\\.', token),
+            (r'%s[a-zA-Z]*' % (term,), token, '#pop'),
+        ]
+
 class ElixirLexer(RegexLexer):
     """
     For the `Elixir language <http://elixir-lang.org>`_.
     filenames = ['*.ex', '*.exs']
     mimetypes = ['text/x-elixir']
 
+    KEYWORD = ['fn', 'do', 'end', 'after', 'else', 'rescue', 'catch']
+    KEYWORD_OPERATOR = ['not', 'and', 'or', 'xor', 'when', 'in']
+    BUILTIN = [
+        'case', 'cond', 'for', 'if', 'unless', 'try', 'receive', 'raise',
+        'quote', 'unquote', 'unquote_splicing', 'throw', 'super'
+    ]
+    BUILTIN_DECLARATION = [
+        'def', 'defp', 'defmodule', 'defprotocol', 'defmacro', 'defmacrop',
+        'defdelegate', 'defexception', 'defstruct', 'defimpl', 'defcallback'
+    ]
+
+    BUILTIN_NAMESPACE = ['import', 'require', 'use', 'alias']
+    CONSTANT = ['nil', 'true', 'false']
+
+    PSEUDO_VAR = ['_', '__MODULE__', '__DIR__', '__ENV__', '__CALLER__']
+
+    OPERATORS3 = ['<<<', '>>>', '|||', '&&&', '^^^', '~~~', '===', '!==']
+    OPERATORS2 = [
+        '==', '!=', '<=', '>=', '&&', '||', '<>', '++', '--', '|>', '=~'
+    ]
+    OPERATORS1 = ['<', '>', '+', '-', '*', '/', '!', '^', '&']
+
+    PUNCTUATION = [
+        '\\\\', '<<', '>>', '::', '->', '<-', '=>', '|', '(', ')',
+        '{', '}', ';', ',', '.', '[', ']', '%', '='
+    ]
+
+    def get_tokens_unprocessed(self, text):
+        for index, token, value in RegexLexer.get_tokens_unprocessed(self, text):
+            if token is Name:
+                if value in self.KEYWORD:
+                    yield index, Keyword, value
+                elif value in self.KEYWORD_OPERATOR:
+                    yield index, Operator.Word, value
+                elif value in self.BUILTIN:
+                    yield index, Keyword, value
+                elif value in self.BUILTIN_DECLARATION:
+                    yield index, Keyword.Declaration, value
+                elif value in self.BUILTIN_NAMESPACE:
+                    yield index, Keyword.Namespace, value
+                elif value in self.CONSTANT:
+                    yield index, Name.Constant, value
+                elif value in self.PSEUDO_VAR:
+                    yield index, Name.Builtin.Pseudo, value
+                else:
+                    yield index, token, value
+            else:
+                yield index, token, value
+
     def gen_elixir_sigil_rules():
-        states = {}
-
-        states['strings'] = [
-            (r'(%[A-Ba-z])?"""(?:.|\n)*?"""', String.Doc),
-            (r"'''(?:.|\n)*?'''", String.Doc),
-            (r'"', String.Double, 'dqs'),
-            (r"'.*?'", String.Single),
-            (r'(?<!\w)\?(\\(x\d{1,2}|\h{1,2}(?!\h)\b|0[0-7]{0,2}(?![0-7])\b|'
-             r'[^x0MC])|(\\[MC]-)+\w|[^\s\\])', String.Other)
+        # these braces are balanced inside the sigil string
+        braces = [
+            (r'\{', r'\}', 'cb'),
+            (r'\[', r'\]', 'sb'),
+            (r'\(', r'\)', 'pa'),
+            (r'\<', r'\>', 'ab'),
         ]
 
-        for lbrace, rbrace, name, in ('\\{', '\\}', 'cb'), \
-                                     ('\\[', '\\]', 'sb'), \
-                                     ('\\(', '\\)', 'pa'), \
-                                     ('\\<', '\\>', 'lt'):
-
-            states['strings'] += [
-                (r'%[a-z]' + lbrace, String.Double, name + 'intp'),
-                (r'%[A-Z]' + lbrace, String.Double, name + 'no-intp')
+        # these are also valid sigil terminators, they are not balanced
+        terms = [
+            (r'/', 'slas'), (r'\|', 'pipe'), ('"', 'quot'), ("'", 'apos'),
+        ]
+
+        # heredocs have slightly different rules, they are not balanced
+        triquotes = [(r'"""', 'triquot'), (r"'''", 'triapos')]
+
+        token = String.Other
+        states = {'sigils': []}
+
+        for term, name in triquotes:
+            states['sigils'] += [
+                (r'(~[a-z])(%s)' % (term,), bygroups(token, String.Heredoc),
+                    (name + '-end', name + '-intp')),
+                (r'(~[A-Z])(%s)' % (term,), bygroups(token, String.Heredoc),
+                    (name + '-end', name + '-no-intp')),
             ]
 
-            states[name +'intp'] = [
-                (r'' + rbrace + '[a-z]*', String.Double, "#pop"),
-                include('enddoublestr')
+            states[name +'-end'] = [(r'[a-zA-Z]*', token, '#pop')]
+            states[name +'-intp'] = [
+                (r'^\s*' + term, String.Heredoc, '#pop'),
+                include('heredoc_interpol'),
             ]
-
-            states[name +'no-intp'] = [
-                (r'.*' + rbrace + '[a-z]*', String.Double , "#pop")
+            states[name +'-no-intp'] = [
+                (r'^\s*' + term, String.Heredoc, '#pop'),
+                include('heredoc_no_interpol'),
             ]
 
+        for term, name in terms:
+            states['sigils'] += [
+                (r'~[a-z]' + term, token, name + '-intp'),
+                (r'~[A-Z]' + term, token, name + '-no-intp'),
+            ]
+
+            # Similar states to the braced sigils, but no balancing of
+            # terminators
+            states[name +'-intp'] = gen_elixir_sigstr_rules(term, token)
+            states[name +'-no-intp'] = \
+                gen_elixir_sigstr_rules(term, token, interpol=False)
+
+        for lbrace, rbrace, name in braces:
+            states['sigils'] += [
+                (r'~[a-z]' + lbrace, token, name + '-intp'),
+                (r'~[A-Z]' + lbrace, token, name + '-no-intp')
+            ]
+
+            states[name +'-intp'] = [
+                (r'\\.', token),
+                (lbrace, token, '#push'),
+            ] + gen_elixir_sigstr_rules(rbrace, token)
+
+            states[name +'-no-intp'] = [
+                (r'\\.', token),
+                (lbrace, token, '#push'),
+            ] + gen_elixir_sigstr_rules(rbrace, token, interpol=False)
+
         return states
 
+    op3_re = "|".join(re.escape(s) for s in OPERATORS3)
+    op2_re = "|".join(re.escape(s) for s in OPERATORS2)
+    op1_re = "|".join(re.escape(s) for s in OPERATORS1)
+    ops_re = r'(?:%s|%s|%s)' % (op3_re, op2_re, op1_re)
+    punctuation_re = "|".join(re.escape(s) for s in PUNCTUATION)
+    name_re = r'[a-z_][a-zA-Z_0-9]*[!\?]?'
+    modname_re = r'[A-Z][A-Za-z_]*(?:\.[A-Z][A-Za-z_]*)*'
+    complex_name_re = r'(?:%s|%s|%s)' % (name_re, modname_re, ops_re)
+    special_atom_re = r'(?:\.\.\.|<<>>|%{}|%|{})'
+
     tokens = {
         'root': [
             (r'\s+', Text),
             (r'#.*$', Comment.Single),
-            (r'\b(case|cond|end|bc|lc|if|unless|try|loop|receive|fn|defmodule|'
-             r'defp?|defprotocol|defimpl|defrecord|defmacrop?|defdelegate|'
-             r'defexception|exit|raise|throw|unless|after|rescue|catch|else)\b(?![?!])|'
-             r'(?<!\.)\b(do|\-\>)\b\s*', Keyword),
-            (r'\b(import|require|use|recur|quote|unquote|super|refer)\b(?![?!])',
-                Keyword.Namespace),
-            (r'(?<!\.)\b(and|not|or|when|xor|in)\b', Operator.Word),
-            (r'%=|\*=|\*\*=|\+=|\-=|\^=|\|\|=|'
-             r'<=>|<(?!<|=)|>(?!<|=|>)|<=|>=|===|==|=~|!=|!~|(?=[ \t])\?|'
-             r'(?<=[ \t])!+|&&|\|\||\^|\*|\+|\-|/|'
-             r'\||\+\+|\-\-|\*\*|\/\/|\<\-|\<\>|<<|>>|=|\.', Operator),
-            (r'(?<!:)(:)([a-zA-Z_]\w*([?!]|=(?![>=]))?|\<\>|===?|>=?|<=?|'
-             r'<=>|&&?|%\(\)|%\[\]|%\{\}|\+\+?|\-\-?|\|\|?|\!|//|[%&`/\|]|'
-             r'\*\*?|=?~|<\-)|([a-zA-Z_]\w*([?!])?)(:)(?!:)', String.Symbol),
-            (r':"', String.Symbol, 'interpoling_symbol'),
-            (r'\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b', Name.Constant),
-            (r'\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])', Name.Builtin.Pseudo),
-            (r'[a-zA-Z_!]\w*[!\?]?', Name),
-            (r'[(){};,/\|:\\\[\]]', Punctuation),
-            (r'@[a-zA-Z_]\w*|&\d', Name.Variable),
-            (r'(?i)\b(0x[\da-f]+)\b', Number.Hex),
-            (r'\b\d(_?\d)*(?!\.)\b', Number.Integer),
-            (r'\b(\d(_?\d)*(\.(?![^\d\s])(_?\d)*)?([eE][-+]?\d(_?\d)*)?)\b',
-             Number.Float),
-            (r'\b(0[bB][01]+)\b', Number.Bin),
-            (r'%r\/.*\/', String.Regex),
-            include('strings'),
+
+            # Various kinds of characters
+            (r'(?i)(\?)(\\x{)([\da-f]+)(})',
+                bygroups(String.Char,
+                    String.Escape, Number.Hex, String.Escape)),
+            (r'(?i)(\?)(\\x[\da-f]{1,2})',
+                bygroups(String.Char, String.Escape)),
+            (r'(\?)(\\[0-7]{1,3})',
+                bygroups(String.Char, String.Escape)),
+            (r'(\?)(\\[abdefnrstv])',
+                bygroups(String.Char, String.Escape)),
+            (r'\?\\?.', String.Char),
+
+            # atoms
+            (r':' + special_atom_re, String.Symbol),
+            (r':' + complex_name_re, String.Symbol),
+            (r':"', String.Symbol, 'string_double_atom'),
+            (r":'", String.Symbol, 'string_single_atom'),
+
+            # [keywords: ...]
+            (r'(%s|%s)(:)(?=\s|\n)' % (special_atom_re, complex_name_re),
+                bygroups(String.Symbol, Punctuation)),
+
+            # @attributes
+            (r'@' + name_re, Name.Attribute),
+
+            # operators and punctuation
+            (op3_re, Operator),
+            (op2_re, Operator),
+            (punctuation_re, Punctuation),
+            (r'&\d', Name.Entity),   # anon func arguments
+            (op1_re, Operator),
+
+            # identifiers
+            (name_re, Name),
+            (modname_re, Name.Class),
+
+            # numbers
+            (r'0[bB][01]+', Number.Bin),
+            (r'0[0-7]+', Number.Oct),
+            (r'(?i)0x[\da-f]+', Number.Hex),
+            (r'\d(_?\d)*\.\d(_?\d)*([eE][-+]?\d(_?\d)*)?', Number.Float),
+            (r'\d(_?\d)*', Number.Integer),
+
+            # strings and heredocs
+            (r'"""\s*', String.Heredoc, 'heredoc_double'),
+            (r"'''\s*$", String.Heredoc, 'heredoc_single'),
+            (r'"', String.Double, 'string_double'),
+            (r"'", String.Single, 'string_single'),
+
+            include('sigils'),
         ],
-        'dqs': [
-            (r'"', String.Double, "#pop"),
-            include('enddoublestr')
+        'heredoc_double': [
+            (r'^\s*"""', String.Heredoc, '#pop'),
+            include('heredoc_interpol'),
         ],
-        'interpoling': [
-            (r'#{', String.Interpol, 'interpoling_string'),
+        'heredoc_single': [
+            (r"^\s*'''", String.Heredoc, '#pop'),
+            include('heredoc_interpol'),
         ],
-        'interpoling_string' : [
+        'heredoc_interpol': [
+            (r'[^#\\\n]+', String.Heredoc),
+            include('escapes'),
+            (r'\\.', String.Heredoc),
+            (r'\n+', String.Heredoc),
+            include('interpol'),
+        ],
+        'heredoc_no_interpol': [
+            (r'[^\\\n]+', String.Heredoc),
+            (r'\\.', String.Heredoc),
+            (r'\n+', String.Heredoc),
+        ],
+        'escapes': [
+            (r'(?i)(\\x{)([\da-f]+)(})',
+                bygroups(String.Escape, Number.Hex, String.Escape)),
+            (r'(?i)\\x[\da-f]{1,2}', String.Escape),
+            (r'\\[0-7]{1,3}', String.Escape),
+            (r'\\[abdefnrstv]', String.Escape),
+        ],
+        'interpol': [
+            (r'#{', String.Interpol, 'interpol_string'),
+        ],
+        'interpol_string' : [
             (r'}', String.Interpol, "#pop"),
             include('root')
         ],
-        'interpoling_symbol': [
-            (r'"', String.Symbol, "#pop"),
-            include('interpoling'),
-            (r'[^#"]+', String.Symbol),
-        ],
-        'enddoublestr' : [
-            include('interpoling'),
-            (r'[^#"]+', String.Double),
-        ]
     }
+    tokens.update(gen_elixir_string_rules('double', '"', String.Double))
+    tokens.update(gen_elixir_string_rules('single', "'", String.Single))
+    tokens.update(gen_elixir_string_rules('double_atom', '"', String.Symbol))
+    tokens.update(gen_elixir_string_rules('single_atom', "'", String.Symbol))
     tokens.update(gen_elixir_sigil_rules())
 
 
     aliases = ['iex']
     mimetypes = ['text/x-elixir-shellsession']
 
-    _prompt_re = re.compile('(iex|\.{3})> ')
+    _prompt_re = re.compile('(iex|\.{3})(\(\d+\))?> ')
 
     def get_tokens_unprocessed(self, text):
         exlexer = ElixirLexer(**self.options)
 
         curcode = ''
+        in_error = False
         insertions = []
         for match in line_re.finditer(text):
             line = match.group()
             if line.startswith(u'** '):
+                in_error = True
                 insertions.append((len(curcode),
                                    [(0, Generic.Error, line[:-1])]))
                 curcode += line[-1:]
             else:
                 m = self._prompt_re.match(line)
                 if m is not None:
+                    in_error = False
                     end = m.end()
                     insertions.append((len(curcode),
                                        [(0, Generic.Prompt, line[:end])]))
                 else:
                     if curcode:
                         for item in do_insertions(insertions,
-                                        exlexer.get_tokens_unprocessed(curcode)):
+                                    exlexer.get_tokens_unprocessed(curcode)):
                             yield item
                         curcode = ''
                         insertions = []
-                    yield match.start(), Generic.Output, line
+                    token = Generic.Error if in_error else Generic.Output
+                    yield match.start(), token, line
         if curcode:
             for item in do_insertions(insertions,
-                                      exlexer.get_tokens_unprocessed(curcode)):
+                                  exlexer.get_tokens_unprocessed(curcode)):
                 yield item
 
 

File tests/examplefiles/example_elixir.ex

-# We cannot use to_char_list because it depends on inspect,
-# which depends on protocol, which depends on this module.
-import Elixir::Builtin, except: [to_char_list: 1]
+# Numbers
+0b0101011
+1234 ; 0x1A ; 0xbeef ; 0763
+3.14 ; 5.0e21 ; 0.5e-12
+100_000_000
 
-defmodule Module do
-  require Erlang.ets, as: ETS
+# Characters
+?a ; ?1 ; ?\n ; ?\s ; ?\c ; ? ; ?,
+?\x{12} ; ?\x{abcd}
+?\x34 ; ?\xf
+?\123 ; ?\12 ; ?\7
 
-  @moduledoc """
-  This module provides many functions to deal with modules during
-  compilation time. It allows a developer to dynamically attach
-  documentation, merge data, register attributes and so forth.
+# Atoms
+:this ; :that
+:'complex atom'
+:"with' \"\" 'quotes"
+:" multi
+ line ' \s \123 \xff
+atom"
+:... ; :<<>> ; :%{} ; :% ; :{}
+:++; :--; :*; :~~~
 
-  After the module is compiled, using many of the functions in
-  this module will raise errors, since it is out of their purpose
-  to inspect runtime data. Most of the runtime data can be inspected
-  via the `__info__(attr)` function attached to each compiled module.
-  """
+# Strings
+"Hello world"
+"Interspersed \x{ff} codes \7 \8 \65 \016 and \t\s\z\+ \\ escapes"
+"Quotes ' inside \" \123 the \"\" \xF string \\\" end"
+"Multiline
+   string"
+
+# Char lists
+'this is a list'
+'escapes \' \t \\\''
+'Multiline
+    char
+  list
+'
+
+# Binaries
+<<1, 2, 3>>
+<<"hello"::binary, c :: utf8, x::[4, unit(2)]>> = "hello™1"
+
+# Sigils
+~r/this + i\s "a" regex/
+~R'this + i\s "a" regex too'
+~w(hello #{ ["has" <> "123", '\c\d', "\123 interpol" | []] } world)s
+~W(hello #{no "123" \c\d \123 interpol} world)s
+
+~S"No escapes \s\t\n and no #{interpolation}"
+
+:"atoms work #{"to" <> "o"}"
+
+# Operators
+x = 1 + 2.0 * 3
+y = true and false; z = false xor true
+... = 144
+... == !x && y || z
+"hello" |> String.upcase |> String.downcase()
+{^z, a} = {true, x}
+
+# Lists, tuples, maps, keywords
+[1, :a, 'hello'] ++ [2, 3]
+[:head | [?t, ?a, ?i, ?l]]
+
+{:one, 2.0, "three"}
+
+[...: "this", <<>>: "is", %{}: "a keyword", %: "list", {}: "too"]
+["this is an atom too": 1, "so is this": 2]
+[option: "value", key: :word]
+[++: "operator", ~~~: :&&&]
+
+map = %{shortcut: "syntax"}
+%{map | "update" => "me"}
+%{ 12 => 13, :weird => ['thing'] }
+
+# Comprehensions
+for x <- 1..10, x < 5, do: {x, x}
+pixels = "12345678"
+for << <<r::4, g::4, b::4, a::4>> <- pixels >> do
+  [r, {g, %{"b" => a}}]
+end
+
+# String interpolation
+"String #{inspect "interpolation"} is quite #{1+4+7} difficult"
+
+# Modules
+defmodule Long.Module.Name do
+  @moduledoc "Simple module docstring"
 
   @doc """
-  Evalutes the quotes contents in the given module context.
-  Raises an error if the module was already compiled.
+  Multiline docstring
+  "with quotes"
+  and #{ %{"interpolation" => "in" <> "action"} }
+  """
+  defstruct [:a, :name, :height]
 
-  ## Examples
+  @doc ~S'''
+  No #{interpolation} of any kind.
+  \000 \x{ff}
 
-      defmodule Foo do
-        contents = quote do: (def sum(a, b), do: a + b)
-        Module.eval_quoted __MODULE__, contents, [], __FILE__, __LINE__
-      end
-
-      Foo.sum(1, 2) #=> 3
-  """
-  def eval_quoted(module, quoted, binding, filename, line) do
-    assert_not_compiled!(:eval_quoted, module)
-    { binding, scope } = Erlang.elixir_module.binding_and_scope_for_eval(line, to_char_list(filename), module, binding)
-    Erlang.elixir_def.reset_last(module)
-    Erlang.elixir.eval_quoted([quoted], binding, line, scope)
-  end
-
-  @doc """
-  Checks if the module is compiled or not.
-
-  ## Examples
-
-      defmodule Foo do
-        Module.compiled?(__MODULE__) #=> false
-      end
-
-      Module.compiled?(Foo) #=> true
-
-  """
-  def compiled?(module) do
-    table = data_table_for(module)
-    table == ETS.info(table, :name)
-  end
-
-  @doc """
-  Reads the data for the given module. This is used
-  to read data of uncompiled modules. If the module
-  was already compiled, you shoul access the data
-  directly by invoking `__info__(:data)` in that module.
-
-  ## Examples
-
-      defmodule Foo do
-        Module.merge_data __MODULE__, value: 1
-        Module.read_data __MODULE__ #=> [value: 1]
-      end
-
-  """
-  def read_data(module) do
-    assert_not_compiled!(:read_data, module)
-    ETS.lookup_element(data_table_for(module), :data, 2)
-  end
-
-  @doc """
-  Reads the data from `module` at the given key `at`.
-
-  ## Examples
-
-      defmodule Foo do
-        Module.merge_data __MODULE__, value: 1
-        Module.read_data __MODULE__, :value #=> 1
-      end
-
-  """
-  def read_data(module, at) do
-    Orddict.get read_data(module), at
-  end
-
-  @doc """
-  Merge the given data into the module, overriding any
-  previous one.
-
-  If any of the given data is a registered attribute, it is
-  automatically added to the attribute set, instead of marking
-  it as data. See register_attribute/2 and add_attribute/3 for
-  more info.
-
-  ## Examples
-
-      defmodule Foo do
-        Module.merge_data __MODULE__, value: 1
-      end
-
-      Foo.__info__(:data) #=> [value: 1]
-
-  """
-  def merge_data(module, data) do
-    assert_not_compiled!(:merge_data, module)
-
-    table      = data_table_for(module)
-    old        = ETS.lookup_element(table, :data, 2)
-    registered = ETS.lookup_element(table, :registered_attributes, 2)
-
-    { attrs, new } = Enum.partition data, fn({k,_}) -> List.member?(registered, k) end
-    Enum.each attrs, fn({k,v}) -> add_attribute(module, k, v) end
-    ETS.insert(table, { :data,  Orddict.merge(old, new) })
-  end
-
-  @doc """
-  Attaches documentation to a given function. It expects
-  the module the function belongs to, the line (a non negative
-  integer), the kind (def or defmacro), a tuple representing
-  the function and its arity and the documentation, which should
-  be either a binary or a boolean.
-
-  ## Examples
-
-      defmodule MyModule do
-        Module.add_doc(__MODULE__, __LINE__ + 1, :def, { :version, 0 }, "Manually added docs")
-        def version, do: 1
-      end
-
-  """
-  def add_doc(module, line, kind, tuple, doc) when
-      is_binary(doc) or is_boolean(doc) do
-    assert_not_compiled!(:add_doc, module)
-    case kind do
-    match: :defp
-      :warn
-    else:
-      table = docs_table_for(module)
-      ETS.insert(table, { tuple, line, kind, doc })
-      :ok
-    end
-  end
-
-  @doc """
-  Checks if a function was defined, regardless if it is
-  a macro or a private function. Use function_defined?/3
-  to assert for an specific type.
-
-  ## Examples
-
-      defmodule Example do
-        Module.function_defined? __MODULE__, { :version, 0 } #=> false
-        def version, do: 1
-        Module.function_defined? __MODULE__, { :version, 0 } #=> true
-      end
-
-  """
-  def function_defined?(module, tuple) when is_tuple(tuple) do
-    assert_not_compiled!(:function_defined?, module)
-    table = function_table_for(module)
-    ETS.lookup(table, tuple) != []
-  end
-
-  @doc """
-  Checks if a function was defined and also for its `kind`.
-  `kind` can be either :def, :defp or :defmacro.
-
-  ## Examples
-
-      defmodule Example do
-        Module.function_defined? __MODULE__, { :version, 0 }, :defp #=> false
-        def version, do: 1
-        Module.function_defined? __MODULE__, { :version, 0 }, :defp #=> false
-      end
-
-  """
-  def function_defined?(module, tuple, kind) do
-    List.member? defined_functions(module, kind), tuple
-  end
-
-  @doc """
-  Return all functions defined in the given module.
-
-  ## Examples
-
-      defmodule Example do
-        def version, do: 1
-        Module.defined_functions __MODULE__ #=> [{:version,1}]
-      end
-
-  """
-  def defined_functions(module) do
-    assert_not_compiled!(:defined_functions, module)
-    table = function_table_for(module)
-    lc { tuple, _, _ } in ETS.tab2list(table), do: tuple
-  end
-
-  @doc """
-  Returns all functions defined in te given module according
-  to its kind.
-
-  ## Examples
-
-      defmodule Example do
-        def version, do: 1
-        Module.defined_functions __MODULE__, :def  #=> [{:version,1}]
-        Module.defined_functions __MODULE__, :defp #=> []
-      end
-
-  """
-  def defined_functions(module, kind) do
-    assert_not_compiled!(:defined_functions, module)
-    table = function_table_for(module)
-    entry = kind_to_entry(kind)
-    ETS.lookup_element(table, entry, 2)
-  end
-
-  @doc """
-  Adds a compilation callback hook that is invoked
-  exactly before the module is compiled.
-
-  This callback is useful when used with `use` as a mechanism
-  to clean up any internal data in the module before it is compiled.
-
-  ## Examples
-
-  Imagine you are creating a module/library that is meant for
-  external usage called `MyLib`. It could be defined as:
-
-      defmodule MyLib do
-        def __using__(target) do
-          Module.merge_data target, some_data: true
-          Module.add_compile_callback(target, __MODULE__, :__callback__)
-        end
-
-        defmacro __callback__(target) do
-          value = Orddict.get(Module.read_data(target), :some_data, [])
-          quote do: (def my_lib_value, do: unquote(value))
-        end
-      end
-
-  And a module could use `MyLib` with:
-
-      defmodule App do
-        use ModuleTest::ToBeUsed
-      end
-
-  In the example above, `MyLib` defines a data to the target. This data
-  can be updated throughout the module definition and therefore, the final
-  value of the data can only be compiled using a compiation callback,
-  which will read the final value of :some_data and compile to a function.
-  """
-  def add_compile_callback(module, target, fun // :__compiling__) do
-    assert_not_compiled!(:add_compile_callback, module)
-    new   = { target, fun }
-    table = data_table_for(module)
-    old   = ETS.lookup_element(table, :compile_callbacks, 2)
-    ETS.insert(table, { :compile_callbacks,  [new|old] })
-  end
-
-  @doc """
-  Adds an Erlang attribute to the given module with the given
-  key and value. The same attribute can be added more than once.
-
-  ## Examples
-
-      defmodule MyModule do
-        Module.add_attribute __MODULE__, :custom_threshold_for_lib, 10
-      end
-
-  """
-  def add_attribute(module, key, value) when is_atom(key) do
-    assert_not_compiled!(:add_attribute, module)
-    table = data_table_for(module)
-    attrs = ETS.lookup_element(table, :attributes, 2)
-    ETS.insert(table, { :attributes, [{key, value}|attrs] })
-  end
-
-  @doc """
-  Deletes all attributes that matches the given key.
-
-  ## Examples
-
-      defmodule MyModule do
-        Module.add_attribute __MODULE__, :custom_threshold_for_lib, 10
-        Module.delete_attribute __MODULE__, :custom_threshold_for_lib
-      end
-
-  """
-  def delete_attribute(module, key) when is_atom(key) do
-    assert_not_compiled!(:delete_attribute, module)
-    table = data_table_for(module)
-    attrs = ETS.lookup_element(table, :attributes, 2)
-    final = lc {k,v} in attrs, k != key, do: {k,v}
-    ETS.insert(table, { :attributes, final })
-  end
-
-  @doc """
-  Registers an attribute. This allows a developer to use the data API
-  but Elixir will register the data as an attribute automatically.
-  By default, `vsn`, `behavior` and other Erlang attributes are
-  automatically registered.
-
-  ## Examples
-
-      defmodule MyModule do
-        Module.register_attribute __MODULE__, :custom_threshold_for_lib
-        @custom_threshold_for_lib 10
-      end
-
-  """
-  def register_attribute(module, new) do
-    assert_not_compiled!(:register_attribute, module)
-    table = data_table_for(module)
-    old = ETS.lookup_element(table, :registered_attributes, 2)
-    ETS.insert(table, { :registered_attributes,  [new|old] })
-  end
+  \n #{\x{ff}}
+  '''
+  def func(a, b \\ []), do: :ok
 
   @doc false
-  # Used internally to compile documentation. This function
-  # is private and must be used only internally.
-  def compile_doc(module, line, kind, pair) do
-    case read_data(module, :doc) do
-    match: nil
-      # We simply discard nil
-    match: doc
-      result = add_doc(module, line, kind, pair, doc)
-      merge_data(module, doc: nil)
-      result
-    end
-  end
-
-  ## Helpers
-
-  defp kind_to_entry(:def),      do: :public
-  defp kind_to_entry(:defp),     do: :private
-  defp kind_to_entry(:defmacro), do: :macros
-
-  defp to_char_list(list) when is_list(list),  do: list
-  defp to_char_list(bin)  when is_binary(bin), do: binary_to_list(bin)
-
-  defp data_table_for(module) do
-    list_to_atom Erlang.lists.concat([:d, module])
-  end
-
-  defp function_table_for(module) do
-    list_to_atom Erlang.lists.concat([:f, module])
-  end
-
-  defp docs_table_for(module) do
-    list_to_atom Erlang.lists.concat([:o, module])
-  end
-
-  defp assert_not_compiled!(fun, module) do
-    compiled?(module) ||
-      raise ArgumentError, message:
-        "could not call #{fun} on module #{module} because it was already compiled"
+  def __before_compile__(_) do
+    :ok
   end
 end
 
-HashDict.new [{'A', 0}, {'T', 0}, {'C', 0}, {'G', 0}]
+# Structs
+defmodule Second.Module do
+  s = %Long.Module.Name{name: "Silly"}
+  %{s | height: {192, :cm}}
+end
+
+# Types, pseudo-vars, attributes
+defmodule M do
+  @custom_attr :some_constant
+
+  @before_compile Long.Module.Name
+
+  @typedoc "This is a type"
+  @type typ :: integer
+
+  @typedoc """
+  Another type
+  """
+  @opaque typtyp :: 1..10
+
+  @spec func(typ, typtyp) :: :ok | :fail
+  def func(a, b) do
+    a || b || :ok || :fail
+    Path.expand("..", __DIR__)
+    IO.inspect __ENV__
+    __NOTAPSEUDOVAR__ = 11
+    __MODULE__.func(b, a)
+  end
+
+  defmacro m() do
+    __CALLER__
+  end
+end
+
+# Functions
+anon = fn x, y, z ->
+  fn(a, b, c) ->
+    &(x + y - z * a / &1 + b + div(&2, c))
+  end
+end
+
+&Set.put(&1, &2) ; & Set.put(&1, &2) ; &( Set.put(&1, &1) )
+
+# Function calls
+anon.(1, 2, 3); self; hd([1,2,3])
+Kernel.spawn(fn -> :ok end)
+IO.ANSI.black
+
+# Control flow
+if :this do
+  :that
+else
+  :otherwise
+end
+
+pid = self
+receive do
+  {:EXIT, _} -> :done
+  {^pid, :_} -> nil
+  after 100 -> :no_luck
+end
+
+case __ENV__.line do
+  x when is_integer(x) -> x
+  x when x in 1..12 -> -x
+end
+
+cond do
+  false -> "too bad"
+  4 > 5 -> "oops"
+  true -> nil
+end
+
+# Lexical scope modifiers
+import Kernel, except: [spawn: 1, +: 2, /: 2, Unless: 2]
+alias Long.Module.Name, as: Namen
+use Bitwise
+
+4 &&& 5
+2 <<< 3
+
+# Protocols
+defprotocol Useless do
+  def func1(this)
+  def func2(that)
+end
+
+defimpl Useless, for: Atom do
+end
+
+# Exceptions
+defmodule NotAnError do
+  defexception [:message]
+end
+
+raise NotAnError, message: "This is not an error"

File tests/examplefiles/iex_example

+iex> :" multi
+...>  line ' \s \123 \x20
+...> atom"
+:" multi\n line '   S  \natom"
+
+iex(1)> <<"hello"::binary, c :: utf8, x::[4, unit(2)]>> = "hello™1"
+"hello™1"
+
+iex(2)> c
+8482
+
+iex> 1 + :atom
+** (ArithmeticError) bad argument in arithmetic expression
+    :erlang.+(1, :atom)
+
+iex(3)> 1 +
+...(3)> 2 +
+...(3)> 3
+6
+
+iex> IO.puts "Hello world"
+Hello world
+:ok