Commits

Kirill Simonov  committed 770b2b5

Implemented locator syntax and `id()` function.

  • Participants
  • Parent commits 8e7070e

Comments (0)

Files changed (24)

File src/htsql/core/domain.py

         return "/%s" % self.item_domain
 
 
+class IdentityDomain(Domain):
+
+    family = 'identity'
+
+    pattern = r"""
+        (?P<ws> \s+ ) |
+        (?P<symbol> \. | \[ | \( | \] | \) ) |
+        (?P<unquoted> [\w-]+ ) |
+        (?P<quoted> ' (?: [^'\0] | '')* ' )
+    """
+    regexp = re.compile(pattern, re.X|re.U)
+
+    def __init__(self, fields):
+        assert isinstance(fields, listof(Domain))
+        self.fields = fields
+        self.arity = 0
+        for field in fields:
+            if isinstance(field, IdentityDomain):
+                self.arity += field.arity
+            else:
+                self.arity += 1
+
+    def __basis__(self):
+        return (tuple(self.fields),)
+
+    def __str__(self):
+        return "[%s]" % ".".join(str(field) for field in self.fields)
+
+    def parse(self, data):
+        # Sanity check on the arguments.
+        assert isinstance(data, maybe(unicode))
+        # `None` represents `NULL` both in literal and native format.
+        if data is None:
+            return None
+        tokens = []
+        start = 0
+        while start < len(data):
+            match = self.regexp.match(data, start)
+            if match is None:
+                raise ValueError("unexpected character %r" % data[start])
+            start = match.end()
+            if match.group('ws'):
+                continue
+            token = match.group()
+            tokens.append(token)
+        tokens.append(None)
+        stack = []
+        value = []
+        arity = 0
+        while tokens:
+            token = tokens.pop(0)
+            while token in u'[(':
+                stack.append((value, arity, token))
+                value = []
+                arity = 0
+                token = tokens.pop(0)
+            if token is None or token in u'[(]).':
+                raise ValueError("ill-formed locator")
+            if token.startswith(u'\'') and token.endswith(u'\''):
+                token = token[1:-1].replace(u'\'\'', u'\'')
+            value.append((token, None))
+            arity += 1
+            token = tokens.pop(0)
+            while token in u'])':
+                if not stack:
+                    raise ValueError("ill-formed locator")
+                parent_value, parent_arity, parent_bracket = stack.pop()
+                if ((token == u']' and parent_bracket != u'[') or
+                    (token == u')' and parent_bracket != u'(')):
+                    raise ValueError("ill-formed locator")
+                parent_value.append((value, arity))
+                value = parent_value
+                arity += parent_arity
+                token = tokens.pop(0)
+            if (token is not None or stack) and token != u'.':
+                raise ValueError("ill-formed locator")
+        def collect(raw, arity, identity):
+            value = []
+            if arity != identity.arity:
+                raise ValueError("ill-formed locator")
+            for field in identity.fields:
+                if isinstance(field, IdentityDomain):
+                    total_arity = 0
+                    items = []
+                    while total_arity < field.arity:
+                        assert raw
+                        item, item_arity = raw.pop(0)
+                        if total_arity == 0 and item_arity == field.arity:
+                            items = item
+                            total_arity = item_arity
+                        elif item_arity is None:
+                            total_arity += 1
+                            items.append(item)
+                        else:
+                            total_arity += item_arity
+                            items.append(item)
+                    if total_arity > field.arity:
+                        raise ValueError("ill-formed locator")
+                    item = collect(items, total_arity, field)
+                    value.append(item)
+                else:
+                    if not raw:
+                        raise ValueError("ill-formed locator")
+                    item, item_arity = raw.pop(0)
+                    if item_arity is not None:
+                        raise ValueError("ill-formed locator")
+                    item = field.parse(item)
+                    assert item is not None
+                    value.append(item)
+            return tuple(value)
+        return collect(value, arity, self)
+
+    def dump(self, value):
+        assert isinstance(value, maybe(tuple))
+        if value is None:
+            return None
+        def convert(value, fields, is_flattened=True):
+            assert isinstance(value, tuple) and len(value) == len(fields)
+            chunks = []
+            is_simple = all(not isinstance(field, IdentityDomain)
+                            for field in fields[1:])
+            for item, field in zip(value, fields):
+                if isinstance(field, IdentityDomain):
+                    is_label_flattened = False
+                    if len(field.fields) == 1:
+                        is_label_flattened = True
+                    if is_simple:
+                        is_label_flattened = True
+                    chunk = convert(item, field.fields, is_label_flattened)
+                    chunks.append(chunk)
+                else:
+                    chunk = field.dump(item)
+                    chunks.append(chunk)
+            data = u".".join(chunks)
+            if not is_flattened:
+                data = u"(%s)" % data
+            return data
+        return convert(value, self.fields)
+
+
 class BooleanDomain(Domain):
     """
     Represents Boolean data type.

File src/htsql/core/fmt/tabular.py

 
     def cells(self, value):
         if value is None:
-            return [None]
+            yield [None]
         else:
-            return [self.domain.dump(value)]
+            yield [self.domain.dump(value)]
 
 
 class VoidToCSV(ToCSV):

File src/htsql/core/tr/assemble.py

 from ..domain import BooleanDomain
 from .coerce import coerce
 from .flow import (Code, SegmentCode, LiteralCode, FormulaCode, CastCode,
-                   RecordCode, AnnihilatorCode, Unit, ColumnUnit, CompoundUnit,
-                   CorrelationCode)
+                   RecordCode, IdentityCode, AnnihilatorCode, CorrelationCode,
+                   Unit, ColumnUnit, CompoundUnit)
 from .term import (PreTerm, Term, UnaryTerm, BinaryTerm, TableTerm,
                    ScalarTerm, FilterTerm, JoinTerm, CorrelationTerm,
                    EmbeddingTerm, ProjectionTerm, OrderTerm, SegmentTerm,
 
 class EvaluateRecord(Evaluate):
 
-    adapt(RecordCode)
+    adapt_many(RecordCode,
+               IdentityCode)
 
     def __call__(self):
         for field in self.code.fields:
         return compose_record
 
 
+class DecomposeIdentity(Decompose):
+
+    adapt(IdentityCode)
+
+    def __call__(self):
+        compose_fields = []
+        for field in self.code.fields:
+            compose_field = self.state.decompose(field)
+            compose_fields.append(compose_field)
+        def compose_identity(row, stream, compose_fields=compose_fields):
+            return tuple(compose_field(row, stream)
+                         for compose_field in compose_fields)
+        return compose_identity
+
+
 class DecomposeAnnihilator(Decompose):
 
     adapt(AnnihilatorCode)

File src/htsql/core/tr/bind.py

 from ..util import maybe, listof, tupleof, similar
 from ..adapter import Adapter, Protocol, adapt, adapt_many
 from ..domain import (Domain, BooleanDomain, IntegerDomain, DecimalDomain,
-                      FloatDomain, UntypedDomain, EntityDomain, RecordDomain,
-                      ListDomain, VoidDomain)
+        FloatDomain, UntypedDomain, EntityDomain, RecordDomain, ListDomain,
+        IdentityDomain, VoidDomain)
 from ..classify import normalize
 from .error import BindError
 from .syntax import (Syntax, QuerySyntax, SegmentSyntax, SelectorSyntax,
-                     ApplicationSyntax, FunctionSyntax, MappingSyntax,
-                     OperatorSyntax, QuotientSyntax, SieveSyntax, LinkSyntax,
-                     HomeSyntax, AssignmentSyntax, SpecifierSyntax, GroupSyntax,
-                     IdentifierSyntax, WildcardSyntax, ReferenceSyntax,
-                     ComplementSyntax, StringSyntax, NumberSyntax)
+        ApplicationSyntax, FunctionSyntax, MappingSyntax, OperatorSyntax,
+        QuotientSyntax, SieveSyntax, LinkSyntax, HomeSyntax, AssignmentSyntax,
+        SpecifierSyntax, LocatorSyntax, LocationSyntax, GroupSyntax,
+        IdentifierSyntax, WildcardSyntax, ReferenceSyntax, ComplementSyntax,
+        StringSyntax, NumberSyntax)
 from .binding import (Binding, WrappingBinding, QueryBinding, SegmentBinding,
-                      RootBinding, HomeBinding, FreeTableBinding,
-                      AttachedTableBinding, ColumnBinding, QuotientBinding,
-                      KernelBinding, ComplementBinding, LinkBinding,
-                      SieveBinding, SortBinding, CastBinding,
-                      ImplicitCastBinding, RescopingBinding,
-                      AssignmentBinding, DefinitionBinding, SelectionBinding,
-                      WildSelectionBinding, RerouteBinding,
-                      ReferenceRerouteBinding, AliasBinding,
-                      LiteralBinding, VoidBinding,
-                      Recipe, LiteralRecipe, SelectionRecipe,
-                      FreeTableRecipe, AttachedTableRecipe,
-                      ColumnRecipe, KernelRecipe, ComplementRecipe,
-                      SubstitutionRecipe, BindingRecipe, ClosedRecipe,
-                      PinnedRecipe, AmbiguousRecipe)
+        RootBinding, HomeBinding, FreeTableBinding, AttachedTableBinding,
+        ColumnBinding, QuotientBinding, KernelBinding, ComplementBinding,
+        LinkBinding, LocatorBinding, SieveBinding, SortBinding, CastBinding,
+        IdentityBinding, ImplicitCastBinding, RescopingBinding,
+        AssignmentBinding, DefinitionBinding, SelectionBinding,
+        WildSelectionBinding, RerouteBinding, ReferenceRerouteBinding,
+        AliasBinding, LiteralBinding, VoidBinding, Recipe, LiteralRecipe,
+        SelectionRecipe, FreeTableRecipe, AttachedTableRecipe, ColumnRecipe,
+        KernelRecipe, ComplementRecipe, IdentityRecipe, ChainRecipe,
+        SubstitutionRecipe, BindingRecipe, ClosedRecipe, PinnedRecipe,
+        AmbiguousRecipe)
 from .lookup import (lookup_attribute, lookup_reference, lookup_complement,
-                     lookup_attribute_set, lookup_reference_set,
-                     expand, direct, guess_tag, lookup_command)
+        lookup_attribute_set, lookup_reference_set, expand, direct, guess_tag,
+        lookup_command, identify)
 from .coerce import coerce
 from .decorate import decorate
 
         return self.binding
 
 
+class SelectIdentity(Select):
+
+    adapt(IdentityDomain)
+
+    def __call__(self):
+        return self.binding
+
+
 class BindSelector(Bind):
 
     adapt(SelectorSyntax)
         return binding
 
 
+
+class BindLocator(Bind):
+
+    adapt(LocatorSyntax)
+
+    def __call__(self):
+        seed = self.state.bind(self.syntax.lbranch)
+        recipe = identify(seed)
+        if recipe is None:
+            raise BindError("cannot determine identity", seed.mark)
+        identity = self.state.use(recipe, self.syntax.rbranch, scope=seed)
+        if identity.domain.arity != self.syntax.rbranch.arity:
+            raise BindError("ill-formed locator", self.syntax.rbranch.mark)
+        def convert(identity, branches):
+            value = []
+            for field in identity.fields:
+                if isinstance(field, IdentityDomain):
+                    total_arity = 0
+                    items = []
+                    while total_arity < field.arity:
+                        assert branches
+                        branch = branches.pop(0)
+                        if (total_arity == 0 and
+                                isinstance(branch, LocationSyntax) and
+                                branch.arity == field.arity):
+                            items = branch.branches[:]
+                            total_arity = branch.arity
+                        elif isinstance(branch, LocationSyntax):
+                            items.append(branch)
+                            total_arity += branch.arity
+                        else:
+                            items.append(branch)
+                            total_arity += 1
+                    if total_arity > field.arity:
+                        raise BindError("ill-formed locator",
+                                        self.syntax.rbranch.mark)
+                    item = convert(field, items)
+                    value.append(item)
+                else:
+                    assert branches
+                    branch = branches.pop(0)
+                    if not isinstance(branch, StringSyntax):
+                        raise BindError("ill-formed locator",
+                                        self.syntax.lbranch.mark)
+                    try:
+                        item = field.parse(branch.value)
+                    except ValueError, exc:
+                        raise BindError(str(exc), branch.mark)
+                    value.append(item)
+            return tuple(value)
+        value = convert(identity.domain, self.syntax.rbranch.branches[:])
+        return LocatorBinding(self.state.scope, seed, identity, value,
+                              self.syntax)
+
+
 class BindGroup(Bind):
 
     adapt(GroupSyntax)
                                  self.recipe.quotient, self.syntax)
 
 
+class BindByIdentity(BindByRecipe):
+
+    adapt(IdentityRecipe)
+
+    def __call__(self):
+        elements = [self.state.use(recipe, self.syntax)
+                    for recipe in self.recipe.elements]
+        return IdentityBinding(self.state.scope, elements, self.syntax)
+
+
 class BindBySubstitution(BindByRecipe):
 
     adapt(SubstitutionRecipe)
         return AliasBinding(binding, self.syntax)
 
 
+class BindByChain(BindByRecipe):
+
+    adapt(ChainRecipe)
+
+    def __call__(self):
+        binding = self.state.scope
+        for recipe in self.recipe.recipes:
+            binding = self.state.use(recipe, self.syntax, scope=binding)
+        return binding
+
+
 class BindByPinned(BindByRecipe):
 
     adapt(PinnedRecipe)

File src/htsql/core/tr/binding.py

 from ..util import maybe, listof, tupleof, Clonable, Printable, Comparable
 from ..entity import TableEntity, ColumnEntity, Join
 from ..domain import (Domain, VoidDomain, BooleanDomain, ListDomain,
-                      RecordDomain, EntityDomain, Profile)
+                      RecordDomain, EntityDomain, IdentityDomain, Profile)
 from .syntax import Syntax, VoidSyntax
 from .signature import Signature, Bag, Formula
 from ..cmd.command import Command
         self.offset = offset
 
 
+class LocatorBinding(ScopingBinding):
+
+    def __init__(self, base, seed, identity, value, syntax):
+        assert isinstance(seed, Binding)
+        assert isinstance(identity, IdentityBinding)
+        assert (isinstance(value, tuple) and
+                len(value) == len(identity.elements))
+        super(LocatorBinding, self).__init__(base, seed.domain, syntax)
+        self.seed = seed
+        self.identity = identity
+        self.value = value
+
+
 class SieveBinding(ChainingBinding):
     """
     Represents a sieve expression.
     """
 
 
+class IdentityBinding(Binding):
+
+    def __init__(self, base, elements, syntax):
+        assert isinstance(elements, listof(Binding))
+        domain = IdentityDomain([element.domain for element in elements])
+        super(IdentityBinding, self).__init__(base, domain, syntax)
+        self.elements = elements
+
+
 class AssignmentBinding(Binding):
     """
     Represents an assignment expression.
         return "%s.*%s" % (self.quotient, self.index+1)
 
 
+class IdentityRecipe(Recipe):
+
+    def __init__(self, elements):
+        assert isinstance(elements, listof(Recipe))
+        self.elements = elements
+
+    def __basis__(self):
+        return (self.elements,)
+
+
 class ComplementRecipe(Recipe):
     """
     Generates a :class:`ComplementBinding` node.
         return "%s -> %s" % (self.scope, self.recipe)
 
 
+class ChainRecipe(Recipe):
+
+    def __init__(self, recipes):
+        assert isinstance(recipes, listof(Recipe))
+        self.recipes = recipes
+
+    def __basis__(self):
+        return (tuple(self.recipes),)
+
+
 class InvalidRecipe(Recipe):
     """
     Generates an error when applied.

File src/htsql/core/tr/coerce.py

 from ..util import listof
 from ..adapter import Adapter, adapt, adapt_many
 from ..domain import (Domain, VoidDomain, ListDomain, RecordDomain,
-                      EntityDomain, UntypedDomain, BooleanDomain, IntegerDomain,
-                      DecimalDomain, FloatDomain, StringDomain, EnumDomain,
-                      DateDomain, TimeDomain, DateTimeDomain, OpaqueDomain)
+                      EntityDomain, IdentityDomain, UntypedDomain,
+                      BooleanDomain, IntegerDomain, DecimalDomain, FloatDomain,
+                      StringDomain, EnumDomain, DateDomain, TimeDomain,
+                      DateTimeDomain, OpaqueDomain)
 
 
 class UnaryCoerce(Adapter):
     adapt_many(VoidDomain,
                ListDomain,
                RecordDomain,
-               EntityDomain)
+               EntityDomain,
+               IdentityDomain)
 
     def __call__(self):
         # Special domains are not coercable.

File src/htsql/core/tr/compile.py

 from .signature import (IsNullSig, IsEqualSig, AndSig, CompareSig,
                         SortDirectionSig, RowNumberSig)
 from .flow import (Expression, QueryExpr, SegmentCode, Code, LiteralCode,
-                   FormulaCode, Flow, RootFlow, ScalarFlow, TableFlow,
-                   QuotientFlow, ComplementFlow, MonikerFlow, ForkedFlow,
-                   LinkedFlow, ClippedFlow, FilteredFlow, OrderedFlow,
-                   Unit, ScalarUnit, ColumnUnit, AggregateUnit, CorrelatedUnit,
-                   KernelUnit, CoveringUnit, CorrelationCode)
+        FormulaCode, Flow, RootFlow, ScalarFlow, TableFlow, QuotientFlow,
+        ComplementFlow, MonikerFlow, LocatorFlow, ForkedFlow, LinkedFlow,
+        ClippedFlow, FilteredFlow, OrderedFlow, Unit, ScalarUnit, ColumnUnit,
+        AggregateUnit, CorrelatedUnit, KernelUnit, CoveringUnit,
+        CorrelationCode)
 from .term import (Term, ScalarTerm, TableTerm, FilterTerm, JoinTerm,
-                   EmbeddingTerm, CorrelationTerm, ProjectionTerm, OrderTerm,
-                   WrapperTerm, PermanentTerm, SegmentTerm, QueryTerm, Joint)
+        EmbeddingTerm, CorrelationTerm, ProjectionTerm, OrderTerm, WrapperTerm,
+        PermanentTerm, SegmentTerm, QueryTerm, Joint)
 from .stitch import arrange, spread, sew, tie
 
 
     adapt_many(MonikerFlow,
                ForkedFlow,
                LinkedFlow,
-               ClippedFlow)
+               ClippedFlow,
+               LocatorFlow)
 
     def __call__(self):
         # Moniker, forked and linked flows are represented as a seed term
                     continue
                 codes.append(code)
                 order.append((code, direction))
+        if isinstance(self.flow, LocatorFlow):
+            codes.append(self.flow.filter)
         # Any companion expressions must also be included.
         codes += self.flow.companions
         seed_term = self.state.inject(seed_term, codes)
 
+        if isinstance(self.flow, LocatorFlow):
+            seed_term = FilterTerm(self.state.tag(), seed_term,
+                                   self.flow.filter,
+                                   seed_term.flow,
+                                   seed_term.baseline,
+                                   seed_term.routes.copy())
+
         # Indicates whether the seed term has a regular shape.
         is_regular = (seed_term.baseline == self.flow.ground)
 
         return FilterTerm(self.state.tag(), term, filter,
                           term.flow, term.baseline, term.routes.copy())
 
-
     def clip_root(self, term, order):
         limit = self.flow.limit
         if limit is None:

File src/htsql/core/tr/encode.py

 
 from ..adapter import Adapter, adapt, adapt_many
 from ..domain import (Domain, UntypedDomain, EntityDomain, RecordDomain,
-                      BooleanDomain, NumberDomain, IntegerDomain,
-                      DecimalDomain, FloatDomain, StringDomain, EnumDomain,
-                      DateDomain, TimeDomain, DateTimeDomain, OpaqueDomain)
+        BooleanDomain, NumberDomain, IntegerDomain, DecimalDomain, FloatDomain,
+        StringDomain, EnumDomain, DateDomain, TimeDomain, DateTimeDomain,
+        OpaqueDomain)
 from .error import EncodeError
 from .coerce import coerce
 from .binding import (Binding, QueryBinding, SegmentBinding, WrappingBinding,
-                      SelectionBinding, HomeBinding, RootBinding,
-                      FreeTableBinding, AttachedTableBinding, ColumnBinding,
-                      QuotientBinding, KernelBinding, ComplementBinding,
-                      CoverBinding, ForkBinding, LinkBinding, ClipBinding,
-                      SieveBinding, SortBinding, CastBinding, RescopingBinding,
-                      LiteralBinding, FormulaBinding)
+        SelectionBinding, HomeBinding, RootBinding, FreeTableBinding,
+        AttachedTableBinding, ColumnBinding, QuotientBinding, KernelBinding,
+        ComplementBinding, IdentityBinding, LocatorBinding, CoverBinding,
+        ForkBinding, LinkBinding, ClipBinding, SieveBinding, SortBinding,
+        CastBinding, RescopingBinding, LiteralBinding, FormulaBinding)
 from .lookup import direct
 from .flow import (RootFlow, ScalarFlow, DirectTableFlow, FiberTableFlow,
-                   QuotientFlow, ComplementFlow, MonikerFlow, ForkedFlow,
-                   LinkedFlow, ClippedFlow, FilteredFlow, OrderedFlow,
-                   QueryExpr, SegmentCode, LiteralCode, FormulaCode,
-                   CastCode, RecordCode, AnnihilatorCode,
-                   ColumnUnit, ScalarUnit, KernelUnit)
-from .signature import Signature, IsNullSig, NullIfSig
+        QuotientFlow, ComplementFlow, MonikerFlow, LocatorFlow, ForkedFlow,
+        LinkedFlow, ClippedFlow, FilteredFlow, OrderedFlow, QueryExpr,
+        SegmentCode, LiteralCode, FormulaCode, CastCode, RecordCode,
+        AnnihilatorCode, IdentityCode, ColumnUnit, ScalarUnit, KernelUnit)
+from .signature import Signature, IsNullSig, NullIfSig, IsEqualSig, AndSig
 import decimal
 
 
                            self.binding.offset, self.binding)
 
 
+class RelateLocator(Relate):
+
+    adapt(LocatorBinding)
+
+    def __call__(self):
+        base = self.state.relate(self.binding.base)
+        seed = self.state.relate(self.binding.seed)
+        def convert(identity, items):
+            filters = []
+            for element, item in zip(identity.elements, items):
+                if isinstance(element, IdentityBinding):
+                    filter = convert(element, item)
+                else:
+                    element = self.state.encode(element)
+                    literal = LiteralCode(item, element.domain,
+                                          self.binding)
+                    filter = FormulaCode(IsEqualSig(+1),
+                                         coerce(BooleanDomain()),
+                                         self.binding,
+                                         lop=element, rop=literal)
+                filters.append(filter)
+            if len(filters) == 1:
+                return filters[0]
+            else:
+                return FormulaCode(AndSig(), coerce(BooleanDomain()),
+                                   self.binding, ops=filters)
+        filter = convert(self.binding.identity, self.binding.value)
+        return LocatorFlow(base, seed, filter, self.binding)
+
+
 class EncodeColumn(Encode):
 
     adapt(ColumnBinding)
         return self.state.relate(self.binding.base)
 
 
+class EncodeIdentity(Encode):
+
+    adapt(IdentityBinding)
+
+    def __call__(self):
+        flow = self.state.relate(self.binding.base)
+        fields = [self.state.encode(element)
+                  for element in self.binding.elements]
+        code = IdentityCode(fields, self.binding)
+        unit = ScalarUnit(code, flow, self.binding)
+        indicator = LiteralCode(True, coerce(BooleanDomain()), self.binding)
+        indicator = ScalarUnit(indicator, flow, self.binding)
+        return AnnihilatorCode(unit, indicator, self.binding)
+
+
 class EncodeWrapping(Encode):
 
     adapt(WrappingBinding)

File src/htsql/core/tr/flow.py

 from ..util import (maybe, listof, tupleof, Clonable, Comparable, Printable,
                     cachedproperty)
 from ..entity import TableEntity, ColumnEntity, Join
-from ..domain import Domain, BooleanDomain, ListDomain
+from ..domain import Domain, BooleanDomain, ListDomain, IdentityDomain
 from .binding import Binding, QueryBinding, SegmentBinding
 from .signature import Signature, Bag, Formula
 
                    self.limit if self.limit is not None else 1)
 
 
+class LocatorFlow(Flow):
+
+    is_axis = True
+
+    def __init__(self, base, seed, filter, binding, companions=[]):
+        assert isinstance(base, Flow)
+        assert isinstance(seed, Flow)
+        assert (isinstance(filter, Code) and
+                isinstance(filter.domain, BooleanDomain))
+        assert seed.spans(base)
+        # We don't need `seed` to be plural or even axial against `base`.
+        #assert not base.spans(seed)
+        assert isinstance(companions, listof(Code))
+        # Determine an axial ancestor of `seed` spanned by `base`
+        # (could be `seed` itself).
+        ground = seed
+        while not ground.is_axis:
+            ground = ground.base
+        if not base.spans(ground):
+            while not base.spans(ground.base):
+                ground = ground.base
+        axis = seed
+        while not axis.is_axis:
+            axis = axis.base
+        is_contracting = (axis.base is None or base.spans(axis.base))
+        super(LocatorFlow, self).__init__(
+                    base=base,
+                    family=seed.family,
+                    is_contracting=is_contracting,
+                    is_expanding=False,
+                    binding=binding)
+        self.seed = seed
+        self.filter = filter
+        self.ground = ground
+        self.companions = companions
+
+    def __basis__(self):
+        return (self.base, self.seed, self.filter)
+
+
 class FilteredFlow(Flow):
     """
     Represents a filtering operation.
         return segments
 
 
+class IdentityCode(Code):
+
+    def __init__(self, fields, binding):
+        assert isinstance(fields, listof(Code))
+        domain = IdentityDomain([field.domain for field in fields])
+        super(IdentityCode, self).__init__(
+                domain=domain,
+                binding=binding)
+        self.fields = fields
+
+    def __basis__(self):
+        return (tuple(self.fields),)
+
+    def get_units(self):
+        units = []
+        for field in self.fields:
+            units.extend(field.units)
+        return units
+
+
 class AnnihilatorCode(Code):
 
     def __init__(self, code, indicator, binding):
                                  MonikerFlow,
                                  ForkedFlow,
                                  LinkedFlow,
-                                 ClippedFlow))
+                                 ClippedFlow,
+                                 LocatorFlow))
         super(CoveringUnit, self).__init__(
                     code=code,
                     flow=flow,

File src/htsql/core/tr/fn/bind.py

 from ..error import BindError
 from ..coerce import coerce
 from ..decorate import decorate
-from ..lookup import direct, expand, guess_tag, lookup_command
+from ..lookup import direct, expand, identify, guess_tag, lookup_command
 from ..signature import (Signature, NullarySig, UnarySig, BinarySig,
                          CompareSig, IsEqualSig, IsTotallyEqualSig, IsInSig,
                          IsNullSig, IfNullSig, NullIfSig, AndSig, OrSig,
         return self.state.bind(lop, scope=binding)
 
 
+class BindId(BindMacro):
+
+    call('id')
+    signature = NullarySig
+
+    def expand(self):
+        recipe = identify(self.state.scope)
+        if recipe is None:
+            raise BindError("cannot determine identity", self.syntax.mark)
+        return self.state.use(recipe, self.syntax)
+
+
 class BindCast(BindFunction):
 
     signature = CastSig

File src/htsql/core/tr/lookup.py

 
 from ..util import Clonable, Printable, maybe
 from ..adapter import Adapter, adapt, adapt_many
+from ..entity import DirectJoin
 from ..model import (HomeNode, TableNode, Arc, TableArc, ChainArc, ColumnArc,
-                     SyntaxArc, InvalidArc, AmbiguousArc)
+        SyntaxArc, InvalidArc, AmbiguousArc)
 from ..classify import classify, relabel, normalize
 from .syntax import IdentifierSyntax
-from .binding import (Binding, ScopingBinding, ChainingBinding, WrappingBinding,
-                      SegmentBinding, HomeBinding, RootBinding, TableBinding,
-                      FreeTableBinding, AttachedTableBinding,
-                      ColumnBinding, QuotientBinding, ComplementBinding,
-                      CoverBinding, ForkBinding, LinkBinding, ClipBinding,
-                      RescopingBinding, DefinitionBinding, SelectionBinding,
-                      WildSelectionBinding, DirectionBinding, RerouteBinding,
-                      ReferenceRerouteBinding, TitleBinding, AliasBinding,
-                      CommandBinding, ImplicitCastBinding,
-                      FreeTableRecipe, AttachedTableRecipe, ColumnRecipe,
-                      ComplementRecipe, KernelRecipe, BindingRecipe,
-                      SubstitutionRecipe, ClosedRecipe, InvalidRecipe,
-                      AmbiguousRecipe)
+from .binding import (Binding, ScopingBinding, ChainingBinding,
+        WrappingBinding, SegmentBinding, HomeBinding, RootBinding,
+        TableBinding, FreeTableBinding, AttachedTableBinding, ColumnBinding,
+        QuotientBinding, ComplementBinding, CoverBinding, ForkBinding,
+        LinkBinding, ClipBinding, LocatorBinding, RescopingBinding,
+        DefinitionBinding, SelectionBinding, WildSelectionBinding,
+        DirectionBinding, RerouteBinding, ReferenceRerouteBinding,
+        TitleBinding, AliasBinding, CommandBinding, ImplicitCastBinding,
+        FreeTableRecipe, AttachedTableRecipe, ColumnRecipe, ComplementRecipe,
+        KernelRecipe, BindingRecipe, IdentityRecipe, ChainRecipe,
+        SubstitutionRecipe, ClosedRecipe, InvalidRecipe, AmbiguousRecipe)
 
 
 class Probe(Clonable, Printable):
         return "?<*>"
 
 
+class IdentityProbe(Probe):
+    pass
+
+
 class GuessNameProbe(Probe):
     """
     Represents a request for an attribute name.
                (ChainingBinding, ReferenceSetProbe),
                (ChainingBinding, ComplementProbe),
                (ChainingBinding, ExpansionProbe),
+               (ChainingBinding, IdentityProbe),
                (ChainingBinding, GuessTitleProbe),
                (ChainingBinding, DirectionProbe))
 
             yield (identifier, recipe)
 
 
+class IdentifyTable(Lookup):
+
+    adapt(TableBinding, IdentityProbe)
+
+    def __call__(self):
+        unique_key = self.binding.table.primary_key
+        if unique_key is None:
+            for key in self.binding.table.unique_keys:
+                if key.is_partial:
+                    continue
+                if all(not column.is_nullable for column in key.origin_columns):
+                    unique_key = key
+                    break
+        if unique_key is None:
+            return None
+        columns = unique_key.origin_columns[:]
+        recipes = []
+        while columns:
+            for foreign_key in self.binding.table.foreign_keys:
+                if foreign_key.is_partial:
+                    continue
+                width = len(foreign_key.origin_columns)
+                if foreign_key.origin_columns == columns[:width]:
+                    join = DirectJoin(foreign_key)
+                    chain = [AttachedTableRecipe([join])]
+                    binding = AttachedTableBinding(self.binding, join,
+                                                   self.binding.syntax)
+                    recipe = lookup(binding, self.probe)
+                    if recipe is None:
+                        return None
+                    chain.append(recipe)
+                    recipe = ChainRecipe(chain)
+                    recipes.append(recipe)
+                    columns = columns[width:]
+                    break
+            else:
+                column = columns.pop(0)
+                recipe = ColumnRecipe(column)
+                recipes.append(recipe)
+        return IdentityRecipe(recipes)
+
+
 class GuessPathForFreeTable(Lookup):
 
     adapt(FreeTableBinding, GuessPathProbe)
 
     adapt_many((ComplementBinding, AttributeProbe),
                (ComplementBinding, AttributeSetProbe),
-               (ComplementBinding, ComplementProbe))
+               (ComplementBinding, ComplementProbe),
+               (ComplementBinding, IdentityProbe))
 
     def __call__(self):
         # Delegate all lookup probes to the seed scope.
         return lookup(self.binding.quotient.seed, probe)
 
 
-
 class LookupAttributeInCover(Lookup):
     # Find an attribute in a cover scope.
 
     adapt_many((CoverBinding, AttributeProbe),
                (CoverBinding, AttributeSetProbe),
                (CoverBinding, ComplementProbe),
+               (CoverBinding, IdentityProbe),
                (ClipBinding, AttributeProbe),
                (ClipBinding, AttributeSetProbe),
-               (ClipBinding, ComplementProbe))
+               (ClipBinding, ComplementProbe),
+               (ClipBinding, IdentityProbe),
+               (LocatorBinding, AttributeProbe),
+               (LocatorBinding, AttributeSetProbe),
+               (LocatorBinding, ComplementProbe),
+               (LocatorBinding, IdentityProbe))
 
     def __call__(self):
         # Delegate all lookup requests to the seed flow.
     # Expand public columns from a cover scope.
 
     adapt_many((CoverBinding, ExpansionProbe),
-               (ClipBinding, ExpansionProbe))
+               (ClipBinding, ExpansionProbe),
+               (LocatorBinding, ExpansionProbe))
 
     def __call__(self):
         # Ignore pure selector expansion probes.
         return lookup(self.binding.seed, probe)
 
 
+class GuessTagAndHeaderFromLocator(Lookup):
+
+    adapt_many((LocatorBinding, GuessTagProbe),
+               (LocatorBinding, GuessHeaderProbe))
+
+    def __call__(self):
+        return lookup(self.binding.seed, self.probe)
+
+
 class LookupAttributeInFork(Lookup):
     # Find an attribute or a complement link in a fork scope.
 
     adapt_many((ForkBinding, AttributeProbe),
                (ForkBinding, AttributeSetProbe),
-               (ForkBinding, ComplementProbe))
+               (ForkBinding, ComplementProbe),
+               (ForkBinding, IdentityProbe))
 
     def __call__(self):
         # Delegate all lookup probes to the parent binding.
 
     adapt_many((LinkBinding, AttributeProbe),
                (LinkBinding, AttributeSetProbe),
-               (LinkBinding, ComplementProbe))
+               (LinkBinding, ComplementProbe),
+               (LinkBinding, IdentityProbe))
 
     def __call__(self):
         # Delegate all lookup probes to the seed scope.
     return recipes
 
 
+def identify(binding):
+    probe = IdentityProbe()
+    return lookup(binding, probe)
+
+
 #def guess_name(binding):
 #    """
 #    Extracts an attribute name from the given binding.

File src/htsql/core/tr/parse.py

 
 from ..mark import Mark
 from .scan import scan
-from .token import NameToken, StringToken, NumberToken, SymbolToken, EndToken
+from .token import (NameToken, StringToken, UnquotedStringToken, NumberToken,
+        SymbolToken, EndToken)
 from .syntax import (QuerySyntax, SegmentSyntax, CommandSyntax, SelectorSyntax,
-                     FunctionSyntax, MappingSyntax, OperatorSyntax,
-                     QuotientSyntax, SieveSyntax, LinkSyntax, HomeSyntax,
-                     AssignmentSyntax, SpecifierSyntax, GroupSyntax,
-                     IdentifierSyntax, WildcardSyntax, ComplementSyntax,
-                     ReferenceSyntax, StringSyntax, NumberSyntax)
+        FunctionSyntax, MappingSyntax, OperatorSyntax, QuotientSyntax,
+        SieveSyntax, LinkSyntax, HomeSyntax, AssignmentSyntax, SpecifierSyntax,
+        LocatorSyntax, LocationSyntax, GroupSyntax, IdentifierSyntax,
+        WildcardSyntax, ComplementSyntax, ReferenceSyntax, StringSyntax,
+        UnquotedStringSyntax, NumberSyntax)
 from .error import ParseError
 
 
         link            ::= '->' flow
         assignment      ::= ':=' top
 
-        specifier       ::= atom ( '.' atom )*
+        specifier       ::= locator ( '.' locator )*
+        locator         ::= atom ( '[' location ']' )?
+        location        ::= label ( '.' label )*
+        label           ::= STRING | '(' location ')' | '[' location ']'
+
         atom            ::= '@' atom | '*' index? | '^' | selector | group |
                             identifier call? | reference | literal
         index           ::= NUMBER | '(' NUMBER ')'
     @classmethod
     def process(cls, tokens):
         # Expect:
-        #   specifier   ::= atom ( '.' atom )*
-        specifier = AtomParser << tokens
+        #   specifier   ::= locator ( '.' locator )*
+        specifier = LocatorParser << tokens
         while tokens.peek(SymbolToken, [u'.']):
             tokens.pop(SymbolToken, [u'.'])
             lbranch = specifier
-            rbranch = AtomParser << tokens
+            rbranch = LocatorParser << tokens
             mark = Mark.union(lbranch, rbranch)
             specifier = SpecifierSyntax(lbranch, rbranch, mark)
         return specifier
 
 
+class LocatorParser(Parser):
+    """
+    Parses a `locator` production.
+    """
+
+    @classmethod
+    def process(cls, tokens):
+        # Expect:
+        #   locator         ::= atom ( '[' location ']' )?
+        locator = AtomParser << tokens
+        if tokens.peek(SymbolToken, [u'[']):
+            location = LocationParser << tokens
+            mark = Mark.union(locator, location)
+            locator = LocatorSyntax(locator, location, mark)
+        return locator
+
+
+class LocationParser(Parser):
+    """
+    Parses a `location` production.
+    """
+
+    @classmethod
+    def process(cls, tokens):
+        # Expect:
+        #   location        ::= label ( '.' label )*
+        open_token = tokens.pop(SymbolToken, [u'[', u'('])
+        labels = [LabelParser << tokens]
+        while tokens.peek(SymbolToken, [u'.']):
+            tokens.pop(SymbolToken, [u'.'])
+            labels.append(LabelParser << tokens)
+        if open_token.value == u'[' and not tokens.peek(SymbolToken, [u']']):
+            mark = Mark.union(open_token, *labels)
+            raise ParseError("cannot find a matching ']'", mark)
+        elif open_token.value == u'(' and not tokens.peek(SymbolToken, [u')']):
+            mark = Mark.union(open_token, *labels)
+            raise ParseError("cannot find a matching ')'", mark)
+        close_token = tokens.pop(SymbolToken, [u']', u')'])
+        mark = Mark.union(open_token, close_token)
+        location = LocationSyntax(labels, mark)
+        return location
+
+
+class LabelParser(Parser):
+    """
+    Parses a `label` production.
+    """
+
+    @classmethod
+    def process(cls, tokens):
+        # Expect:
+        #   label           ::= STRING | '(' location ')' | '[' location ']'
+        if tokens.peek(UnquotedStringToken):
+            token = tokens.pop(UnquotedStringToken)
+            return UnquotedStringSyntax(token.value, token.mark)
+        elif tokens.peek(StringToken):
+            token = tokens.pop(StringToken)
+            return StringSyntax(token.value, token.mark)
+        elif tokens.peek(SymbolToken, [u'[', u'(']):
+            label = LocationParser << tokens
+            return label
+        if tokens.peek(EndToken):
+            token = tokens.pop(EndToken)
+            raise ParseError("unexpected end of query", token.mark)
+        else:
+            token = tokens.pop()
+            raise ParseError("unexpected symbol '%s'" % token, token.mark)
+
+
 class AtomParser(Parser):
     """
     Parses an `atom` production.
             token = tokens.pop(NumberToken)
             return NumberSyntax(token.value, token.mark)
         # Can't find anything suitable; produce an error message.
-        token = tokens.pop()
-        if token.is_end:
+        if tokens.peek(EndToken):
+            token = tokens.pop(EndToken)
             raise ParseError("unexpected end of query", token.mark)
         else:
+            token = tokens.pop()
             raise ParseError("unexpected symbol '%s'" % token, token.mark)
         # Not reachable.
         assert False

File src/htsql/core/tr/rewrite.py

 from .error import EncodeError
 from .coerce import coerce
 from .flow import (Expression, QueryExpr, SegmentCode, Flow, RootFlow,
-                   QuotientFlow, ComplementFlow, MonikerFlow, ForkedFlow,
-                   LinkedFlow, ClippedFlow, FilteredFlow, OrderedFlow,
-                   Code, LiteralCode, CastCode, RecordCode,
-                   AnnihilatorCode, FormulaCode, Unit,
-                   CompoundUnit, ScalarUnit, AggregateUnitBase, AggregateUnit,
-                   KernelUnit, CoveringUnit)
+        QuotientFlow, ComplementFlow, MonikerFlow, ForkedFlow, LinkedFlow,
+        ClippedFlow, LocatorFlow, FilteredFlow, OrderedFlow, Code, LiteralCode,
+        CastCode, RecordCode, IdentityCode, AnnihilatorCode, FormulaCode, Unit,
+        CompoundUnit, ScalarUnit, AggregateUnitBase, AggregateUnit, KernelUnit,
+        CoveringUnit)
 from .signature import Signature, OrSig, AndSig
 # FIXME: move `IfSig` and `SwitchSig` to `htsql.core.tr.signature`.
 from .fn.signature import IfSig
         return self.flow.clone(base=base, seed=seed)
 
 
+class RewriteLocator(RewriteFlow):
+
+    adapt(LocatorFlow)
+
+    def __call__(self):
+        # Apply the adapter to all child nodes.
+        base = self.state.rewrite(self.flow.base)
+        seed = self.state.rewrite(self.flow.seed)
+        filter = self.state.rewrite(self.flow.filter)
+        return self.flow.clone(base=base, seed=seed, filter=filter)
+
+
+class UnmaskLocator(UnmaskFlow):
+
+    adapt(LocatorFlow)
+
+    def __call__(self):
+        # Unmask the seed flow against the parent flow.
+        seed = self.state.unmask(self.flow.seed, mask=self.flow.base)
+        # Unmask the parent flow against the current mask.
+        base = self.state.unmask(self.flow.base)
+        filter = self.state.unmask(self.flow.filter, mask=self.flow.seed)
+        return self.flow.clone(base=base, seed=seed, filter=filter)
+
+
+class ReplaceLocator(Replace):
+
+    adapt(LocatorFlow)
+
+    def __call__(self):
+        base = self.state.replace(self.flow.base)
+        substate = self.state.spawn()
+        substate.collect(self.flow.seed)
+        substate.collect(self.flow.filter)
+        substate.recombine()
+        seed = substate.replace(self.flow.seed)
+        filter = substate.replace(self.flow.filter)
+        return self.flow.clone(base=base, seed=seed, filter=filter)
+
+
 class RewriteForked(RewriteFlow):
 
     adapt(ForkedFlow)
 
 class RewriteRecord(Rewrite):
 
-    adapt(RecordCode)
+    adapt_many(RecordCode,
+               IdentityCode)
 
     def __call__(self):
         fields = [self.state.rewrite(field)
 
 class UnmaskRecord(Unmask):
 
-    adapt(RecordCode)
+    adapt_many(RecordCode,
+               IdentityCode)
 
     def __call__(self):
         fields = [self.state.unmask(field)
 
 class CollectRecord(Collect):
 
-    adapt(RecordCode)
+    adapt_many(RecordCode,
+               IdentityCode)
 
     def __call__(self):
         for field in self.code.fields:
 
 class ReplaceRecord(Replace):
 
-    adapt(RecordCode)
+    adapt_many(RecordCode,
+               IdentityCode)
 
     def __call__(self):
         fields = [self.state.replace(field)

File src/htsql/core/tr/scan.py

 """
 
 
-from .token import (Token, SpaceToken, NameToken, StringToken, NumberToken,
-                    SymbolToken, EndToken)
+from .token import (Token, NameToken, StringToken, UnquotedStringToken,
+        NumberToken, SymbolToken, EndToken)
 from .error import ScanError, ParseError
 from ..mark import Mark
 from ..util import maybe, listof
                          do_pop=True, do_force=do_force)
 
 
+class ScanRule(object):
+
+    def __init__(self, pattern, token_class,
+                 unquote=None, jump=None, error=None, is_end=False):
+        assert isinstance(pattern, str)
+        assert isinstance(token_class, maybe(type))
+        assert isinstance(jump, maybe(str))
+        assert isinstance(error, maybe(str))
+        assert isinstance(is_end, bool)
+        self.pattern = pattern
+        self.token_class = token_class
+        self.unquote = unquote
+        self.jump = jump
+        self.error = error
+        self.is_end = is_end
+
+
+class ScanGroup(object):
+
+    def __init__(self, name, *rules):
+        assert isinstance(name, str)
+        rules = list(rules)
+        assert isinstance(rules, listof(ScanRule))
+        self.name = name
+        self.rules = rules
+        self.init_regexp()
+
+    def init_regexp(self):
+        patterns = []
+        for idx, rule in enumerate(self.rules):
+            pattern = r"(?P<_%s> %s)" % (idx, rule.pattern)
+            patterns.append(pattern)
+        pattern = r"|".join(patterns)
+        self.regexp = re.compile(pattern, re.X|re.U)
+
+
 class Scanner(object):
     """
     Implements the HTSQL scanner.
         An HTSQL query.
     """
 
-    # List of tokens generated by the scanner.
-    tokens = [
-            SpaceToken,
-            NameToken,
-            NumberToken,
-            StringToken,
-            SymbolToken,
-            EndToken,
+    groups = [
+            ScanGroup('top',
+                ScanRule(r""" \s+ """, None),
+                ScanRule(r""" (?! \d) \w+ """, NameToken),
+                ScanRule(r""" ' (?: [^'\0] | '')* ' """, StringToken,
+                         unquote=(lambda v: v[1:-1].replace(u'\'\'', u'\''))),
+                ScanRule(r""" ' """, None,
+                         error="""cannot find a matching quote mark"""),
+                ScanRule(r""" (?: \d* \.)? \d+ [eE] [+-]? \d+ |"""
+                         r""" \d* \. \d+ | \d+ \.? """, NumberToken),
+                ScanRule(r""" ~ | !~ | <= | < | >= | > | == | = | !== | != |"""
+                         r""" ! | & | \| | -> | \. | , | \? | \^ | / | \* |"""
+                         r""" \+ | - | \( | \) | \{ | \} | := | : | \$ | @ """,
+                         SymbolToken),
+                ScanRule(r""" \[ """, SymbolToken, jump='locator'),
+                ScanRule(r""" \] """, None,
+                         error="""cannot find a matching '['"""),
+                ScanRule(r""" $ """, EndToken, is_end=True)),
+            ScanGroup('locator',
+                ScanRule(r""" \s+ """, None),
+                ScanRule(r""" \[ | \( """, SymbolToken,
+                         jump='locator'),
+                ScanRule(r""" \] | \) """, SymbolToken,
+                         is_end=True),
+                ScanRule(r""" \. """, SymbolToken),
+                ScanRule(r""" [\w-]+ """, UnquotedStringToken),
+                ScanRule(r""" ' (?: [^'\0] | '')* ' """, StringToken,
+                         unquote=(lambda v: v[1:-1].replace(u'\'\'', u'\''))),
+                ScanRule(r""" ' """, None,
+                         error="""cannot find a matching quote mark"""),
+                ScanRule(r""" $ """, None,
+                         error="""cannot find a matching ']'""")),
     ]
-    # A regular expression that matches an HTSQL token.  The expression
-    # is compiled from individual token patterns on the first instantiation
-    # of the scanner.
-    regexp = None
 
     # The regular expression to match %-escape sequences.
     escape_pattern = r"""%(?P<code>[0-9A-Fa-f]{2})?"""
         assert isinstance(input, (str, unicode))
         if isinstance(input, unicode):
             input = input.encode('utf-8')
-
         self.input = input
-        self.init_regexp()
-
-    @classmethod
-    def init_regexp(cls):
-        # Compile the regular expression that matches all token types.
-
-        # If it is already done, return.
-        if cls.regexp is not None:
-            return
-
-        # Get patterns for each token type grouped by the class name.
-        token_patterns = []
-        for token_class in cls.tokens:
-            pattern = r"(?P<%s> %s)" % (token_class.name, token_class.pattern)
-            token_patterns.append(pattern)
-
-        # Combine the individual patterns and compile the expression.
-        pattern = r"|".join(token_patterns)
-        cls.regexp = re.compile(pattern, re.X|re.U)
 
     def unquote(self, data):
         # Decode %-escape sequences.
 
         # The beginning of the next token (and the start of the mark slice).
         start = 0
-        # Have we reached the final token?
-        is_end = False
+        stack = [self.groups[0]]
+        group_by_name = dict((group.name, group) for group in self.groups)
         # The list of generated tokens.
         tokens = []
 
-        # Loop till we get the final token.
-        while not is_end:
-            # Match the next token.
-            match = self.regexp.match(input, start)
+        while stack:
+            group = stack[-1]
+            match = group.regexp.match(input, start)
             if match is None:
                 mark = Mark(input, start, start+1)
-                # FIXME: generalize?
-                if input[start] == u"'":
-                    raise ScanError("cannot find a matching quote mark", mark)
-                else:
-                    raise ScanError("unexpected symbol %r"
-                                    % input[start].encode('utf-8'), mark)
-
-            # Find the token class that matched the token.
-            for token_class in self.tokens:
-                group = match.group(token_class.name)
-                if group is not None:
+                raise ScanError("unexpected symbol %r"
+                                % input[start].encode('utf-8'), mark)
+            end = match.end()
+            mark = Mark(input, start, end)
+            for idx, rule in enumerate(group.rules):
+                value = match.group("_%s" % idx)
+                if value is not None:
                     break
             else:
-                # Unreachable.
+                # Unreachable
                 assert False
-
-            # Unquote the token value.
-            value = token_class.unquote(group)
-            # The end of the token (and the mark slice).
-            end = match.end()
-
-            # Skip whitespace tokens; otherwise produce the next token.
-            if not token_class.is_ws:
-                mark = Mark(input, start, end)
-                token = token_class(value, mark)
+            if rule.unquote is not None:
+                value = rule.unquote(value)
+            if rule.token_class is not None:
+                token = rule.token_class(value, mark)
                 tokens.append(token)
-
-            # Check if we reached the final token.
-            if token_class.is_end:
-                is_end = True
-
+            if rule.is_end:
+                stack.pop()
+            if rule.jump is not None:
+                assert rule.jump in group_by_name
+                stack.append(group_by_name[rule.jump])
+            if rule.error is not None:
+                raise ScanError(rule.error, mark)
             # Advance the pointer to the beginning of the next token.
             start = end
 

File src/htsql/core/tr/stitch.py

 from .error import CompileError
 from .syntax import IdentifierSyntax
 from .flow import (Flow, ScalarFlow, TableFlow, FiberTableFlow, QuotientFlow,
-                   ComplementFlow, MonikerFlow, ForkedFlow, LinkedFlow,
-                   ClippedFlow, OrderedFlow, ColumnUnit, KernelUnit,
-                   CoveringUnit)
+        ComplementFlow, MonikerFlow, ForkedFlow, LinkedFlow, ClippedFlow,
+        LocatorFlow, OrderedFlow, ColumnUnit, KernelUnit, CoveringUnit)
 from .term import Joint
 
 
                MonikerFlow,
                ForkedFlow,
                LinkedFlow,
-               ClippedFlow)
+               ClippedFlow,
+               LocatorFlow)
 
     def __call__(self):
         # Start with the parent ordering.
                MonikerFlow,
                ForkedFlow,
                LinkedFlow,
-               ClippedFlow)
+               ClippedFlow,
+               LocatorFlow)
 
     def __call__(self):
         # Native units of the complement are inherited from the seed flow.
                MonikerFlow,
                LinkedFlow,
                ForkedFlow,
-               ClippedFlow)
+               ClippedFlow,
+               LocatorFlow)
 
     def __call__(self):
         # To sew two terms representing a covering flow, we sew all axial flows
             yield joint.clone(rop=rop)
 
 
+class TieLocator(Tie):
+
+    adapt(LocatorFlow)
+
+    def __call__(self):
+        flow = self.flow.inflate()
+        for joint in tie(flow.ground):
+            rop = CoveringUnit(joint.rop, flow, joint.rop.binding)
+            yield joint.clone(rop=rop)
+
+
 class TieClipped(Tie):
 
     adapt(ClippedFlow)

File src/htsql/core/tr/syntax.py

 
 
 from ..mark import Mark, EmptyMark
-from ..util import maybe, listof, Printable, Clonable, Comparable
+from ..util import maybe, listof, oneof, Printable, Clonable, Comparable
 import re
 
 
         super(SpecifierSyntax, self).__init__(u'.', lbranch, rbranch, mark)
 
 
+class LocatorSyntax(Syntax):
+
+    def __init__(self, lbranch, rbranch, mark):
+        assert isinstance(lbranch, Syntax)
+        assert isinstance(rbranch, LocationSyntax)
+        super(LocatorSyntax, self).__init__(mark)
+        self.lbranch = lbranch
+        self.rbranch = rbranch
+
+    def __basis__(self):
+        return (self.lbranch, self.rbranch)
+
+    def __unicode__(self):
+        return u"%s%s" % (self.lbranch, self.rbranch)
+
+
+class LocationSyntax(Syntax):
+
+    def __init__(self, branches, mark):
+        assert isinstance(branches, listof(oneof(LocationSyntax,
+                                                 StringSyntax)))
+        super(LocationSyntax, self).__init__(mark)
+        self.branches = branches
+        self.arity = 0
+        for branch in self.branches:
+            if isinstance(branch, LocationSyntax):
+                self.arity += branch.arity
+            else:
+                self.arity += 1
+
+    def __basis__(self):
+        return (tuple(self.branches),)
+
+    def __unicode__(self):
+        return u"[%s]" % u".".join(str(branch) for branch in self.branches)
+
+
 class GroupSyntax(Syntax):
     """
     Represents an expression in parentheses.
         return value
 
 
+class UnquotedStringSyntax(StringSyntax):
+
+    def __unicode__(self):
+        return self.escape_regexp.sub(self.escape_replace, self.value)
+
+
 class NumberSyntax(LiteralSyntax):
     """
     Represents a number literal.

File src/htsql/core/tr/token.py

     subclass of :class:`Token` and override the following class attributes:
 
     `name` (a string)
-        The name of the token category.  Must be a valid identifier since
-        it is used by the scanner as the name of the pattern group.
-
-    `pattern` (a string)
-        The regular expression to match the token (in the verbose format).
-
-    `is_ws` (Boolean)
-        If set, indicates that the token is to be discarded.
-
-    `is_end` (Boolean)
-        If set, forces the scanner to stop the processing.
-
-    When adding a subclass of :class:`Token`, you may also want to override
-    methods :meth:`unquote` and :meth:`quote`.
+        The name of the token category.
 
     The constructor of :class:`Token` accepts the following parameters:
 
     """
 
     name = None
-    pattern = None
-    is_ws = False
-    is_end = False
-
-    @classmethod
-    def unquote(cls, value):
-        """
-        Converts a raw string that matches the token pattern to a token value.
-        """
-        return value
-
-    @classmethod
-    def quote(cls, value):
-        """
-        Reverses :meth:`unquote`.
-        """
-        return value
 
     def __init__(self, value, mark):
         assert isinstance(value, unicode)
         return self.value.encode('utf-8')
 
 
-class SpaceToken(Token):
-    """
-    Represents a whitespace token.
-
-    In HTSQL, whitespace characters are space, tab, LF, CR, FF, VT and
-    those Unicode characters that are classified as space.
-
-    Whitespace tokens are discarded by the scanner without passing them
-    to the parser.
-    """
-
-    name = 'whitespace'
-    pattern = r""" \s+ """
-    # Do not pass the token to the parser.
-    is_ws = True
-
-
 class NameToken(Token):
     """
     Represents a name token.
     """
 
     name = 'name'
-    pattern = r""" (?! \d) \w+ """
 
 
 class StringToken(Token):
     """
 
     name = 'string'
-    # Note: we do not permit `NUL` characters in a string literal.
-    pattern = r""" ' (?: [^'\0] | '')* ' """
 
-    @classmethod
-    def unquote(cls, value):
-        # Strip leading and trailing quotes and replace `''` with `'`.
-        return value[1:-1].replace(u'\'\'', u'\'')
 
-    @classmethod
-    def quote(cls, value):
-        # Replace all occurences of `'` with `''`, enclose the string
-        # in the quotes.
-        return u'\'%s\'' % value.replace(u'\'', u'\'\'')
+class UnquotedStringToken(StringToken):
+    pass
 
 
 class NumberToken(Token):
     """
 
     name = 'number'
-    pattern = r""" (?: \d* \.)? \d+ [eE] [+-]? \d+ | \d* \. \d+ | \d+ \.? """
 
 
 class SymbolToken(Token):
     """
 
     name = 'symbol'
-    pattern = r"""
-        ~ | !~ | <= | < | >= | > | == | = | !== | != | ! |
-        & | \| | -> | \. | , | \? | \^ | / | \* | \+ | - |
-        \( | \) | \{ | \} | \[ | \] | := | : | \$ | @
-    """
 
 
 class EndToken(Token):
     """
 
     name = 'end'
-    pattern = r""" $ """
-    is_end = True
 
 

File test/input/format.yaml

            date('2010-04-15'), time('20:13:04.5'), datetime('2010-04-15 20:13')}
           /:txt
 
+- title: Identity
+  tests:
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:raw
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:json
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:csv
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:tsv
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:xml
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:html
+  - uri: /enrollment[1010.((mth.101).(2008.fall).001)]{id()}
+         /:txt
+
 - title: No Rows
   tests:
   - uri: /school?false()/:raw

File test/input/translation.yaml

     # parsing a command after an infix function call
     - uri: /school:top/:csv
 
+  - title: Identity
+    tests:
+    - uri: /school[ns]
+    - uri: /school[ns].department
+    - uri: /school[ns].program
+    - uri: /program?school[ns]
+    - uri: /program[ns.uchem]
+    - uri: /school{id(), *}
+    - uri: /department{id(), school{id()}}
+    - uri: /course[astro.105].class.id()
+    - uri: /program^degree{*, /^.id()}
+    - uri: /program[ns]
+      expect: 400
+    - uri: /school[ns.uchem]
+      expect: 400
+    - uri: /course[ns.uchem]
+      expect: 400
+    - uri: /program^degree{id()}
+      expect: 400
+
   - title: Table Expressions
     tests:
     - uri: /(school?code='art').department

File test/output/mssql.yaml

             - [Content-Type, text/csv; charset=UTF-8]
             - [Content-Disposition, 'attachment; filename="school:top.csv"']
             body: "code,name,campus\r\nart,School of Art & Design,old\r\n"
+        - id: identity
+          tests:
+          - uri: /school[ns]
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | school                                     |
+               +------+----------------------------+--------+
+               | code | name                       | campus |
+              -+------+----------------------------+--------+-
+               | ns   | School of Natural Sciences | old    |
+
+               ----
+               /school[ns]
+               SELECT [school].[code],
+                      [school].[name],
+                      [school].[campus]
+               FROM [ad].[school]
+               WHERE ([school].[code] = 'ns')
+               ORDER BY 1 ASC
+          - uri: /school[ns].department
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | department                        |
+               +-------+-------------+-------------+
+               | code  | name        | school_code |
+              -+-------+-------------+-------------+-
+               | astro | Astronomy   | ns          |
+               | chem  | Chemistry   | ns          |
+               | mth   | Mathematics | ns          |
+               | phys  | Physics     | ns          |
+
+               ----
+               /school[ns].department
+               SELECT [department].[code],
+                      [department].[name],
+                      [department].[school_code]
+               FROM [ad].[school]
+                    INNER JOIN [ad].[department]
+                               ON ([school].[code] = [department].[school_code])
+               WHERE ([school].[code] = 'ns')
+               ORDER BY [school].[code] ASC, 1 ASC
+          - uri: /school[ns].program
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | program                                                                         |
+               +-------------+--------+----------------------------------+--------+--------------+
+               | school_code | code   | title                            | degree | part_of_code |
+              -+-------------+--------+----------------------------------+--------+--------------+-
+               | ns          | gmth   | Masters of Science in            | ms     | pmth         |
+               :             :        : Mathematics                      :        :              :
+               | ns          | pmth   | Doctorate of Science in          | ph     |              |
+               :             :        : Mathematics                      :        :              :
+               | ns          | uastro | Bachelor of Science in Astronomy | bs     |              |
+               | ns          | uchem  | Bachelor of Science in Chemistry | bs     |              |
+               | ns          | umth   | Bachelor of Science in           | bs     | gmth         |
+               :             :        : Mathematics                      :        :              :
+               | ns          | uphys  | Bachelor of Science in Physics   | bs     |              |
+
+               ----
+               /school[ns].program
+               SELECT [program].[school_code],
+                      [program].[code],
+                      [program].[title],
+                      [program].[degree],
+                      [program].[part_of_code]
+               FROM [ad].[school]
+                    INNER JOIN [ad].[program]
+                               ON ([school].[code] = [program].[school_code])
+               WHERE ([school].[code] = 'ns')
+               ORDER BY [school].[code] ASC, 1 ASC, 2 ASC
+          - uri: /program?school[ns]
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | program                                                                         |
+               +-------------+--------+----------------------------------+--------+--------------+
+               | school_code | code   | title                            | degree | part_of_code |
+              -+-------------+--------+----------------------------------+--------+--------------+-
+               | ns          | gmth   | Masters of Science in            | ms     | pmth         |
+               :             :        : Mathematics                      :        :              :
+               | ns          | pmth   | Doctorate of Science in          | ph     |              |
+               :             :        : Mathematics                      :        :              :
+               | ns          | uastro | Bachelor of Science in Astronomy | bs     |              |
+               | ns          | uchem  | Bachelor of Science in Chemistry | bs     |              |
+               | ns          | umth   | Bachelor of Science in           | bs     | gmth         |
+               :             :        : Mathematics                      :        :              :
+               | ns          | uphys  | Bachelor of Science in Physics   | bs     |              |
+
+               ----
+               /program?school[ns]
+               SELECT [program].[school_code],
+                      [program].[code],
+                      [program].[title],
+                      [program].[degree],
+                      [program].[part_of_code]
+               FROM [ad].[program]
+                    LEFT OUTER JOIN (SELECT 1 AS [!],
+                                            [school].[code]
+                                     FROM [ad].[school]
+                                     WHERE ([school].[code] = 'ns')) AS [school]
+                                    ON ([program].[school_code] = [school].[code])
+               WHERE ([school].[!] IS NOT NULL)
+               ORDER BY 1 ASC, 2 ASC
+          - uri: /program[ns.uchem]
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | program                                                                        |
+               +-------------+-------+----------------------------------+--------+--------------+
+               | school_code | code  | title                            | degree | part_of_code |
+              -+-------------+-------+----------------------------------+--------+--------------+-
+               | ns          | uchem | Bachelor of Science in Chemistry | bs     |              |
+
+               ----
+               /program[ns.uchem]
+               SELECT [program].[school_code],
+                      [program].[code],
+                      [program].[title],
+                      [program].[degree],
+                      [program].[part_of_code]
+               FROM [ad].[program]
+                    INNER JOIN [ad].[school]
+                               ON ([program].[school_code] = [school].[code])
+               WHERE ([school].[code] = 'ns')
+                     AND ([program].[code] = 'uchem')
+               ORDER BY 1 ASC, 2 ASC
+          - uri: /school{id(), *}
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | school                                               |
+               +------+------+-------------------------------+--------+
+               | id() | code | name                          | campus |
+              -+------+------+-------------------------------+--------+-
+               | art  | art  | School of Art & Design        | old    |
+               | bus  | bus  | School of Business            | south  |
+               | edu  | edu  | College of Education          | old    |
+               | eng  | eng  | School of Engineering         | north  |
+               | la   | la   | School of Arts and Humanities | old    |
+               | mus  | mus  | School of Music & Dance       | south  |
+               | ns   | ns   | School of Natural Sciences    | old    |
+               | ph   | ph   | Public Honorariums            |        |
+               | sc   | sc   | School of Continuing Studies  |        |
+
+               ----
+               /school{id(),*}
+               SELECT [school].[code],
+                      [school].[name],
+                      [school].[campus]
+               FROM [ad].[school]
+               ORDER BY 1 ASC
+          - uri: /department{id(), school{id()}}
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | department      |
+               +--------+--------+
+               |        | school |
+               |        +--------+
+               | id()   | id()   |
+              -+--------+--------+-
+               | acc    | bus    |
+               | arthis | la     |
+               | astro  | ns     |
+               | be     | eng    |
+               | bursar |        :
+               | career |        :
+               | chem   | ns     |
+               | comp   | eng    |
+               | econ   | bus    |
+               | edpol  | edu    |
+               | ee     | eng    |
+               | eng    | la     |
+               | hist   | la     |
+               | lang   | la     |
+               | me     | eng    |
+               | mm     | bus    |
+               | mth    | ns     |
+               | parent |        :
+               | phys   | ns     |
+               | pia    | mus    |
+               | poli   | la     |
+               | psych  | la     |
+               | stdart | art    |
+               | str    | mus    |
+               | tched  | edu    |
+               | voc    | mus    |
+               | win    | mus    |
+
+               ----
+               /department{id(),school{id()}}
+               SELECT [department].[code],
+                      [school].[id],
+                      [school].[code]
+               FROM [ad].[department]
+                    LEFT OUTER JOIN (SELECT 1 AS [id],
+                                            [school].[code]
+                                     FROM [ad].[school]) AS [school]
+                                    ON ([department].[school_code] = [school].[code])
+               ORDER BY 1 ASC
+          - uri: /course[astro.105].class.id()
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | id()                          |
+              -+-------------------------------+-
+               | (astro.105).(2007.spring).001 |
+               | (astro.105).(2008.spring).001 |
+               | (astro.105).(2009.spring).001 |
+               | (astro.105).(2010.spring).001 |
+
+               ----
+               /course[astro.105].class.id()
+               SELECT [department_2].[code],
+                      [course_2].[no],
+                      [semester].[year],
+                      [semester].[season],
+                      [class].[section]
+               FROM [ad].[course] AS [course_1]
+                    INNER JOIN [ad].[department] AS [department_1]
+                               ON ([course_1].[department_code] = [department_1].[code])
+                    INNER JOIN [cd].[class]
+                               ON (([course_1].[department_code] = [class].[department_code]) AND ([course_1].[no] = [class].[course_no]))
+                    INNER JOIN [ad].[course] AS [course_2]
+                               ON (([class].[department_code] = [course_2].[department_code]) AND ([class].[course_no] = [course_2].[no]))
+                    INNER JOIN [ad].[department] AS [department_2]
+                               ON ([course_2].[department_code] = [department_2].[code])
+                    INNER JOIN [cd].[semester]
+                               ON (([class].[year] = [semester].[year]) AND ([class].[season] = [semester].[season]))
+               WHERE ([department_1].[code] = 'astro')
+                     AND ([course_1].[no] = 105)
+               ORDER BY [course_1].[department_code] ASC, [course_1].[no] ASC, [class].[department_code] ASC, [class].[course_no] ASC, [class].[year] ASC, [class].[season] ASC, 5 ASC
+          - uri: /program^degree{*, /^.id()}
+            status: 200 OK
+            headers:
+            - [Content-Type, text/plain; charset=UTF-8]
+            - [Vary, Accept]
+            body: |2
+               | program^degree        |
+               +--------+--------------+
+               | degree | id()         |
+              -+--------+--------------+-
+               | ba     | art.uhist    |
+               :        | art.ustudio  |
+               :        | bus.uecon    |
+               :        | edu.umath    |
+               :        | edu.usci     |
+               :        | la.uengl     |
+               :        | la.uhist     |
+               :        | la.upolisci  |
+               :        | la.upsych    |
+               :        | la.uspan     |
+               | bs     | bus.uacct    |
+               :        | bus.ubusad   |
+               :        | eng.ubio     |
+               :        | eng.ucompsci |
+               :        | eng.uelec    |
+               :        | eng.umech    |