Source

pyyaml / tests / test_tokens.py

Full commit
Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 

Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 

















Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 



















Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 













Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 
Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 


Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 




Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 
Kirill Simonov ed172c1 
Kirill Simonov 659ccb6 


Kirill Simonov ed172c1 

import yaml
import pprint

# Tokens mnemonic:
# directive:            %
# document_start:       ---
# document_end:         ...
# alias:                *
# anchor:               &
# tag:                  !
# scalar                _
# block_sequence_start: [[
# block_mapping_start:  {{
# block_end:            ]}
# flow_sequence_start:  [
# flow_sequence_end:    ]
# flow_mapping_start:   {
# flow_mapping_end:     }
# entry:                ,
# key:                  ?
# value:                :

_replaces = {
    yaml.DirectiveToken: '%',
    yaml.DocumentStartToken: '---',
    yaml.DocumentEndToken: '...',
    yaml.AliasToken: '*',
    yaml.AnchorToken: '&',
    yaml.TagToken: '!',
    yaml.ScalarToken: '_',
    yaml.BlockSequenceStartToken: '[[',
    yaml.BlockMappingStartToken: '{{',
    yaml.BlockEndToken: ']}',
    yaml.FlowSequenceStartToken: '[',
    yaml.FlowSequenceEndToken: ']',
    yaml.FlowMappingStartToken: '{',
    yaml.FlowMappingEndToken: '}',
    yaml.BlockEntryToken: ',',
    yaml.FlowEntryToken: ',',
    yaml.KeyToken: '?',
    yaml.ValueToken: ':',
}

def test_tokens(data_filename, tokens_filename, verbose=False):
    tokens1 = []
    tokens2 = open(tokens_filename, 'rb').read().split()
    try:
        for token in yaml.scan(open(data_filename, 'rb')):
            if not isinstance(token, (yaml.StreamStartToken, yaml.StreamEndToken)):
                tokens1.append(_replaces[token.__class__])
    finally:
        if verbose:
            print "TOKENS1:", ' '.join(tokens1)
            print "TOKENS2:", ' '.join(tokens2)
    assert len(tokens1) == len(tokens2), (tokens1, tokens2)
    for token1, token2 in zip(tokens1, tokens2):
        assert token1 == token2, (token1, token2)

test_tokens.unittest = ['.data', '.tokens']

def test_scanner(data_filename, canonical_filename, verbose=False):
    for filename in [data_filename, canonical_filename]:
        tokens = []
        try:
            for token in yaml.scan(open(filename, 'rb')):
                tokens.append(token.__class__.__name__)
        finally:
            if verbose:
                pprint.pprint(tokens)

test_scanner.unittest = ['.data', '.canonical']

if __name__ == '__main__':
    import test_appliance
    test_appliance.run(globals())