Source

python-clinic / Tools / scripts / pindent.py

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
#! /usr/bin/env python3

# This file contains a class and a main program that perform three
# related (though complimentary) formatting operations on Python
# programs.  When called as "pindent -c", it takes a valid Python
# program as input and outputs a version augmented with block-closing
# comments.  When called as "pindent -d", it assumes its input is a
# Python program with block-closing comments and outputs a commentless
# version.   When called as "pindent -r" it assumes its input is a
# Python program with block-closing comments but with its indentation
# messed up, and outputs a properly indented version.

# A "block-closing comment" is a comment of the form '# end <keyword>'
# where <keyword> is the keyword that opened the block.  If the
# opening keyword is 'def' or 'class', the function or class name may
# be repeated in the block-closing comment as well.  Here is an
# example of a program fully augmented with block-closing comments:

# def foobar(a, b):
#    if a == b:
#        a = a+1
#    elif a < b:
#        b = b-1
#        if b > a: a = a-1
#        # end if
#    else:
#        print 'oops!'
#    # end if
# # end def foobar

# Note that only the last part of an if...elif...else... block needs a
# block-closing comment; the same is true for other compound
# statements (e.g. try...except).  Also note that "short-form" blocks
# like the second 'if' in the example must be closed as well;
# otherwise the 'else' in the example would be ambiguous (remember
# that indentation is not significant when interpreting block-closing
# comments).

# The operations are idempotent (i.e. applied to their own output
# they yield an identical result).  Running first "pindent -c" and
# then "pindent -r" on a valid Python program produces a program that
# is semantically identical to the input (though its indentation may
# be different). Running "pindent -e" on that output produces a
# program that only differs from the original in indentation.

# Other options:
# -s stepsize: set the indentation step size (default 8)
# -t tabsize : set the number of spaces a tab character is worth (default 8)
# -e         : expand TABs into spaces
# file ...   : input file(s) (default standard input)
# The results always go to standard output

# Caveats:
# - comments ending in a backslash will be mistaken for continued lines
# - continuations using backslash are always left unchanged
# - continuations inside parentheses are not extra indented by -r
#   but must be indented for -c to work correctly (this breaks
#   idempotency!)
# - continued lines inside triple-quoted strings are totally garbled

# Secret feature:
# - On input, a block may also be closed with an "end statement" --
#   this is a block-closing comment without the '#' sign.

# Possible improvements:
# - check syntax based on transitions in 'next' table
# - better error reporting
# - better error recovery
# - check identifier after class/def

# The following wishes need a more complete tokenization of the source:
# - Don't get fooled by comments ending in backslash
# - reindent continuation lines indicated by backslash
# - handle continuation lines inside parentheses/braces/brackets
# - handle triple quoted strings spanning lines
# - realign comments
# - optionally do much more thorough reformatting, a la C indent

# Defaults
STEPSIZE = 8
TABSIZE = 8
EXPANDTABS = False

import io
import re
import sys

next = {}
next['if'] = next['elif'] = 'elif', 'else', 'end'
next['while'] = next['for'] = 'else', 'end'
next['try'] = 'except', 'finally'
next['except'] = 'except', 'else', 'finally', 'end'
next['else'] = next['finally'] = next['with'] = \
    next['def'] = next['class'] = 'end'
next['end'] = ()
start = 'if', 'while', 'for', 'try', 'with', 'def', 'class'

class PythonIndenter:

    def __init__(self, fpi = sys.stdin, fpo = sys.stdout,
                 indentsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
        self.fpi = fpi
        self.fpo = fpo
        self.indentsize = indentsize
        self.tabsize = tabsize
        self.lineno = 0
        self.expandtabs = expandtabs
        self._write = fpo.write
        self.kwprog = re.compile(
                r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'
                r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'
                r'[^\w]')
        self.endprog = re.compile(
                r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'
                r'(\s+(?P<id>[a-zA-Z_]\w*))?'
                r'[^\w]')
        self.wsprog = re.compile(r'^[ \t]*')
    # end def __init__

    def write(self, line):
        if self.expandtabs:
            self._write(line.expandtabs(self.tabsize))
        else:
            self._write(line)
        # end if
    # end def write

    def readline(self):
        line = self.fpi.readline()
        if line: self.lineno += 1
        # end if
        return line
    # end def readline

    def error(self, fmt, *args):
        if args: fmt = fmt % args
        # end if
        sys.stderr.write('Error at line %d: %s\n' % (self.lineno, fmt))
        self.write('### %s ###\n' % fmt)
    # end def error

    def getline(self):
        line = self.readline()
        while line[-2:] == '\\\n':
            line2 = self.readline()
            if not line2: break
            # end if
            line += line2
        # end while
        return line
    # end def getline

    def putline(self, line, indent):
        tabs, spaces = divmod(indent*self.indentsize, self.tabsize)
        i = self.wsprog.match(line).end()
        line = line[i:]
        if line[:1] not in ('\n', '\r', ''):
            line = '\t'*tabs + ' '*spaces + line
        # end if
        self.write(line)
    # end def putline

    def reformat(self):
        stack = []
        while True:
            line = self.getline()
            if not line: break      # EOF
            # end if
            m = self.endprog.match(line)
            if m:
                kw = 'end'
                kw2 = m.group('kw')
                if not stack:
                    self.error('unexpected end')
                elif stack.pop()[0] != kw2:
                    self.error('unmatched end')
                # end if
                self.putline(line, len(stack))
                continue
            # end if
            m = self.kwprog.match(line)
            if m:
                kw = m.group('kw')
                if kw in start:
                    self.putline(line, len(stack))
                    stack.append((kw, kw))
                    continue
                # end if
                if kw in next and stack:
                    self.putline(line, len(stack)-1)
                    kwa, kwb = stack[-1]
                    stack[-1] = kwa, kw
                    continue
                # end if
            # end if
            self.putline(line, len(stack))
        # end while
        if stack:
            self.error('unterminated keywords')
            for kwa, kwb in stack:
                self.write('\t%s\n' % kwa)
            # end for
        # end if
    # end def reformat

    def delete(self):
        begin_counter = 0
        end_counter = 0
        while True:
            line = self.getline()
            if not line: break      # EOF
            # end if
            m = self.endprog.match(line)
            if m:
                end_counter += 1
                continue
            # end if
            m = self.kwprog.match(line)
            if m:
                kw = m.group('kw')
                if kw in start:
                    begin_counter += 1
                # end if
            # end if
            self.write(line)
        # end while
        if begin_counter - end_counter < 0:
            sys.stderr.write('Warning: input contained more end tags than expected\n')
        elif begin_counter - end_counter > 0:
            sys.stderr.write('Warning: input contained less end tags than expected\n')
        # end if
    # end def delete

    def complete(self):
        stack = []
        todo = []
        currentws = thisid = firstkw = lastkw = topid = ''
        while True:
            line = self.getline()
            i = self.wsprog.match(line).end()
            m = self.endprog.match(line)
            if m:
                thiskw = 'end'
                endkw = m.group('kw')
                thisid = m.group('id')
            else:
                m = self.kwprog.match(line)
                if m:
                    thiskw = m.group('kw')
                    if thiskw not in next:
                        thiskw = ''
                    # end if
                    if thiskw in ('def', 'class'):
                        thisid = m.group('id')
                    else:
                        thisid = ''
                    # end if
                elif line[i:i+1] in ('\n', '#'):
                    todo.append(line)
                    continue
                else:
                    thiskw = ''
                # end if
            # end if
            indentws = line[:i]
            indent = len(indentws.expandtabs(self.tabsize))
            current = len(currentws.expandtabs(self.tabsize))
            while indent < current:
                if firstkw:
                    if topid:
                        s = '# end %s %s\n' % (
                                firstkw, topid)
                    else:
                        s = '# end %s\n' % firstkw
                    # end if
                    self.write(currentws + s)
                    firstkw = lastkw = ''
                # end if
                currentws, firstkw, lastkw, topid = stack.pop()
                current = len(currentws.expandtabs(self.tabsize))
            # end while
            if indent == current and firstkw:
                if thiskw == 'end':
                    if endkw != firstkw:
                        self.error('mismatched end')
                    # end if
                    firstkw = lastkw = ''
                elif not thiskw or thiskw in start:
                    if topid:
                        s = '# end %s %s\n' % (
                                firstkw, topid)
                    else:
                        s = '# end %s\n' % firstkw
                    # end if
                    self.write(currentws + s)
                    firstkw = lastkw = topid = ''
                # end if
            # end if
            if indent > current:
                stack.append((currentws, firstkw, lastkw, topid))
                if thiskw and thiskw not in start:
                    # error
                    thiskw = ''
                # end if
                currentws, firstkw, lastkw, topid = \
                          indentws, thiskw, thiskw, thisid
            # end if
            if thiskw:
                if thiskw in start:
                    firstkw = lastkw = thiskw
                    topid = thisid
                else:
                    lastkw = thiskw
                # end if
            # end if
            for l in todo: self.write(l)
            # end for
            todo = []
            if not line: break
            # end if
            self.write(line)
        # end while
    # end def complete
# end class PythonIndenter

# Simplified user interface
# - xxx_filter(input, output): read and write file objects
# - xxx_string(s): take and return string object
# - xxx_file(filename): process file in place, return true iff changed

def complete_filter(input = sys.stdin, output = sys.stdout,
                    stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.complete()
# end def complete_filter

def delete_filter(input= sys.stdin, output = sys.stdout,
                        stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.delete()
# end def delete_filter

def reformat_filter(input = sys.stdin, output = sys.stdout,
                    stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.reformat()
# end def reformat_filter

def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    input = io.StringIO(source)
    output = io.StringIO()
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.complete()
    return output.getvalue()
# end def complete_string

def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    input = io.StringIO(source)
    output = io.StringIO()
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.delete()
    return output.getvalue()
# end def delete_string

def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    input = io.StringIO(source)
    output = io.StringIO()
    pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs)
    pi.reformat()
    return output.getvalue()
# end def reformat_string

def make_backup(filename):
    import os, os.path
    backup = filename + '~'
    if os.path.lexists(backup):
        try:
            os.remove(backup)
        except OSError:
            print("Can't remove backup %r" % (backup,), file=sys.stderr)
        # end try
    # end if
    try:
        os.rename(filename, backup)
    except OSError:
        print("Can't rename %r to %r" % (filename, backup), file=sys.stderr)
    # end try
# end def make_backup

def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    with open(filename, 'r') as f:
        source = f.read()
    # end with
    result = complete_string(source, stepsize, tabsize, expandtabs)
    if source == result: return 0
    # end if
    make_backup(filename)
    with open(filename, 'w') as f:
        f.write(result)
    # end with
    return 1
# end def complete_file

def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    with open(filename, 'r') as f:
        source = f.read()
    # end with
    result = delete_string(source, stepsize, tabsize, expandtabs)
    if source == result: return 0
    # end if
    make_backup(filename)
    with open(filename, 'w') as f:
        f.write(result)
    # end with
    return 1
# end def delete_file

def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
    with open(filename, 'r') as f:
        source = f.read()
    # end with
    result = reformat_string(source, stepsize, tabsize, expandtabs)
    if source == result: return 0
    # end if
    make_backup(filename)
    with open(filename, 'w') as f:
        f.write(result)
    # end with
    return 1
# end def reformat_file

# Test program when called as a script

usage = """
usage: pindent (-c|-d|-r) [-s stepsize] [-t tabsize] [-e] [file] ...
-c         : complete a correctly indented program (add #end directives)
-d         : delete #end directives
-r         : reformat a completed program (use #end directives)
-s stepsize: indentation step (default %(STEPSIZE)d)
-t tabsize : the worth in spaces of a tab (default %(TABSIZE)d)
-e         : expand TABs into spaces (default OFF)
[file] ... : files are changed in place, with backups in file~
If no files are specified or a single - is given,
the program acts as a filter (reads stdin, writes stdout).
""" % vars()

def error_both(op1, op2):
    sys.stderr.write('Error: You can not specify both '+op1+' and -'+op2[0]+' at the same time\n')
    sys.stderr.write(usage)
    sys.exit(2)
# end def error_both

def test():
    import getopt
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'cdrs:t:e')
    except getopt.error as msg:
        sys.stderr.write('Error: %s\n' % msg)
        sys.stderr.write(usage)
        sys.exit(2)
    # end try
    action = None
    stepsize = STEPSIZE
    tabsize = TABSIZE
    expandtabs = EXPANDTABS
    for o, a in opts:
        if o == '-c':
            if action: error_both(o, action)
            # end if
            action = 'complete'
        elif o == '-d':
            if action: error_both(o, action)
            # end if
            action = 'delete'
        elif o == '-r':
            if action: error_both(o, action)
            # end if
            action = 'reformat'
        elif o == '-s':
            stepsize = int(a)
        elif o == '-t':
            tabsize = int(a)
        elif o == '-e':
            expandtabs = True
        # end if
    # end for
    if not action:
        sys.stderr.write(
                'You must specify -c(omplete), -d(elete) or -r(eformat)\n')
        sys.stderr.write(usage)
        sys.exit(2)
    # end if
    if not args or args == ['-']:
        action = eval(action + '_filter')
        action(sys.stdin, sys.stdout, stepsize, tabsize, expandtabs)
    else:
        action = eval(action + '_file')
        for filename in args:
            action(filename, stepsize, tabsize, expandtabs)
        # end for
    # end if
# end def test

if __name__ == '__main__':
    test()
# end if