1. Ronald Oussoren
  2. pyobjc

Commits

Ronald Oussoren  committed 348aeec

- Modified BlockComment scanner because it ran into
limitation of the regexp engine of the SystemConfiguration
headers
- Teach the tokenizer how to parse function argument
lists.
- Use this to generate wrappers for functions (using
loadBundleFunctions)

  • Participants
  • Parent commits 2a60125
  • Branches pyobjc-ancient

Comments (0)

Files changed (2)

File sandbox/parsing/scanframework.py

View file
+# TODO
+# - documentation
+# - scan other frameworks for type definitions
+#   (fall back to compiling test programs if all else fails)
+# - generate wrappers for 'static inline' functions
+# - parse protocols and categories -> generate informal_protocol definitions
+# - deal with embedded frameworks
+# - add hinting facility: which definitions should not be wrapped, hints
+#   about input/output arguments, ... . 
+# - compare results of scanframeworks with those of the 'old' generator
+#   scripts
+# - test, test and even more testing
+# - integrate into build process
 import os
 try:
     set
             pdb.Pdb().set_trace()
         return self._scanner.iterscan(file(fn).read(), dead=deadraise)
 
-class EnumCollector(object):
-    def __init__(self):
-        self.seen = []
-        self.code = []
-
-    def add(self, enum):
-        value = -1
-        for token in enum.matches():
-            if isinstance(token, EnumValueMember):
-                value = token['value']
+def contains_instances_of(iterator, types):
+    for i in iterator:
+        if isinstance(i, types):
+            return True
+    return False
 
 def do_enum(enum):
     if isinstance(enum, NamedEnum):
                 try:
                     etp = types[tp]
                 except KeyError:
-                    print "Ignore hard struct", token
                     return
 
                 elems.extend([types[tp]]*len(nm))
         externalname, externalname, externalname, fieldnames))
 
 def normalize_whitespace(value):
-    value = value.replace('\t', ' ')
+    value = value.strip().replace('\t', ' ')
     while '  ' in value:
         value = value.replace('  ', ' ')
     return value
 
+def do_function(token, types, functions):
+    returns = normalize_whitespace(token['returns'])
+    name = token['name'].strip()
+
+    if isinstance(token, ExportVoidFunction):
+        if returns in types:
+            functions.append("\n        (%r, %s, '%s %s(void)'),"%(
+                    name, 
+                    types[returns],
+                    returns.replace(' *', '*'), name))
+        else:
+            print "Ignore function %s (retval) %r"%(name, returns)
+        return
+
+    if returns not in types and returns != 'void':
+        print "Ignore function %s (retval) %r"%(name, returns)
+        return
+
+    arglist = []
+    if returns == 'void':
+        argtypes = ['_objc._C_VOID',]
+    else:
+        argtypes = [types[returns],]
+
+    for arg in token.matches()[:-1]:
+        tp = normalize_whitespace(arg['type'])
+        nm = arg['name']
+        if nm is None:
+            arglist.append('%s'%(tp.replace(' *', '*'),))
+        else:
+            nm = nm.strip()
+            arglist.append('%s %s'%(tp.replace(' *', '*'), nm))
+        if tp not in types:
+            print 'Ignore function %s (arg) nm: %r, tp: %r'%(name, nm, tp)
+            return
+        argtypes.append(types[tp])
+
+    encoded = "''.join(("+ ', '.join(argtypes) + ",))"
+    functions.append("\n        (%r, %s, '%s %s(%s)'),"%(
+        name, encoded, returns.replace(' *', '*'), name, ', '.join(arglist)))
 
 def makeInit(framework, out):
     framework_name = filter(None, os.path.split(framework))[0]
         'NSString *': 'objc._C_ID',
         'NSString*': 'objc._C_ID',
         'CFStringRef': 'objc._C_ID',
+        'SEL': 'objc._C_SEL',
+        'BOOL': 'objc._C_BOOL',
+        'Class': 'objc._C_CLS',
 
         'char': 'objc._C_CHR',
         'unsigned char': 'objc._C_UCHR',
 
         'int': 'objc._C_INT',
         'signed int': 'objc._C_INT',
+        'signed': 'objc._C_INT',
         'unsigned int': 'objc._C_UINT',
+        'unsigned': 'objc._C_UINT',
 
         'short': 'objc._C_SHT',
         'signed short': 'objc._C_SHT',
         BlockComment,
         SingleLineComment,
         CPPCrap,
-        UninterestingStruct, # XXX: Not sure about this
+
+        # Stuff below here might be interesting
+        UninterestingStruct, 
+        MacroDefine,
     ])
     dependencies = set()
     globthings = []
     simple_defines=[]
     imports = []
     structs = []
+    functions = []
     for token in ifilter(None, f.scanframework(framework)):
         if isinstance(token, GlobalThing):
+
+            # Skip private variables
+            if token['name'][0] == '_': continue
+
             tp = normalize_whitespace(token['type'])
             if tp not in types:
                 print 'ignore', token
                 continue
 
-            globthings.append('        (%r, %s),\n' % (unicode(token['name']), types[tp]))
+            globthings.append('\n        (%r, %s),' % (unicode(token['name']), types[tp]))
         elif isinstance(token, (Enum, NamedEnum)):
+            if isinstance(token, NamedEnum):
+                nm = token.matches()[-1]['name'].strip()
+                types[nm] = types['int']
+
             for s in do_enum(token):
                 enums.append(s)
         elif isinstance(token, Dependency):
 
         
         elif isinstance(token, NamedStruct):
-            # Dunno how to wrap C unions
+
+            # Skip structs containing function pointers, those cannot be
+            # wrapped automaticly (yet?)
+            if contains_instances_of(token.matches(), FunctionStructMember): continue
+
             do_struct(token, types, structs)
 
         elif isinstance(token, UninterestingTypedef):
 
 
         elif isinstance(token, Protocol):
-            # TODO: generate informal-protocol definition
+            # TODO: generate informal-protocol definition. 
+            # Need to enhance the tokenizer for that
             pass
 
         elif isinstance(token, (Interface, ForwardClassReference)):
             types[token['name'] + '*'] = types['id']
             types[token['name'] + ' *'] = types['id']
 
-        elif isinstance(token, MacroDefine):
-            # What can we do for these?
-            pass
-
-        elif isinstance(token, ExportFunction):
-            # XXX
-            pass
+        elif isinstance(token, (ExportFunction, ExportVoidFunction)):
+            do_function(token, types, functions)
 
         elif isinstance(token, StaticInlineFunction):
             # TODO: emit wrapper inside a C file.
     print >>out, '\n# struct definitions'
     out.writelines(structs)
     bundle_variables = ''.join(globthings)
+    bundle_functions = ''.join(functions)
     print >>out, """
 
 def _initialize():
     p = objc.pathForFramework(%(framework_path)r)
     objc.loadBundle(%(framework_name)r, globals(), bundle_path=p)
     b = NSBundle.bundleWithPath_(p)
-    objc.loadBundleVariables(b, globals(), [
-%(bundle_variables)s
+    objc.loadBundleVariables(b, globals(), [%(bundle_variables)s
+    ])
+
+    objc.loadBundleFunctions(b, globals(), [%(bundle_functions)s
     ])
 
     # XXX - hack to fix-up NSError** args

File sandbox/parsing/tokenize_header.py

View file
     ''')
 
 class BlockComment(Token):
+    # Note: we use non-greedy matching instead of spelling out that the
+    # contents of a comment cannot contain '*/' because we ran into 
+    # limitation of the regexp-engine when parsing files with very big
+    # comments, such as in SystemConfiguration.framework
     pattern = pattern(r'''
     \/\*
-    (?P<comment>([^*]|\*(?!/))*)
+    (?P<comment>.*?)
     \*\/
     ''')
     example = example('''
     /*************\t\tThis is an annoying one\t\t**********/
     /* this is a block comment */
     ''')
+    # The really nasty one is this:
+    # /* foo \*/ bar */
+    # Don't worry about it until we see this in real headers
 
 class SingleLineComment(Token):
     pattern = pattern(r'//(?P<comment>[^\n]*)(\n|$)')
         } FooBar AVAILABLE_MAC_OS_X_10_3_AND_LATER;
     ''')
 
+
+class FunctionStructMember(Token):
+    pattern = pattern(r'''
+    \s*(?P<returns>%(IDENTIFIER)s%(INDIRECTION)s*)
+    \s*\(\s*\*\s*%(IDENTIFIER)s\s*\)
+    \(
+        (?P<args>\s*[^)]*)
+    \);
+    ''')
+    example = example(r'''
+        void* (*pfunc)(int, float);
+    ''')
+
 class NestedStructMember(Token):
     pattern = pattern(r'''
-    \s*union\s+(?P<structname>%(IDENTIFIER)s)?
+    \s*(union|struct)\s+(?P<structname>%(IDENTIFIER)s)?
     \s*{\s*
         (?P<body>[^}]*)
     }\s+(?P<name>%(IDENTIFIER)s)\s*;
         int my;
         int thy;
     } foo;
+    struct {
+        int foo;
+        int bar;
+    } bar;
     ''')
 
 class StructMember(Token):
     lexicon = [
         CompilerDirective,
         NestedStructMember,
+        FunctionStructMember,
         StructMember,
     ]
     example = example(r'''
         } foo;
         int bar;
     } FooBarStruct;
+    typedef struct {
+        int (*pfunc)(void);
+    } FunctionStruct;
     ''')
  
 class Enum(ScanningToken):
     ''')
 
 class FunctionEnd(Token):
-    # XXX - UNUSED
     pattern = pattern(r'''
     \)
+    (\s*(?P<available>%(AVAILABLE)s))?
     \s*;
     ''')
     example = example(r'''
     );
     )  ;
+    ) AVAILABLE_SOMEHOW;
     ''')
 
 class FunctionParameter(Token):
-    # XXX - UNUSED
     pattern = pattern(r'''
-    (%(IDENTIFIER)s\s*)+
-    \s*%(INDIRECTION)s
-    \s*%(IDENTIFIER)s?
-    \s*,?\s*
+    \s*(?P<type>
+        (%(KEYWORD)s\s*)*
+        %(IDENTIFIER)s\s*
+        (%(INDIRECTION)s|\s+%(KEYWORD)s)*
+        \s*
+        (?<=\*|\s)
+    )\s*
+    (?P<name>%(IDENTIFIER)s\s*)?\s*,?\s*
     ''')
     example = example(r'''
+    NSString* foo
     NSString *foo, NSString *bar
     ''')
 
-#class ExportFunction(ScanningToken):
-#    pattern = pattern(r'''
-#    %(EXPORT)s
-#    \s*(?P<returns>%(IDENTIFIER)s%(INDIRECTION)s*)
-#    \s*%(IDENTIFIER)s
-#    \s*\(
-#    ''')
-#    endtoken = FunctionEnd
-#    lexicon = [
-#        InsignificantWhitespace,
-#        FunctionParameter,
-#    ]
-#    example = example(r'''
-#    FOUNDATION_EXPORT SomeResult **SomeName(const Foo *, const Foo *Bar);
-#    FOUNDATION_EXPORT SomeResult SomeName(int,float);
-#    ''')
-
-class ExportFunction(Token):
-    # XXX handle comments? need its own internal parser?
+class ExportVoidFunction (Token):
     pattern = pattern(r'''
     %(EXPORT)s?
     \s*(?P<returns>
-        (%(KEYWORD)s\s*)*
+        (%(KEYWORD)s(\s+%(KEYWORD)s)*)?
         %(IDENTIFIER)s
         (%(INDIRECTION)s|\s+%(KEYWORD)s)*
     )
     (\s*(?P<protocols>%(PROTOCOLS)s))?
+    \s*(\s%(IDENTIFIER)s\s)?
+    \s*(\/\*.*?\*\/)?
+    \s*(?P<name>%(IDENTIFIER)s)
+    \s*\(\s*void\s*\)
+    (\s*(?P<available>%(AVAILABLE)s))?;''')
+
+
+class ExportFunction(ScanningToken):
+    pattern = pattern(r'''
+    %(EXPORT)s?
+    \s*(?P<returns>
+        (%(KEYWORD)s(\s+%(KEYWORD)s)*)?
+        %(IDENTIFIER)s
+        (%(INDIRECTION)s|\s+%(KEYWORD)s)*
+    )
+    (\s*(?P<protocols>%(PROTOCOLS)s))?
+    \s*(\s%(IDENTIFIER)s\s)?
+    \s*(\/\*.*?\*\/)?
     \s*(?P<name>%(IDENTIFIER)s)
     \s*\(
-        (?P<args>\s*[^)]*)
-    \s*\)
-    (\s*(?P<available>%(AVAILABLE)s))?
-    \s*;
     ''')
+    #(\s*\/\*.*?\*\/\s*)?
+    endtoken = FunctionEnd
+    lexicon = [
+        InsignificantWhitespace,
+        FunctionParameter,
+    ]
     example = example(r'''
+    extern NSString *ABLocalizedPropertyOrLabel(NSString *propertyOrLabel);
     APPKIT_EXTERN NSString *NSSomething(NSString *arg, NSString *arg)
         AVAILABLE_SOMEWHERE;
     FOUNDATION_EXPORT void *NSSomething(unsigned long arg, unsigned long arg) AVAILABLE_SOMEWHERE;
     FOUNDATION_EXPORT SomeResult <NSObject> SomeName(const Foo *, const Foo *Bar);
     FOUNDATION_EXPORT SomeResult **SomeName(const Foo *, const Foo *Bar);
     FOUNDATION_EXPORT SomeResult SomeName(int,float);
+    NPError NP_LOADSS   NP_New(void);
+    CFArrayRef /* of SCFoo's */ SCNetworkInterfaceCopyAll    (void) AVALABLE_FOO;
     ''')
 
+#!# class ExportFunction(Token):
+#!#     # XXX handle comments? need its own internal parser?
+#!#     pattern = pattern(r'''
+#!#     %(EXPORT)s?
+#!#     \s*(?P<returns>
+#!#         (%(KEYWORD)s\s*)*
+#!#         %(IDENTIFIER)s
+#!#         (%(INDIRECTION)s|\s+%(KEYWORD)s)*
+#!#     )
+#!#     (\s*(?P<protocols>%(PROTOCOLS)s))?
+#!#     \s*(?P<name>%(IDENTIFIER)s)
+#!#     \s*\(
+#!#         (?P<args>\s*[^)]*)
+#!#     \s*\)
+#!#     (\s*(?P<available>%(AVAILABLE)s))?
+#!#     \s*;
+#!#     ''')
+#!#     example = example(r'''
+#!#     APPKIT_EXTERN NSString *NSSomething(NSString *arg, NSString *arg)
+#!#         AVAILABLE_SOMEWHERE;
+#!#     FOUNDATION_EXPORT void *NSSomething(unsigned long arg, unsigned long arg) AVAILABLE_SOMEWHERE;
+#!#     FOUNDATION_EXPORT SomeResult <NSObject> SomeName(const Foo *, const Foo *Bar);
+#!#     FOUNDATION_EXPORT SomeResult **SomeName(const Foo *, const Foo *Bar);
+#!#     FOUNDATION_EXPORT SomeResult SomeName(int,float);
+#!#     ''')
+
 class StaticInlineFunction(Token):
     # XXX need to figure out how to find a close brace
     #     will probably need something stateful I guess
     Enum,
     NamedStruct,
     Struct,
-    ExportFunction,
-    StaticInlineFunction,
-    UninterestingTypedef,
-    UninterestingStruct,
     MacroDefine,
     CPPDecls,
     CPPCrap,
     CompilerDirective,
+    StaticInlineFunction,
+    ExportVoidFunction,
+    ExportFunction,
+    UninterestingTypedef,
+    UninterestingStruct,
 ]
 
 if __name__ == '__main__':
     import re
     import sys
     frameworks = """
+    SystemConfiguration
     AddressBook
     AppKit
     ExceptionHandling
     SecurityInterface
     System
     SystemConfiguration
-    #Security
-    #Carbon
-    #CoreServices
+    Security
+    Carbon
+    CoreServices
     """.split()
     files = sys.argv[1:]
     if not files: