Commits

Jim Baker  committed 28a66ba

Fixes bug 1878 by resolving complex diamond inheritance of java.lang.Iterable, java.util.Map.
Enables interoperation of Jython with Clojure types, including systems that build on Clojure
like Storm.

  • Participants
  • Parent commits aa079dc

Comments (0)

Files changed (8)

File .idea/compiler.xml

       <entry name="?*.tld" />
       <entry name="?*.ftl" />
     </wildcardResourcePatterns>
-    <annotationProcessing enabled="false" useClasspath="true" />
+    <annotationProcessing>
+      <profile default="true" name="Default" enabled="false">
+        <processorPath useClasspath="true" />
+      </profile>
+    </annotationProcessing>
   </component>
 </project>
 

File .idea/libraries/extlibs.xml

     <JAVADOC />
     <SOURCES />
   </library>
-</component>
+</component>

File .idea/libraries/extlibs2.xml

     <JAVADOC />
     <SOURCES />
   </library>
-</component>
+</component>

File Lib/test/pounce.py

+from java.io import Serializable
+from java.util.concurrent import Callable
+
+class Cat(Callable, Serializable):
+    def whoami(self):
+        return "Socks"
+    def call(self):
+        print "meow"  # force use of PySystemState
+

File Lib/test/test_java_integration.py

 import copy
+import glob
 import operator
 import os
+import os.path
 import unittest
+import shutil
 import subprocess
 import sys
+import tempfile
 import re
 
 from collections import deque
 from org.python.tests import (BeanImplementation, Child, Child2,
                               CustomizableMapHolder, Listenable, ToUnicode)
 from org.python.tests.mro import (ConfusedOnGetitemAdd, FirstPredefinedGetitem, GetitemAdder)
+from org.python.util import PythonInterpreter
+import org.python.core.Options
+
 from javatests import Issue1833
 from javatests.ProxyTests import NullToString, Person
 
         self.assertEqual(str(nts), '')
         self.assertEqual(unicode(nts), '')
 
+    def test_diamond_inheritance_of_iterable_and_map(self):
+        """Test deeply nested diamond inheritance of Iterable and Map, as see in some Clojure classes"""
+        # http://bugs.jython.org/issue1878
+        from javatests import DiamondIterableMapMRO  # this will raise a TypeError re MRO conflict without the fix
+        # Verify the correct MRO is generated - order is of course *important*;
+        # the following used types are implemented as empty interfaces/abstract classes, but match the inheritance graph
+        # and naming of Clojure/Storm.
+        #
+        # Also instead of directly importing, which would cause annoying bloat in javatests by making lots of little files,
+        # just match using str - this will still be stable/robust.
+        self.assertEqual(
+            str(DiamondIterableMapMRO.__mro__),
+            "(<type 'javatests.DiamondIterableMapMRO'>, <type 'javatests.ILookup'>, <type 'javatests.IPersistentMap'>, <type 'java.lang.Iterable'>, <type 'javatests.Associative'>, <type 'javatests.IPersistentCollection'>, <type 'javatests.Seqable'>, <type 'javatests.Counted'>, <type 'java.util.Map'>, <type 'javatests.AFn'>, <type 'javatests.IFn'>, <type 'java.util.concurrent.Callable'>, <type 'java.lang.Runnable'>, <type 'java.lang.Object'>, <type 'object'>)")
+        # And usable with __iter__ and map functionality
+        m = DiamondIterableMapMRO()
+        m["abc"] = 42
+        m["xyz"] = 47
+        self.assertEqual(set(m), set(["abc", "xyz"]))
+        self.assertEqual(m["abc"], 42)
 
 def roundtrip_serialization(obj):
     """Returns a deep copy of an object, via serializing it
         return self.output.classQueue.popleft()
 
 
+def find_jython_jars():
+    # Uses the same classpath resolution as bin/jython
+    jython_jar_path = os.path.normpath(os.path.join(sys.executable, "../../jython.jar"))
+    jython_jar_dev_path = os.path.normpath(os.path.join(sys.executable, "../../jython-dev.jar"))
+    if os.path.exists(jython_jar_dev_path):
+        jars = [jython_jar_dev_path]
+        jars.extend(glob.glob(os.path.normpath(os.path.join(jython_jar_dev_path, "../javalib/*.jar"))))
+    elif os.path.exists(jython_jar_path):
+        jars = [jython_jar_path]
+    else:
+        raise Exception("Cannot find jython jar")
+    return jars
+
+
 class SerializationTest(unittest.TestCase):
 
     def test_java_serialization(self):
         names = [x for x in dir(__builtin__)]
         self.assertEqual(names, roundtrip_serialization(names))
 
+    def test_proxy_serialization(self):
+        """Proxies can be deserializable in a fresh JVM, including being able to "findPython" to get a PySystemState"""
+        tempdir = tempfile.mkdtemp()
+        old_proxy_debug_dir = org.python.core.Options.proxyDebugDirectory
+        try:
+            # Generate a proxy for Cat class;
+            org.python.core.Options.proxyDebugDirectory = tempdir
+            from pounce import Cat
+            cat = Cat()
+            self.assertEqual(cat.whoami(), "Socks")
+
+            # Create a jar file containing the Cat proxy; could use Java to do this; do it the easy way for now
+            proxies_jar_path = os.path.join(tempdir, "proxies.jar")
+            subprocess.check_call(["jar", "cf", proxies_jar_path, "-C", tempdir, "org/"])
+
+            # Serialize our cat
+            output = ByteArrayOutputStream()
+            serializer = CloneOutput(output)
+            serializer.writeObject(cat)
+            serializer.close()
+            cat_path = os.path.join(tempdir, "serialized-cat")
+            with open(cat_path, "wb") as f:
+                f.write(output.toByteArray())
+
+            # Then in a completely different JVM running
+            # ProxyDeserialization, verify we get "meow" printed to
+            # stdout, which in turn ensures that PySystemState (and
+            # Jython runtime) is initialized for the proxy
+            jars = find_jython_jars()
+            jars.append(proxies_jar_path)
+            classpath = ":".join(jars)
+            cmd = [os.path.join(System.getProperty("java.home"), "bin/java"),
+                   "-classpath", classpath, "ProxyDeserialization", cat_path]
+            self.assertEqual(subprocess.check_output(cmd), "meow\n")
+        finally:
+            org.python.core.Options.proxyDebugDirectory = old_proxy_debug_dir
+            shutil.rmtree(tempdir)
 
 
 class CopyTest(unittest.TestCase):

File src/org/python/core/PyJavaType.java

                 continue;
             }
             for (String method : type.modified) {
-                if (!allModified.add(method)) { // Another type in conflict has this method, fail
+                if (!allModified.add(method)) { // Another type in conflict has this method, possibly fail
                     PyList types = new PyList();
+                    Set<Class> proxySet = Generic.set();
                     for (PyJavaType othertype : conflictedAttributes) {
                         if (othertype.modified != null && othertype.modified.contains(method)) {
                             types.add(othertype);
+                            proxySet.add(othertype.getProxyType());
                         }
                     }
+                    // Need to special case collections that implement both Iterable and Map. Ignore the conflict
+                    // in having duplicate __iter__ added (see getCollectionProxies), while still allowing each
+                    // path on the inheritance hierarchy to get an __iter__. Annoying but necessary logic.
+                    // See http://bugs.jython.org/issue1878
+                    if (method.equals("__iter__") && proxySet.equals(Generic.set(Iterable.class, Map.class))) {
+                        continue;
+                    }
                     throw Py.TypeError(String.format("Supertypes that share a modified attribute "
-                            + " have an MRO conflict[attribute=%s, types=%s]", method, types));
+                            + "have an MRO conflict[attribute=%s, supertypes=%s, type=%s]",
+                            method, types, this.getName()));
                 }
             }
         }
                 }
             };
             collectionProxies.put(Map.class, new PyBuiltinMethod[] {mapLenProxy,
+                    // map IterProxy can conflict with Iterable.class; fix after the fact in handleMroError
                                                                     mapIterProxy,
                                                                     mapContainsProxy,
                                                                     mapGetProxy,

File tests/java/javatests/DiamondIterableMapMRO.java

+// example from Storm and its use of Clojure
+
+package javatests;
+
+import java.io.Serializable;
+import java.util.AbstractMap;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import org.python.util.Generic;
+
+// The following tag interfaces duplicate the interface/abstract class supertypes of
+// Storm's IndifferentAccessMap, including Clojure types
+
+interface Seqable {}
+interface IPersistentCollection extends Seqable {}
+interface ILookup {}
+interface Associative extends IPersistentCollection, ILookup {}
+interface Counted {}
+interface IPersistentMap extends Iterable, Associative, Counted {}
+abstract class AFn implements IFn {}
+interface IFn extends Callable, Runnable {}
+
+public class DiamondIterableMapMRO extends AFn implements ILookup, IPersistentMap, Map {
+    private final Map backing;
+
+    public DiamondIterableMapMRO() {
+        backing = Generic.map();
+    }
+
+    public Object call() { return null; }
+
+    public void run() {}
+
+    public Iterator iterator() {
+        return backing.keySet().iterator();
+    }
+
+    public Set entrySet() {
+        return backing.entrySet();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return backing.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return backing.hashCode();
+    }
+
+    @Override
+    public int size() {
+        return backing.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return backing.isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return backing.containsKey(key);
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        return backing.containsValue(value);
+    }
+
+    @Override
+    public Object get(Object key) {
+        return backing.get(key);
+    }
+
+    public Object put(Object key, Object value) {
+        return backing.put(key, value);
+    }
+
+    @Override
+    public Object remove(Object key) {
+        return backing.remove(key);
+    }
+
+    public void putAll(Map m) {
+        backing.putAll(m);
+    }
+
+    @Override
+    public void clear() {
+        backing.clear();
+    }
+
+    @Override
+    public Set keySet() {
+        return backing.keySet();
+    }
+
+    @Override
+    public Collection values() {
+        return backing.values();
+    }
+}
+

File tests/java/javatests/ProxyDeserialization.java

+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.Callable;
+import java.io.FileInputStream;
+import java.io.ObjectInputStream;
+
+
+public class ProxyDeserialization {
+
+    public static void main(String[] args) {
+        try {
+            ObjectInputStream ois = new ObjectInputStream(new FileInputStream(args[0]));
+            Callable cat = (Callable) ois.readObject(); // abuse! obviously not concurrent
+            cat.call();
+        }
+        catch(Exception e) {
+            System.err.println(e);
+        }
+    }
+}