1. anatoly techtonik
  2. absub

Commits

anatoly techtonik  committed 6b0c760

Implement Popen.communicate() using non-blocking layer (WIP)
This doesn't work well currently due to a delay between process
exit and its final output appearance in Queue.

The fix to avoid race condition in communicate is to wait until
stdout/stderr pipes are closed (which meand threads are terminated)
and only then read out the queues.

  • Participants
  • Parent commits df035f5
  • Branches default

Comments (0)

Files changed (3)

File README

View file
 ------------
  What's New 
 ------------
+Version 0.5
+* (techtonik) Implement standard communicate() using non-blocking layer.
+
 Version 0.4
 * (techtonik) Non-blocking communicate() is renamed to asyncomm() to allow
   making AsyncPopen() class a drop-in Popen() replacement that doesn't break

File async_subprocess.py

View file
       IOError: close() called during concurrent operation on the same file object.
 '''
 
-__version__ = '0.4'
+__version__ = '0.5dev'
 
 from subprocess import PIPE, Popen
 from threading  import Thread, Lock
 
 from collections import deque
 
+# --- debugging helper ---
+
+def echo(msg):
+  import sys
+  sys.stderr.write(msg)
+  sys.stderr.flush()
+
 # --- functions that run in separate threads ---
 def threadedOutputQueue(pipe, queue, lock):
     '''
             self.stderr_thread.start()
         if self._stdin:
             self.stdin = StdinQueue(self.stdin)
+
+    def communicate(self, input=None):
+        if self._stdin:
+            if input:
+                self.stdin.write(input)
+            # close stdin pipe for the children process. If
+            # it is blocked waiting for input, this will make
+            # it receive EOF to unblock.
+            self.stdin.close()
+
+        stdoutdata = b'' if self._stdout else None
+        stderrdata = b'' if self._stderr else None
+        while self.poll() == None:
+            # [ ] this causes 100% CPU load
+            (out, err) = self.asyncomm()
+            if out:
+                stdoutdata += out
+            if err:
+                stderrdata += err
+
+        return (stdoutdata, stderrdata)
     
     def asyncomm(self, input=None):
         '''

File tests/communicate.py

View file
 
 # Popen returns immediately leaving child process
 # executed in background
-if 1:
+if 0:
     from subprocess import Popen, PIPE
     p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 else:
 
 #print repr(stdout), repr(stderr)
 # split helps avoid line end differences when comparing
+print repr(stdout.split()), repr(stderr)
 assert stdout.split() == ['stdout', 'stuff', "''"]
 assert stderr.split() == ['stderr', 'stuff']
 
 # [ ] p = AsyncPopen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
 #   [ ] stdout lineends on different platforms after communicate
 #     [x] always \r\n on Windows for the script above
+#   [ ] stdout return type on Python 3