Commits

Matt Joiner committed 8f7fdc2

Get the http/urllib packages to work

Comments (0)

Files changed (10)

examples/http_get.py

 import sys
 import urllib.parse
 import green380
-
-def http_get(url, timeout=None):
-    @green380.spawn
-    def timeout_routine():
-        yield from green380.sleep(timeout)
-        yield from ch.send()
-    split = urllib.parse.urlsplit(url)
-    sock = green380.Socket()
-    addr = split.netloc.split(':')
-    if len(addr) == 1:
-        addr.append(80)
-    else:
-        addr[-1] = int(addr[-1])
-    addr = tuple(addr)
-    @green380.spawn
-    def connect_routine():
-        yield from sock.connect(addr)
-        yield from ch.send()
-    ch = green380.Channel()
-    value, sender = yield from ch.recv_from()
-    assert value is None, value
-    if sender is timeout_routine:
-        raise Exception('timed out')
-    elif sender is not connect_routine:
-        assert False, sender
-    @green380.spawn
-    def send_header():
-        return sock.sendall(
-            'GET {split[2]}?{split[3]} HTTP/1.1\r\nHost: {addr[0]}\r\nConnection: close\r\n\r\n'.format(
-                split=split, addr=addr).encode())
-    response_header_lines = (yield from sock.recv_until(b'\r\n\r\n')).split(b'\r\n')
-    headers = collections.defaultdict(list)
-    for l in map(bytes.decode, response_header_lines[1:]):
-        if not l:
-            continue
-        h, *v = l.split(':', 1)
-        headers[h.lower().strip()].append(v[0])
-    return response_header_lines[0].decode(), headers, sock
+from green380.http import get as http_get
 
 def main():
     for url in sys.argv[1:]:
-        status, headers, sock = yield from http_get(url, 0.1)
+        status, headers, sock = yield from http_get(url, 1)
         print(status, file=sys.stderr)
         print(headers, file=sys.stderr)
         while True:

examples/prime_sieve.py

 #!/usr/bin/env python3.3
 
 from green380 import *
+import sys
 
 def generate(ch, n):
     for i in range(2, n+1):
         spawn(filter, p, ch, ch1)
         ch = ch1
 
-spawn(sieve, 10000)
+spawn(sieve, int(sys.argv[1]) if len(sys.argv) > 1 else 10000)
 run()

green380/_core.py

 from .time import sleep
 
 logger = logging.getLogger('green380')
-logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.INFO)
 
 
 def alt(*gens):
 class Timeout(Exception):
 
     def __init__(self, timeout):
-        spawn(self._timeout_routine, timeout)
+        self._timeout_routine = spawn(self._timeout_routine, timeout)
         self._channels = set()
         self._expired = False
 
             raise self
         ch = Channel()
         self._channels.add(ch)
-        @schedule
+        @spawn
         def routine():
             result = yield from gen
-            ch.send(result)
+            yield from ch.send(result)
         value, sender = yield from ch.recv_from()
         self._channels.remove(ch)
         if sender is routine:
 
     put = send
 
+    def send_nowait(self
+
 
 def fiber_bottom_location(gen):
     import pdb
             except:
                 logging.exception('Error handling event for fiber %r: %s', fiber, fiber_bottom_location(fiber))
                 raise
-        except:
+        except ():
             logger.critical('Error handling event %r for fiber %r', event, fiber, exc_info=True)
             raise
 
         mask |= select.EPOLLOUT if self._writers[fd] else 0
         if fd in self._registered:
             if mask:
-                self._poll_obj.modify(fd, mask)
+                try:
+                    self._poll_obj.modify(fd, mask)
+                except FileNotFoundError:
+                    logger.warning('File descriptor %r was dropped from polling object', fd)
+                    # TODO investigate if closure should be propagated to waiting fibers
+                else:
+                    return
             else:
                 self._poll_obj.unregister(fd)
                 self._registered.remove(fd)
-        else:
-            if mask:
-                self._poll_obj.register(fd, mask)
-                self._registered.add(fd)
+                return
+        if mask:
+            self._poll_obj.register(fd, mask)
+            self._registered.add(fd)
 
     def _handle_signal(self, signum, frame):
         #~ logger.critical('signal handler: %s', signum)

green380/_thread.py

         yield from self.acquire()
         return self
 
+    def _is_owned(self):
+        return self._owner is _core.current
+
     def acquire(self, blocking=True, timeout=-1):
         if not self._locked:
             self._locked = True

green380/http/__init__.py

-# This directory is a Python package.
+import green380
+import urllib.parse
+
+def get(url, timeout=None):
+    timeout = green380.Timeout(timeout)
+    split = urllib.parse.urlsplit(url)
+    sock = green380.Socket()
+    addr = split.netloc.split(':')
+    if len(addr) == 1:
+        addr.append(80)
+    else:
+        addr[-1] = int(addr[-1])
+    addr = tuple(addr)
+    yield from timeout.multiplex(sock.connect(addr))
+    @green380.spawn
+    def send_header():
+        return sock.sendall(
+            'GET {split[2]}?{split[3]} HTTP/1.1\r\nHost: {addr[0]}\r\nConnection: close\r\n\r\n'.format(
+                split=split, addr=addr).encode())
+    response_header_lines = (yield from timeout.multiplex(sock.recv_until(b'\r\n\r\n'))).split(b'\r\n')
+    headers = collections.defaultdict(list)
+    for l in map(bytes.decode, response_header_lines[1:]):
+        if not l:
+            continue
+        h, *v = l.split(':', 1)
+        headers[h.lower().strip()].append(v[0])
+    return response_header_lines[0].decode(), headers, sock
+

green380/http/client.py

         self.will_close = _UNKNOWN      # conn will close at end of response
 
     def _read_status(self):
+        yield 'read', self.fp
         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
         if len(line) > _MAXLINE:
             raise LineTooLong("status line")
         except ValueError:
             raise BadStatusLine(line)
         return version, status, reason
+        yield
 
     def begin(self):
         if self.headers is not None:
 
         # read until we get a non-100 response
         while True:
-            version, status, reason = self._read_status()
+            version, status, reason = yield from self._read_status()
             if status != CONTINUE:
                 break
             # skip the header from the 100 response
             while True:
-                skip = self.fp.readline(_MAXLINE + 1)
+                skip = yield from self.fp.readline(_MAXLINE + 1)
                 if len(skip) > _MAXLINE:
                     raise LineTooLong("header line")
                 skip = skip.strip()
                     break
                 if encode:
                     datablock = datablock.encode("iso-8859-1")
-                self.sock.sendall(datablock)
+                yield from self.sock.sendall(datablock)
 
         try:
             yield from self.sock.sendall(data)
         except TypeError:
             if isinstance(data, collections.Iterable):
                 for d in data:
-                    self.sock.sendall(d)
+                    yield from self.sock.sendall(d)
             else:
                 raise TypeError("data should be a bytes-like object "
                                 "or an iterable, got %r" % type(data))
         if isinstance(message_body, bytes):
             msg += message_body
             message_body = None
-        self.send(msg)
+        yield from self.send(msg)
         if message_body is not None:
             # message_body was not a string (i.e. it is a file), and
             # we must run the risk of Nagle.
-            self.send(message_body)
+            yield from self.send(message_body)
 
     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
         """Send a request to the server.
             self.__state = _CS_REQ_SENT
         else:
             raise CannotSendHeader()
-        self._send_output(message_body)
+        yield from self._send_output(message_body)
 
     def request(self, method, url, body=None, headers={}):
         """Send a complete request to the server."""
-        self._send_request(method, url, body, headers)
+        return self._send_request(method, url, body, headers)
 
     def _set_content_length(self, body):
         # Set the content-length based on the body.
             # RFC 2616 Section 3.7.1 says that text default has a
             # default charset of iso-8859-1.
             body = body.encode('iso-8859-1')
-        self.endheaders(body)
+        yield from self.endheaders(body)
 
     def getresponse(self):
         """Get the response from the server.
         else:
             response = self.response_class(self.sock, method=self._method)
 
-        response.begin()
+        yield from response.begin()
         assert response.will_close != _UNKNOWN
         self.__state = _CS_IDLE
 

green380/queue.py

         Only enqueue the item if a free slot is immediately available.
         Otherwise raise the Full exception.
         '''
-        return self.put(item, block=False)
+        self._put(item)
+        self.unfinished_tasks += 1
+        self.not_empty.notify()
 
     def get_nowait(self):
         '''Remove and return an item from the queue without blocking.

green380/socket.py

         return buf
 
     def recv(self, count):
+        if self._closed:
+            return
         yield 'read', self
         return super().recv(count)
 
         self._checkReadable()
         if self._timeout_occurred:
             raise IOError("cannot read from timed out object")
+        gen = self._sock.recv_into(b)
         while True:
             try:
-                return (yield from self._sock.recv_into(b))
+                next(gen)
+            except StopIteration as exc:
+                return exc.value
             except timeout:
                 self._timeout_occurred = True
                 raise

green380/threading.py

 """Thread module emulating a subset of Java's threading model."""
 
+import green380
 import sys as _sys
 from . import _thread
 
 _PyRLock = _RLock
 
 
-class Condition(_Verbose):
+class Condition:
 
-    def __init__(self, lock=None, verbose=None):
-        _Verbose.__init__(self, verbose)
+    def __init__(self, lock=None):
         if lock is None:
             lock = RLock()
         self._lock = lock
-        # Export the lock's acquire() and release() methods
         self.acquire = lock.acquire
         self.release = lock.release
-        # If the lock defines _release_save() and/or _acquire_restore(),
-        # these override the default implementations (which just call
-        # release() and acquire() on the lock).  Ditto for _is_owned().
-        try:
-            self._release_save = lock._release_save
-        except AttributeError:
-            pass
-        try:
-            self._acquire_restore = lock._acquire_restore
-        except AttributeError:
-            pass
-        try:
-            self._is_owned = lock._is_owned
-        except AttributeError:
-            pass
-        self._waiters = []
+        self._is_owned = lock._is_owned
+        self._waiters = set()
 
     def __enter__(self):
         return self._lock.__enter__()
         return self._lock.__exit__(*args)
 
     def __iter__(self):
-        return iter(self._lock)
+        return self._lock.__iter__()
 
     def __repr__(self):
         return "<Condition(%s, %d)>" % (self._lock, len(self._waiters))
 
-    def _release_save(self):
-        self._lock.release()           # No state to save
+    def wait(self, timeout=None):
+        if not self._is_owned():
+            raise RuntimeError("cannot wait on un-acquired lock")
+        ch = green380.Channel()
+        @green380.spawn
+        def timeout_routine():
+            yield from green380.sleep(timeout)
+            yield from ch.send(False)
+        self._waiters.add(ch)
+        gotit = yield from ch.recv()
+        self._waiters.discard(ch)
+        return gotit
 
-    def _acquire_restore(self, x):
-        yield from self._lock.acquire()           # Ignore saved state
-
-    def _is_owned(self):
-        # Return True if lock is owned by current_thread.
-        # This method is called only if __lock doesn't have _is_owned().
-        if (yield from self._lock.acquire(0)):
-            self._lock.release()
-            return False
-        else:
-            return True
-
-    def wait(self, timeout=None):
-        if not (yield from self._is_owned()):
-            raise RuntimeError("cannot wait on un-acquired lock")
-        waiter = _allocate_lock()
-        yield from waiter.acquire()
-        self._waiters.append(waiter)
-        saved_state = self._release_save()
-        try:    # restore state no matter what (e.g., KeyboardInterrupt)
-            if timeout is None:
-                yield from waiter.acquire()
-                gotit = True
-                if __debug__:
-                    self._note("%s.wait(): got it", self)
-            else:
-                if timeout > 0:
-                    gotit = yield from waiter.acquire(True, timeout)
-                else:
-                    gotit = yield from waiter.acquire(False)
-                if not gotit:
-                    if __debug__:
-                        self._note("%s.wait(%s): timed out", self, timeout)
-                    try:
-                        self._waiters.remove(waiter)
-                    except ValueError:
-                        pass
-                else:
-                    if __debug__:
-                        self._note("%s.wait(%s): got it", self, timeout)
-            return gotit
-        finally:
-            yield from self._acquire_restore(saved_state)
-
-    def wait_for(self, predicate, timeout=None):
-        endtime = None
-        waittime = timeout
-        result = predicate()
-        while not result:
-            if waittime is not None:
-                if endtime is None:
-                    endtime = _time() + waittime
-                else:
-                    waittime = endtime - _time()
-                    if waittime <= 0:
-                        if __debug__:
-                            self._note("%s.wait_for(%r, %r): Timed out.",
-                                       self, predicate, timeout)
-                        break
-            if __debug__:
-                self._note("%s.wait_for(%r, %r): Waiting with timeout=%s.",
-                           self, predicate, timeout, waittime)
-            yield from self.wait(waittime)
-            result = predicate()
-        else:
-            if __debug__:
-                self._note("%s.wait_for(%r, %r): Success.",
-                           self, predicate, timeout)
-        return result
+    #~ def wait_for(self, predicate, timeout=None):
+        #~ endtime = None
+        #~ waittime = timeout
+        #~ result = predicate()
+        #~ while not result:
+            #~ if waittime is not None:
+                #~ if endtime is None:
+                    #~ endtime = _time() + waittime
+                #~ else:
+                    #~ waittime = endtime - _time()
+                    #~ if waittime <= 0:
+                        #~ if __debug__:
+                            #~ self._note("%s.wait_for(%r, %r): Timed out.",
+                                       #~ self, predicate, timeout)
+                        #~ break
+            #~ if __debug__:
+                #~ self._note("%s.wait_for(%r, %r): Waiting with timeout=%s.",
+                           #~ self, predicate, timeout, waittime)
+            #~ yield from self.wait(waittime)
+            #~ result = predicate()
+        #~ else:
+            #~ if __debug__:
+                #~ self._note("%s.wait_for(%r, %r): Success.",
+                           #~ self, predicate, timeout)
+        #~ return result
 
     def notify(self, n=1):
-        if not (yield from self._is_owned()):
-            raise RuntimeError("cannot notify on un-acquired lock")
-        __waiters = self._waiters
-        waiters = __waiters[:n]
-        if not waiters:
-            if __debug__:
-                self._note("%s.notify(): no waiters", self)
-            return
-        self._note("%s.notify(): notifying %d waiter%s", self, n,
-                   n!=1 and "s" or "")
-        for waiter in waiters:
-            waiter.release()
-            try:
-                __waiters.remove(waiter)
-            except ValueError:
-                pass
+        while self._waiters and n > 0:
+            self._waiters.pop().send_nowait(True)
+            n -= 1
 
     def notify_all(self):
         yield from self.notify(len(self._waiters))

green380/urllib/request.py

 import bisect
 import email
 import hashlib
-from green380.http import client as http_client
+from green380 import http
+import green380.http.client
 import io
-import os
+from .. import os
 import posixpath
 import re
-import socket
+from .. import socket
 import sys
-import time
+from .. import time
 import collections
 
-from urllib.error import URLError, HTTPError, ContentTooShortError
-from urllib.parse import (
+from ..urllib.error import URLError, HTTPError, ContentTooShortError
+from ..urllib.parse import (
     urlparse, urlsplit, urljoin, unwrap, quote, unquote,
     splittype, splithost, splitport, splituser, splitpasswd,
     splitattr, splitquery, splitvalue, splittag, to_bytes, urlunparse)
-from urllib.response import addinfourl, addclosehook
+from ..urllib.response import addinfourl, addclosehook
 
 # check for SSL
 try:
     default_classes = [ProxyHandler, UnknownHandler, HTTPHandler,
                        HTTPDefaultErrorHandler, HTTPRedirectHandler,
                        FTPHandler, FileHandler, HTTPErrorProcessor]
-    if hasattr(http_client, "HTTPSConnection"):
+    if hasattr(http.client, "HTTPSConnection"):
         default_classes.append(HTTPSHandler)
     skip = set()
     for klass in default_classes:
             h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
 
         try:
-            h.request(req.get_method(), req.selector, req.data, headers)
+            yield from h.request(req.get_method(), req.selector, req.data, headers)
         except socket.error as err: # timeout error
             h.close()
             raise URLError(err)
 class HTTPHandler(AbstractHTTPHandler):
 
     def http_open(self, req):
-        return self.do_open(http_client.HTTPConnection, req)
+        return self.do_open(http.client.HTTPConnection, req)
 
     http_request = AbstractHTTPHandler.do_request_
 
-if hasattr(http_client, 'HTTPSConnection'):
+if hasattr(http.client, 'HTTPSConnection'):
 
     class HTTPSHandler(AbstractHTTPHandler):
 
             self._check_hostname = check_hostname
 
         def https_open(self, req):
-            return self.do_open(http_client.HTTPSConnection, req,
+            return self.do_open(http.client.HTTPSConnection, req,
                 context=self._context, check_hostname=self._check_hostname)
 
         https_request = AbstractHTTPHandler.do_request_
 
         try:
             response = http_conn.getresponse()
-        except http_client.BadStatusLine:
+        except http.client.BadStatusLine:
             # something went wrong with the HTTP status line
             raise URLError("http protocol error: bad status line")
 
 
     def open_http(self, url, data=None):
         """Use HTTP protocol."""
-        return self._open_generic_http(http_client.HTTPConnection, url, data)
+        return self._open_generic_http(http.client.HTTPConnection, url, data)
 
     def http_error(self, url, fp, errcode, errmsg, headers, data=None):
         """Handle http errors.
 
     if _have_ssl:
         def _https_connection(self, host):
-            return http_client.HTTPSConnection(host,
+            return http.client.HTTPSConnection(host,
                                            key_file=self.key_file,
                                            cert_file=self.cert_file)