import time
import thread
import httplib
import socket
import select
import os
import Queue
from StringIO import StringIO
import random


# ------------------------------------------------------------------------------------
class ERROR:
	CONNECTION_FAILED = -100
	CONNECTION_DROPED = -101
	MAX_QUEUE_SIZE = -110

# ------------------------------------------------------------------------------------
class SETTINGS:
	# After INACTIVE_TIMEOUT after last received data
	# connection will be removed from pool.
	INACTIVE_TIMEOUT = 3 * 60

	# Timeout for connecting to host. Host connections are blocking - do not use too big value.
	CONNECTION_TIMEOUT = 5

	# Number of retries if failure.
	DEFAULT_ATTEMPTS = 3

	# Send error response if too many requests incoming.
	MAX_QUEUE_SIZE = 500

	
#   Implementation
# ====================================================================================

# ------------------------------------------------------------------------------------
class _StrSocket():
	def __init__(self, response_str):
		self._file = StringIO(response_str)
	def makefile(self, *args, **kwargs):
		return self._file

# ------------------------------------------------------------------------------------
class _Connection:

	# ------------------------------------------------------------------------------------
	def __init__(self, host):
		self.host = host
		self.conn = httplib.HTTPConnection(host, timeout=SETTINGS.CONNECTION_TIMEOUT)
		self.conn.connect()
		self.conn.sock.setblocking(0)
		self.clear()
		self.lastResponse = time.time()

	# ------------------------------------------------------------------------------------
	def clear(self):
		self.__buffer = ''
		self.request = None

	# ------------------------------------------------------------------------------------
	def fileno(self):
		return self.conn.sock.fileno()

	# ------------------------------------------------------------------------------------
	def sendRequest(self, httpRequest):
		self.request = httpRequest
		self.conn._HTTPConnection__state = httplib._CS_IDLE
		self.conn.request(httpRequest.method, httpRequest.query, httpRequest.data)

	# ------------------------------------------------------------------------------------
	def processResponse(self):
		resp = self.conn.sock.recv(1024)
		if len(resp) > 0:
			self.lastResponse = time.time()
		self.__buffer += resp
		source = _StrSocket(self.__buffer)
		response = httplib.HTTPResponse(source)
		response.begin()
		status = response.status
		data = response.read()
		return (status, data, self.request)

	# ------------------------------------------------------------------------------------
	def isActive(self):
		return time.time() < self.lastResponse + SETTINGS.INACTIVE_TIMEOUT

# ------------------------------------------------------------------------------------
class _ConnectionManager:
	def __init__(self):
		self.__connections = {} # host -> list of connections

	# ------------------------------------------------------------------------------------
	def getConnection(self, host):
		hostConnections = self.__connections.get(host, None)
		if hostConnections is None:
			return _Connection(host)
		n = random.randint(0, len(hostConnections) - 1)
		conn = hostConnections[n]
		del hostConnections[n]
		if len(hostConnections) == 0:
			del self.__connections[host]
		return conn

	# ------------------------------------------------------------------------------------
	def returnConnection(self, connection):
		connection.clear()
		self.__connections.setdefault(connection.host, []).append(connection)

	# ------------------------------------------------------------------------------------
	def removeOldConnections(self):
		newConnections = {}
		for host, connections in self.__connections.iteritems():
			goodConnections = []
			for conn in connections:
				if conn.isActive():
					goodConnections.append(conn)
			if len(goodConnections) > 0:
				newConnections[host] = goodConnections
		self.__connections = newConnections


# ------------------------------------------------------------------------------------
class _AsyncFetcher:

	# ------------------------------------------------------------------------------------
	def __init__(self):
		self.__epoll = select.epoll()
		self.__requestQueue = Queue.Queue()
		self.__responseQueue = Queue.Queue()
		self.__connections = {} # fileno => http connection
		self.__connectionManager = _ConnectionManager()
		self.pipeToThread = os.pipe()
		self.pipeToMain = os.pipe()
		thread.start_new_thread(self.__workerThread, ())
		thread.start_new_thread(self.__mainThread, ())

	# ------------------------------------------------------------------------------------
	def fetch(self, request):
		if self.__responseQueue.qsize() > SETTINGS.MAX_QUEUE_SIZE or\
				self.__responseQueue.qsize() > SETTINGS.MAX_QUEUE_SIZE:
			request.callback(ERROR.MAX_QUEUE_SIZE, '', request)
			return
		self.__requestQueue.put(request)
		os.write(self.pipeToThread[1], '\n')

	# ------------------------------------------------------------------------------------
	def __mainThread(self):
		pipeIn = self.pipeToMain[0]
		while True:
			os.read(pipeIn, 1)
			while not self.__responseQueue.empty():
				status, data, request = self.__responseQueue.get()
				request.callback(status, data, request)

	# ------------------------------------------------------------------------------------
	def __workerThread(self):
		pipeIn = self.pipeToThread[0]
		self.__epoll.register(pipeIn, select.EPOLLIN)
		last10SecondsTime = time.time()
		while True:
			try:
				events = self.__epoll.poll(1)
				for fileno, event in events:
					if fileno == pipeIn:
						os.read(pipeIn, 1)
						while not self.__requestQueue.empty():
							request = self.__requestQueue.get()
							self.__processRequest(request)
					if fileno in self.__connections:
						self.__processIncoming(fileno)
				if time.time() > last10SecondsTime + 10.0:
					last10SecondsTime = time.time()
					self.__connectionManager.removeOldConnections()
			except Exception as e:
				print e


	# ------------------------------------------------------------------------------------
	def __publishResponse(self, status, data, request):
		self.__responseQueue.put((status, data, request))
		os.write(self.pipeToMain[1], '\n')

	# ------------------------------------------------------------------------------------
	def __processRequest(self, request):
		try:
			conn = self.__connectionManager.getConnection(request.host)
		except (socket.gaierror, socket.timeout, socket.error):
			if request.attempts <= 0:
				self.__publishResponse(ERROR.CONNECTION_FAILED, '', request)
			else:
				request.attempts -= 1
				self.__processRequest(request)
			return
		fno = conn.fileno()
		self.__connections[fno] = conn
		self.__epoll.register(fno, select.EPOLLIN)
		conn.sendRequest(request)

	# ------------------------------------------------------------------------------------
	def __processIncoming(self, fileno):
		conn = self.__connections[fileno]
		try:
			res = conn.processResponse()
		except (httplib.IncompleteRead, socket.error):
			return
		except httplib.BadStatusLine:
			self.__epoll.unregister(fileno)
			del self.__connections[fileno]
			if conn.request.attempts <= 0:
				self.__publishResponse(ERROR.CONNECTION_DROPED, '', conn.request)
			else:
				conn.request.attempts -= 1
				self.__processRequest(conn.request)
			return

		self.__epoll.unregister(fileno)
		status, data, callback = res
		self.__publishResponse(status, data, callback)
		self.__connectionManager.returnConnection(conn)
		del self.__connections[fileno]

# Interface
# ====================================================================================

# ------------------------------------------------------------------------------------
class HttpRequest:
	def __init__(self, host, method, query, data, callback):
		self.host = host
		self.method = method
		self.query = query
		self.data = data
		self.callback = callback
		self.attempts = SETTINGS.DEFAULT_ATTEMPTS

# ------------------------------------------------------------------------------------
_g_asyncFetcher = _AsyncFetcher()

# ------------------------------------------------------------------------------------
def fetch(request):
	_g_asyncFetcher.fetch(request)



#   Usage sample
# ====================================================================================

# ------------------------------------------------------------------------------------
def sampleCallback(status, data, request):
	print 'fetched:', status, len(data)
	print data

# ------------------------------------------------------------------------------------
if __name__ == '__main__':
	while True:
		fetch(HttpRequest('google.com:80', 'GET', '/', None, sampleCallback))
		time.sleep(4.0)