dhellmann / feedcache (http://doughellmann.com/projects/feedcache/)
A Python class to wrap Mark Pilgrim's Universal Feed Parser module so that parameters can be used to cache the feed results locally instead of fetching the feed every time it is requested. Uses both etag and modified times for caching. The cache is parameterized to use different backend storage options.
Clone this repository (size: 62.8 KB): HTTPS / SSH
$ hg clone http://bitbucket.org/dhellmann/feedcache/
| commit 15: | b6ac6bc8c649 |
| parent 14: | 642f8ece9e78 |
| branch: | default |
API change for Cache
Changed (Δ460 bytes):
raw changeset »
ChangeLog (6 lines added, 0 lines removed)
feedcache/cache.py (1 lines added, 1 lines removed)
feedcache/example.py (0 lines added, 5 lines removed)
feedcache/test_cache.py (11 lines added, 10 lines removed)
1 |
1 |
2007-08-06 Doug Hellmann <doug.hellmann@gmail.com> |
2 |
2 |
|
3 |
* feedcache/cache.py (Cache.fetch): Change __getitem__ to fetch |
|
4 |
since Cache does not support the rest of the dictionary API. |
|
5 |
||
6 |
* feedcache/example.py: Remove logging configuration so log |
|
7 |
messages are discarded quietly. |
|
8 |
||
3 |
9 |
* feedcache/test_cache.py: Use dictionary instead of custom |
4 |
10 |
MemoryStorage class. |
5 |
11 |
Up to file-list feedcache/cache.py:
| … | … | @@ -76,7 +76,7 @@ class Cache: |
76 |
76 |
self.user_agent = userAgent |
77 |
77 |
return |
78 |
78 |
|
79 |
def |
|
79 |
def fetch(self, url): |
|
80 |
80 |
"Return the feed at url." |
81 |
81 |
logger.debug('url="%s"' % url) |
82 |
82 |
Up to file-list feedcache/example.py:
29 |
29 |
|
30 |
30 |
__module_id__ = "$Id$" |
31 |
31 |
|
32 |
import logging |
|
33 |
logging.basicConfig(level=logging.DEBUG, |
|
34 |
format='%(asctime)s %(levelname)-8s %(name)s %(message)s', |
|
35 |
) |
|
36 |
||
37 |
32 |
# |
38 |
33 |
# Import system modules |
39 |
34 |
# |
Up to file-list feedcache/test_cache.py:
| … | … | @@ -39,7 +39,6 @@ logger = logging.getLogger('feedcache.te |
39 |
39 |
# Import system modules |
40 |
40 |
# |
41 |
41 |
import os |
42 |
import tempfile |
|
43 |
42 |
import threading |
44 |
43 |
import time |
45 |
44 |
import unittest |
| … | … | @@ -56,6 +55,7 @@ from test_server import TestHTTPServer, |
56 |
55 |
# Module |
57 |
56 |
# |
58 |
57 |
|
58 |
||
59 |
59 |
class CacheTestBase(unittest.TestCase): |
60 |
60 |
"Base class for Cache tests" |
61 |
61 |
|
| … | … | @@ -89,6 +89,7 @@ class CacheTestBase(unittest.TestCase): |
89 |
89 |
ignore = urllib.urlretrieve('http://localhost:9999/shutdown') |
90 |
90 |
time.sleep(1) |
91 |
91 |
self.server.server_close() |
92 |
self.server_thread.join() |
|
92 |
93 |
return |
93 |
94 |
|
94 |
95 |
|
| … | … | @@ -102,7 +103,7 @@ class CacheTest(CacheTestBase): |
102 |
103 |
|
103 |
104 |
def testRetrieveNotInCache(self): |
104 |
105 |
# Retrieve data not already in the cache. |
105 |
feed_data = self.cache |
|
106 |
feed_data = self.cache.fetch(self.TEST_URL) |
|
106 |
107 |
self.failUnless(feed_data) |
107 |
108 |
self.failUnlessEqual(feed_data.feed.title, 'CacheTest test data') |
108 |
109 |
return |
| … | … | @@ -113,10 +114,10 @@ class CacheTest(CacheTestBase): |
113 |
114 |
# to the first. |
114 |
115 |
|
115 |
116 |
# First fetch |
116 |
feed_data = self.cache |
|
117 |
feed_data = self.cache.fetch(self.TEST_URL) |
|
117 |
118 |
|
118 |
119 |
# Second fetch |
119 |
feed_data2 = self.cache |
|
120 |
feed_data2 = self.cache.fetch(self.TEST_URL) |
|
120 |
121 |
|
121 |
122 |
# Since it is the in-memory storage, we should have the |
122 |
123 |
# exact same object. |
| … | … | @@ -129,14 +130,14 @@ class CacheTest(CacheTestBase): |
129 |
130 |
# is different from the first. |
130 |
131 |
|
131 |
132 |
# First fetch |
132 |
feed_data = self.cache |
|
133 |
feed_data = self.cache.fetch(self.TEST_URL) |
|
133 |
134 |
|
134 |
135 |
# Change the timeout and sleep to move the clock |
135 |
136 |
self.cache.time_to_live = 0 |
136 |
137 |
time.sleep(1) |
137 |
138 |
|
138 |
139 |
# Second fetch |
139 |
feed_data2 = self.cache |
|
140 |
feed_data2 = self.cache.fetch(self.TEST_URL) |
|
140 |
141 |
|
141 |
142 |
# Since we reparsed, the cache response should be different. |
142 |
143 |
self.failIf(feed_data is feed_data2) |
| … | … | @@ -172,7 +173,7 @@ class CacheUpdateTest(CacheTestBase): |
172 |
173 |
# codes cause us to use the same data. |
173 |
174 |
|
174 |
175 |
# First fetch populates the cache |
175 |
response1 = self.cache |
|
176 |
response1 = self.cache.fetch('http://localhost:9999/') |
|
176 |
177 |
self.failUnlessEqual(response1.feed.title, 'CacheTest test data') |
177 |
178 |
|
178 |
179 |
# Remove the modified setting from the cache so we know |
| … | … | @@ -190,7 +191,7 @@ class CacheUpdateTest(CacheTestBase): |
190 |
191 |
# update the storage, so our SingleWriteMemoryStorage |
191 |
192 |
# should not raise and we should have the same |
192 |
193 |
# response object. |
193 |
response2 = self.cache |
|
194 |
response2 = self.cache.fetch('http://localhost:9999/') |
|
194 |
195 |
self.failUnless(response1 is response2) |
195 |
196 |
|
196 |
197 |
# Should have hit the server twice |
| … | … | @@ -203,7 +204,7 @@ class CacheUpdateTest(CacheTestBase): |
203 |
204 |
# codes cause us to use the same data. |
204 |
205 |
|
205 |
206 |
# First fetch populates the cache |
206 |
response1 = self.cache |
|
207 |
response1 = self.cache.fetch('http://localhost:9999/') |
|
207 |
208 |
self.failUnlessEqual(response1.feed.title, 'CacheTest test data') |
208 |
209 |
|
209 |
210 |
# Remove the etag setting from the cache so we know |
| … | … | @@ -221,7 +222,7 @@ class CacheUpdateTest(CacheTestBase): |
221 |
222 |
# update the storage, so our SingleWriteMemoryStorage |
222 |
223 |
# should not raise and we should have the same |
223 |
224 |
# response object. |
224 |
response2 = self.cache |
|
225 |
response2 = self.cache.fetch('http://localhost:9999/') |
|
225 |
226 |
self.failUnless(response1 is response2) |
226 |
227 |
|
227 |
228 |
# Should have hit the server twice |