lannybroo / numpyIO
Implementation of scipy.io.numpyio.fread and scipy.io.numpyio.fwrite without depending on scipy. Uses numpy's tofile/fromfile instead.
Clone this repository (size: 28.0 KB): HTTPS / SSH
$ hg clone http://bitbucket.org/lannybroo/numpyio/
| commit 14: | 69175a505b96 |
| parent 13: | ace73c5c06a4 |
| branch: | default |
Moved all tests into test_numpyIO.py and made compatible with nose.
Changed (Δ3.5 KB):
raw changeset »
numpyIO.py (17 lines added, 133 lines removed)
1 |
''' |
|
2 |
Author: Benyang Tang, btang(noSpam)@pacific.jpl.nasa.gov |
|
3 |
Modified by: Nickolas Fotopoulos, nvf(noSpam)@mit.edu |
|
4 |
Modified by: Alan C. Brooks, alancbrooks(noSpam)@gmail.com |
|
1 |
''' fread and fwrite using only numpy |
|
5 |
2 |
|
6 |
2005 Version posted at: |
|
7 |
http://projects.scipy.org/scipy/scipy/attachment/ticket/14 |
|
3 |
Authors: Benyang Tang, btang(noSpam)@pacific.jpl.nasa.gov |
|
4 |
Nickolas Fotopoulos, nvf(noSpam)@mit.edu |
|
5 |
Alan C. Brooks, alancbrooks(noSpam)@gmail.com |
|
8 |
6 |
|
9 |
7 |
======== |
10 |
8 |
Documentation: |
| … | … | @@ -22,27 +20,16 @@ The interfaces of the 2 functions here a |
22 |
20 |
numpyio. If you have code using numpyio, you don't have to change anything to |
23 |
21 |
call fread and fwrite, except changing numpyio to numpyIO. |
24 |
22 |
|
25 |
======= |
|
26 |
Roadmap: |
|
27 |
======= |
|
28 |
Scipy's "IO Roadmap" on http://projects.scipy.org/scipy/scipy/roadmap says: |
|
29 |
||
30 |
Reworking of IO package |
|
31 |
||
32 |
The IO code in both NumPy and SciPy is undergoing a major reworking. NumPy |
|
23 |
See SciPy's "IO Roadmap" on http://projects.scipy.org/scipy/scipy/roadmap which |
|
24 |
says: |
|
25 |
The IO code in both NumPy and SciPy is undergoing a major reworking. NumPy |
|
33 |
26 |
will be where basic code for reading and writing NumPy arrays is located, |
34 |
27 |
while SciPy will house file readers and writers for various data formats |
35 |
(data, audio, video, images, matlab, excel, etc.). This reworking started |
|
36 |
NumPy 1.1.0 and will take place over many release. SciPy 0.7.0 has several |
|
37 |
changes including: |
|
38 |
||
39 |
Several functions in scipy.io have been deprecated and will be removed in the |
|
40 |
0.8.0 release including npfile, save, load, create_module, create_shelf, |
|
41 |
objload, objsave, fopen, read_array, write_array, fread, fwrite, bswap, |
|
42 |
packbits, unpackbits, and convert_objectarray. Some of these functions have |
|
43 |
been replaced by NumPy's raw reading and writing capabilities, memory-mapping |
|
44 |
capabilities, or array methods. Others have been moved from SciPy to NumPy, |
|
45 |
|
|
28 |
(data, audio, video, images, matlab, excel, etc.). |
|
29 |
Several functions in scipy.io have been deprecated and will be removed in the |
|
30 |
0.8.0 release including npfile, ... fread, fwrite, ... . Some of these |
|
31 |
functions have been replaced by NumPy's raw reading and writing capabilities, |
|
32 |
memory-mapping capabilities, or array methods. |
|
46 |
33 |
|
47 |
34 |
======= |
48 |
35 |
History: |
| … | … | @@ -53,7 +40,7 @@ 2009-01-03: Read the roadmap and realize |
53 |
40 |
2009-01-01: Happy with refactored implementation and complete tests (ACB). |
54 |
41 |
2008-12-30: Cleanup and port to numpy 1.3.0 (ACB). |
55 |
42 |
2005-09-02: Added a few new read_types to support Matlab R14 mat-files. Not |
56 |
comprehensive. |
|
43 |
comprehensive. http://projects.scipy.org/scipy/scipy/ticket/14 |
|
57 |
44 |
2003-03-31: Coded and tested. |
58 |
45 |
''' |
59 |
46 |
|
| … | … | @@ -91,7 +78,6 @@ OPTIONAL |
91 |
78 |
|
92 |
79 |
return a |
93 |
80 |
|
94 |
||
95 |
81 |
def fwrite(fid, num, a, write_type=None, byteswap=0): |
96 |
82 |
'''numpyIO.fwrite(fid, num, myArray, {write_type, byteswap}) |
97 |
83 |
|
| … | … | @@ -130,113 +116,11 @@ OPTIONAL |
130 |
116 |
# write |
131 |
117 |
a.tofile(fid) |
132 |
118 |
|
133 |
||
134 |
def _timed(func): |
|
135 |
'''Decorator that prints & returns the time it takes to run a function''' |
|
136 |
import time |
|
137 |
def wrapper(*__args,**__kw): |
|
138 |
start = time.time() |
|
139 |
out = func(*__args,**__kw) |
|
140 |
end = time.time() |
|
141 |
dt = end-start |
|
142 |
print "in %f sec" % dt, |
|
143 |
return (out, dt) |
|
144 |
return wrapper |
|
145 |
||
146 |
def _subtestReadWrite(a, size, fn, fread, fwrite, what='Some'): |
|
147 |
import os |
|
148 |
print " %s testing .." % what, |
|
149 |
||
150 |
# Speed tests |
|
151 |
@_timed |
|
152 |
def innerSpeedTestWrite(): |
|
153 |
fwrite(fid, size, a) |
|
154 |
@_timed |
|
155 |
def innerSpeedTestRead(): |
|
156 |
return fread(fid, size, 'd') |
|
157 |
with open(fn,'w+b') as fid: |
|
158 |
tmp, dtW = innerSpeedTestWrite() |
|
159 |
with open(fn,'rb') as fid: |
|
160 |
a1, dtR = innerSpeedTestRead() |
|
161 |
os.remove(fn) |
|
162 |
||
163 |
# Test more options |
|
164 |
b = a[:10] # only use a few values for these tests |
|
165 |
with open(fn,'w+b') as fid: |
|
166 |
fwrite(fid, b.size, b, 'd', 1) |
|
167 |
fid.seek(0) |
|
168 |
# all data types: 'bBcS1hHfiIlu4dFD' |
|
169 |
b2 = fread(fid, b.size, 'd', 'f', 1) # convert to floats |
|
170 |
fid.seek(0) |
|
171 |
b3 = fread(fid, b.size, 'd', 'i', 1) # convert to int32 |
|
172 |
fid.seek(0) |
|
173 |
b4 = fread(fid, b.size, 'd', 'b', 1) # convert to int8 |
|
174 |
fid.seek(0) |
|
175 |
b5 = fread(fid, b.size, 'd', 'h', 1) # convert to int16 |
|
176 |
os.remove(fn) |
|
177 |
||
178 |
# Finalize stuff |
|
179 |
if all(a == a1) and all(abs(b-b2) < 0.01) and \ |
|
180 |
all(abs(b-b3) < 1) and all(abs(b-b4) < 1) and all(abs(b-b5) < 1): |
|
181 |
print ".. passed" |
|
182 |
else: |
|
183 |
print ".. failed" |
|
184 |
print " a=%s\na1=%s\nb2=%s\nb3=%s\nb4=%s\nb5=%s" % ( |
|
185 |
a, a1, b2, b3, b4, b5) |
|
186 |
return dtW+dtR |
|
187 |
||
188 |
@_timed |
|
189 |
def _testFreadFwrite(): |
|
190 |
# Unit tests of fread/fwrite |
|
191 |
from scipy.io import numpyio |
|
192 |
import time |
|
193 |
||
194 |
n = 10e5 #10e5 for quick dev; 10e6 for longer benchmark |
|
195 |
a = 127*(np.random.random_sample(n)-0.5) |
|
196 |
fn = 'temp.bin' |
|
197 |
dt = [] |
|
198 |
print "Testing numpyIO with %d random samples written/read:" % n |
|
199 |
||
200 |
# Baseline scipy.io.numpyio |
|
201 |
t1 = _subtestReadWrite(a, a.size, fn, numpyio.fread, numpyio.fwrite, |
|
202 |
'numpyio') |
|
203 |
||
204 |
# Test this new package numpyIO & compare speed to baseline |
|
205 |
t2 = _subtestReadWrite(a, a.size, fn, fread, fwrite, 'numpyIO') |
|
206 |
||
207 |
# Mix the two |
|
208 |
t3 = _subtestReadWrite(a, a.size, fn, fread, numpyio.fwrite, 'numpyIo') |
|
209 |
t4 = _subtestReadWrite(a, a.size, fn, numpyio.fread, fwrite, 'numpyiO') |
|
210 |
||
211 |
# Print the relative results |
|
212 |
def printSlower(t1, t2, what1='a', what2='b'): |
|
213 |
speed = ('slower','faster')[t2<t1] |
|
214 |
print " %s is %2d%% %s than %s" % ( |
|
215 |
what2, 100*abs(t2-t1)/t1, speed, what1) |
|
216 |
printSlower(t1, t2, "SciPy's numpyio", "numpyIO") |
|
217 |
printSlower(t1, t3, "SciPy's numpyio", "numpyIo") |
|
218 |
printSlower(t1, t4, "SciPy's numpyio", "numpyiO") |
|
219 |
||
220 |
def _testAutoNum(): |
|
221 |
import os |
|
222 |
print "Testing auto num feature of numpyIO ..", |
|
223 |
a = np.array((1,2,3),'d') |
|
224 |
fn = 'temp2.bin' |
|
225 |
with open(fn,'w+b') as fid: |
|
226 |
fwrite(fid, -1, a) |
|
227 |
fid.seek(0) |
|
228 |
a1 = fread(fid, -1, 'd') |
|
229 |
os.remove(fn) |
|
230 |
if all(a == a1): |
|
231 |
print "passed" |
|
232 |
else: |
|
233 |
print "failed" |
|
234 |
print " a=%s\na1=%s" % (a, a1) |
|
235 |
119 |
|
236 |
120 |
def test(): |
237 |
'''Run all unit tests''' |
|
238 |
_testAutoNum() |
|
239 |
_testFreadFwrite() |
|
240 |
||
121 |
'''Run all unit tests (recommend running with 'nosetests -v')''' |
|
122 |
import test_numpyIO |
|
123 |
test_numpyIO.runAll() |
|
124 |
||
241 |
125 |
if __name__=='__main__': |
242 |
126 |
test() |
