lannybroo / numpyIO

Implementation of scipy.io.numpyio.fread and scipy.io.numpyio.fwrite without depending on scipy. Uses numpy's tofile/fromfile instead.

Clone this repository (size: 28.0 KB): HTTPS / SSH
$ hg clone http://bitbucket.org/lannybroo/numpyio/
commit 14: 69175a505b96
parent 13: ace73c5c06a4
branch: default
Moved all tests into test_numpyIO.py and made compatible with nose.
Alan Brooks / lannybroo
15 months ago

Changed (Δ3.5 KB):

raw changeset »

numpyIO.py (17 lines added, 133 lines removed)

Up to file-list numpyIO.py:

1
'''
2
Author: Benyang Tang, btang(noSpam)@pacific.jpl.nasa.gov
3
Modified by: Nickolas Fotopoulos, nvf(noSpam)@mit.edu
4
Modified by: Alan C. Brooks, alancbrooks(noSpam)@gmail.com
1
''' fread and fwrite using only numpy
5
2
6
2005 Version posted at:
7
    http://projects.scipy.org/scipy/scipy/attachment/ticket/14
3
Authors: Benyang Tang, btang(noSpam)@pacific.jpl.nasa.gov
4
         Nickolas Fotopoulos, nvf(noSpam)@mit.edu
5
         Alan C. Brooks, alancbrooks(noSpam)@gmail.com
8
6
9
7
========
10
8
Documentation:
@@ -22,27 +20,16 @@ The interfaces of the 2 functions here a
22
20
numpyio. If you have code using numpyio, you don't have to change anything to
23
21
call fread and fwrite, except changing numpyio to numpyIO.
24
22
25
=======
26
Roadmap:
27
=======
28
Scipy's "IO Roadmap" on http://projects.scipy.org/scipy/scipy/roadmap says:
29
30
Reworking of IO package
31
32
The IO code in both NumPy and SciPy is undergoing a major reworking. NumPy
23
See SciPy's "IO Roadmap" on http://projects.scipy.org/scipy/scipy/roadmap which
24
says:
25
    The IO code in both NumPy and SciPy is undergoing a major reworking. NumPy
33
26
will be where basic code for reading and writing NumPy arrays is located,
34
27
while SciPy will house file readers and writers for various data formats
35
(data, audio, video, images, matlab, excel, etc.). This reworking started
36
NumPy 1.1.0 and will take place over many release. SciPy 0.7.0 has several
37
changes including:
38
39
Several functions in scipy.io have been deprecated and will be removed in the
40
0.8.0 release including npfile, save, load, create_module, create_shelf,
41
objload, objsave, fopen, read_array, write_array, fread, fwrite, bswap,
42
packbits, unpackbits, and convert_objectarray. Some of these functions have
43
been replaced by NumPy's raw reading and writing capabilities, memory-mapping
44
capabilities, or array methods. Others have been moved from SciPy to NumPy,
45
since basic array reading and writing capability is now handled by NumPy.
28
(data, audio, video, images, matlab, excel, etc.).
29
    Several functions in scipy.io have been deprecated and will be removed in the
30
0.8.0 release including npfile, ... fread, fwrite, ... . Some of these
31
functions have been replaced by NumPy's raw reading and writing capabilities,
32
memory-mapping capabilities, or array methods.
46
33
47
34
=======
48
35
History:
@@ -53,7 +40,7 @@ 2009-01-03: Read the roadmap and realize
53
40
2009-01-01: Happy with refactored implementation and complete tests (ACB).
54
41
2008-12-30: Cleanup and port to numpy 1.3.0 (ACB).
55
42
2005-09-02: Added a few new read_types to support Matlab R14 mat-files. Not
56
            comprehensive.
43
            comprehensive. http://projects.scipy.org/scipy/scipy/ticket/14
57
44
2003-03-31: Coded and tested.
58
45
'''
59
46
@@ -91,7 +78,6 @@ OPTIONAL
91
78
92
79
    return a
93
80
94
95
81
def fwrite(fid, num, a, write_type=None, byteswap=0):
96
82
    '''numpyIO.fwrite(fid, num, myArray, {write_type, byteswap})
97
83
     
@@ -130,113 +116,11 @@ OPTIONAL
130
116
    # write
131
117
    a.tofile(fid)
132
118
    
133
134
def _timed(func):
135
    '''Decorator that prints & returns the time it takes to run a function'''
136
    import time
137
    def wrapper(*__args,**__kw):
138
        start = time.time()
139
        out = func(*__args,**__kw)
140
        end = time.time()
141
        dt = end-start
142
        print "in %f sec" % dt,
143
        return (out, dt)
144
    return wrapper
145
    
146
def _subtestReadWrite(a, size, fn, fread, fwrite, what='Some'):
147
    import os
148
    print " %s testing .." % what,
149
    
150
    # Speed tests
151
    @_timed
152
    def innerSpeedTestWrite():
153
        fwrite(fid, size, a)
154
    @_timed
155
    def innerSpeedTestRead():
156
        return fread(fid, size, 'd')
157
    with open(fn,'w+b') as fid:
158
        tmp, dtW = innerSpeedTestWrite()
159
    with open(fn,'rb') as fid:
160
        a1, dtR = innerSpeedTestRead()
161
    os.remove(fn)
162
    
163
    # Test more options
164
    b = a[:10] # only use a few values for these tests
165
    with open(fn,'w+b') as fid:
166
        fwrite(fid, b.size, b, 'd', 1)
167
        fid.seek(0)
168
        # all data types: 'bBcS1hHfiIlu4dFD'
169
        b2 = fread(fid, b.size, 'd', 'f', 1) # convert to floats
170
        fid.seek(0)
171
        b3 = fread(fid, b.size, 'd', 'i', 1) # convert to int32
172
        fid.seek(0)
173
        b4 = fread(fid, b.size, 'd', 'b', 1) # convert to int8
174
        fid.seek(0)
175
        b5 = fread(fid, b.size, 'd', 'h', 1) # convert to int16
176
    os.remove(fn)
177
    
178
    # Finalize stuff
179
    if all(a == a1) and all(abs(b-b2) < 0.01) and \
180
       all(abs(b-b3) < 1) and all(abs(b-b4) < 1) and all(abs(b-b5) < 1):
181
        print ".. passed"
182
    else:
183
        print ".. failed"
184
        print " a=%s\na1=%s\nb2=%s\nb3=%s\nb4=%s\nb5=%s" % (
185
            a, a1, b2, b3, b4, b5)
186
    return dtW+dtR
187
188
@_timed
189
def _testFreadFwrite():
190
    # Unit tests of fread/fwrite
191
    from scipy.io import numpyio
192
    import time
193
    
194
    n = 10e5 #10e5 for quick dev; 10e6 for longer benchmark
195
    a = 127*(np.random.random_sample(n)-0.5)
196
    fn = 'temp.bin'
197
    dt = []
198
    print "Testing numpyIO with %d random samples written/read:" % n
199
    
200
    # Baseline scipy.io.numpyio
201
    t1 = _subtestReadWrite(a, a.size, fn, numpyio.fread, numpyio.fwrite, 
202
                           'numpyio')
203
    
204
    # Test this new package numpyIO & compare speed to baseline
205
    t2 = _subtestReadWrite(a, a.size, fn, fread, fwrite, 'numpyIO')
206
    
207
    # Mix the two
208
    t3 = _subtestReadWrite(a, a.size, fn, fread, numpyio.fwrite, 'numpyIo')
209
    t4 = _subtestReadWrite(a, a.size, fn, numpyio.fread, fwrite, 'numpyiO')
210
    
211
    # Print the relative results
212
    def printSlower(t1, t2, what1='a', what2='b'):
213
        speed = ('slower','faster')[t2<t1]
214
        print " %s is %2d%% %s than %s" % (
215
            what2, 100*abs(t2-t1)/t1, speed, what1)
216
    printSlower(t1, t2, "SciPy's numpyio", "numpyIO")
217
    printSlower(t1, t3, "SciPy's numpyio", "numpyIo")
218
    printSlower(t1, t4, "SciPy's numpyio", "numpyiO")
219
        
220
def _testAutoNum():
221
    import os
222
    print "Testing auto num feature of numpyIO ..",
223
    a = np.array((1,2,3),'d')
224
    fn = 'temp2.bin'
225
    with open(fn,'w+b') as fid:
226
        fwrite(fid, -1, a)
227
        fid.seek(0)
228
        a1 = fread(fid, -1, 'd')
229
    os.remove(fn)
230
    if all(a == a1):
231
        print "passed"
232
    else:
233
        print "failed"
234
        print " a=%s\na1=%s" % (a, a1)
235
119
    
236
120
def test():
237
    '''Run all unit tests'''
238
    _testAutoNum()
239
    _testFreadFwrite()
240
    
121
    '''Run all unit tests (recommend running with 'nosetests -v')'''
122
    import test_numpyIO
123
    test_numpyIO.runAll()
124
        
241
125
if __name__=='__main__':
242
126
    test()