1. kmdaily
  2. galaxy-central (ngs)

Source

galaxy-central (ngs) / test / base / twilltestcase.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
import pkg_resources
pkg_resources.require( "twill==0.9" )

import StringIO, os, sys, random, filecmp, time, unittest, urllib, logging, difflib, tarfile, zipfile, tempfile, re, shutil, subprocess
from itertools import *

import twill
import twill.commands as tc
from twill.other_packages._mechanize_dist import ClientForm
pkg_resources.require( "elementtree" )
pkg_resources.require( "MarkupSafe" )
from markupsafe import escape
from elementtree import ElementTree
from galaxy.web import security
from galaxy.web.framework.helpers import iff

buffer = StringIO.StringIO()

#Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose?
twill.set_output(buffer)
tc.config('use_tidy', 0)

# Dial ClientCookie logging down (very noisy)
logging.getLogger( "ClientCookie.cookies" ).setLevel( logging.WARNING )
log = logging.getLogger( __name__ )

class TwillTestCase( unittest.TestCase ):

    def setUp( self ):
        # Security helper
        self.security = security.SecurityHelper( id_secret='changethisinproductiontoo' )
        self.history_id = os.environ.get( 'GALAXY_TEST_HISTORY_ID', None )
        self.host = os.environ.get( 'GALAXY_TEST_HOST' )
        self.port = os.environ.get( 'GALAXY_TEST_PORT' )
        self.url = "http://%s:%s" % ( self.host, self.port )
        self.file_dir = os.environ.get( 'GALAXY_TEST_FILE_DIR' )
        self.keepOutdir = os.environ.get( 'GALAXY_TEST_SAVE', '' )
        if self.keepOutdir > '':
           try:
               os.makedirs(self.keepOutdir)
           except:
               pass
        self.home()

        #self.set_history()

    # Functions associated with files
    def files_diff( self, file1, file2, attributes=None ):
        """Checks the contents of 2 files for differences"""
        def get_lines_diff( diff ):
            count = 0
            for line in diff:
                if ( line.startswith( '+' ) and not line.startswith( '+++' ) ) or ( line.startswith( '-' ) and not line.startswith( '---' ) ):
                    count += 1
            return count
        if not filecmp.cmp( file1, file2 ):
            files_differ = False
            local_file = open( file1, 'U' ).readlines()
            history_data = open( file2, 'U' ).readlines()
            if attributes is None:
                attributes = {}
            if attributes.get( 'sort', False ):
                history_data.sort()
            ##Why even bother with the check loop below, why not just use the diff output? This seems wasteful.
            if len( local_file ) == len( history_data ):
                for i in range( len( history_data ) ):
                    if local_file[i].rstrip( '\r\n' ) != history_data[i].rstrip( '\r\n' ):
                        files_differ = True
                        break
            else:
                files_differ = True
            if files_differ:
                allowed_diff_count = int(attributes.get( 'lines_diff', 0 ))
                diff = list( difflib.unified_diff( local_file, history_data, "local_file", "history_data" ) )
                diff_lines = get_lines_diff( diff )
                if diff_lines > allowed_diff_count:
                    diff_slice = diff[0:40]
                    #FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff 
                    #due to unknown desired behavior when used in conjunction with a non-zero lines_diff
                    #PDF forgiveness can probably be handled better by not special casing by __extension__ here
                    #and instead using lines_diff or a regular expression matching
                    #or by creating and using a specialized pdf comparison function
                    if file1.endswith( '.pdf' ) or file2.endswith( '.pdf' ):
                        # PDF files contain creation dates, modification dates, ids and descriptions that change with each
                        # new file, so we need to handle these differences.  As long as the rest of the PDF file does
                        # not differ we're ok.
                        valid_diff_strs = [ 'description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator' ]
                        valid_diff = False
                        invalid_diff_lines = 0
                        for line in diff_slice:
                            # Make sure to lower case strings before checking.
                            line = line.lower()
                            # Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n'
                            if ( line.startswith( '+' ) or line.startswith( '-' ) ) and line.find( 'local_file' ) < 0 and line.find( 'history_data' ) < 0:
                                for vdf in valid_diff_strs:
                                    if line.find( vdf ) < 0:
                                        valid_diff = False
                                    else:
                                        valid_diff = True
                                        # Stop checking as soon as we know we have a valid difference
                                        break
                                if not valid_diff:
                                    invalid_diff_lines += 1
                        log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1,file2,allowed_diff_count,diff_lines,invalid_diff_lines))
                        if invalid_diff_lines > allowed_diff_count:
                            # Print out diff_slice so we can see what failed
                            print "###### diff_slice ######"
                            raise AssertionError( "".join( diff_slice ) )
                    else:
                        log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1,file2,allowed_diff_count,diff_lines))
                        for line in diff_slice:
                            for char in line:
                                if ord( char ) > 128:
                                    raise AssertionError( "Binary data detected, not displaying diff" )
                        raise AssertionError( "".join( diff_slice )  )

    def files_re_match( self, file1, file2, attributes=None ):
        """Checks the contents of 2 files for differences using re.match"""
        local_file = open( file1, 'U' ).readlines() #regex file
        history_data = open( file2, 'U' ).readlines()
        assert len( local_file ) == len( history_data ), 'Data File and Regular Expression File contain a different number of lines (%s != %s)\nHistory Data (first 40 lines):\n%s' % ( len( local_file ), len( history_data ), ''.join( history_data[:40] ) )
        if attributes is None:
            attributes = {}
        if attributes.get( 'sort', False ):
            history_data.sort()
        lines_diff = int(attributes.get( 'lines_diff', 0 ))
        line_diff_count = 0
        diffs = []
        for i in range( len( history_data ) ):
            if not re.match( local_file[i].rstrip( '\r\n' ), history_data[i].rstrip( '\r\n' ) ):
                line_diff_count += 1
                diffs.append( 'Regular Expression: %s\nData file         : %s' % ( local_file[i].rstrip( '\r\n' ),  history_data[i].rstrip( '\r\n' ) ) )
            if line_diff_count > lines_diff:
                raise AssertionError, "Regular expression did not match data file (allowed variants=%i):\n%s" % ( lines_diff, "".join( diffs ) )

    def files_re_match_multiline( self, file1, file2, attributes=None ):
        """Checks the contents of 2 files for differences using re.match in multiline mode"""
        local_file = open( file1, 'U' ).read() #regex file
        if attributes is None:
            attributes = {}
        if attributes.get( 'sort', False ):
            history_data = open( file2, 'U' ).readlines()
            history_data.sort()
            history_data = ''.join( history_data )
        else:
            history_data = open( file2, 'U' ).read()
        #lines_diff not applicable to multiline matching
        assert re.match( local_file, history_data, re.MULTILINE ), "Multiline Regular expression did not match data file"

    def files_contains( self, file1, file2, attributes=None ):
        """Checks the contents of file2 for substrings found in file1, on a per-line basis"""
        local_file = open( file1, 'U' ).readlines() #regex file
        #TODO: allow forcing ordering of contains
        history_data = open( file2, 'U' ).read()
        lines_diff = int( attributes.get( 'lines_diff', 0 ) )
        line_diff_count = 0
        while local_file:
            contains = local_file.pop( 0 ).rstrip( '\n\r' )
            if contains not in history_data:
                line_diff_count += 1
            if line_diff_count > lines_diff:
                raise AssertionError, "Failed to find '%s' in history data. (lines_diff=%i):\n" % ( contains, lines_diff )

    def get_filename( self, filename ):
        full = os.path.join( self.file_dir, filename)
        return os.path.abspath(full)

    def save_log( *path ):
        """Saves the log to a file"""
        filename = os.path.join( *path )
        file(filename, 'wt').write(buffer.getvalue())

    def upload_file( self, filename, ftype='auto', dbkey='unspecified (?)', space_to_tab = False, metadata = None, composite_data = None ):
        """Uploads a file"""
        self.visit_url( "%s/tool_runner?tool_id=upload1" % self.url )
        try: 
            self.refresh_form( "file_type", ftype ) #Refresh, to support composite files
            tc.fv("1","dbkey", dbkey)
            if metadata:
                for elem in metadata:
                    tc.fv( "1", "files_metadata|%s" % elem.get( 'name' ), elem.get( 'value' ) )
            if composite_data:
                for i, composite_file in enumerate( composite_data ):
                    filename = self.get_filename( composite_file.get( 'value' ) )
                    tc.formfile( "1", "files_%i|file_data" % i, filename )
                    tc.fv( "1", "files_%i|space_to_tab" % i, composite_file.get( 'space_to_tab', False ) )
            else:
                filename = self.get_filename( filename )
                tc.formfile( "1", "file_data", filename )
                tc.fv( "1", "space_to_tab", space_to_tab )
            tc.submit("runtool_btn")
            self.home()
        except AssertionError, err:
            errmsg = "Uploading file resulted in the following exception.  Make sure the file (%s) exists.  " % filename
            errmsg += str( err )
            raise AssertionError( errmsg )
        # Make sure every history item has a valid hid
        hids = self.get_hids_in_history()
        for hid in hids:
            try:
                valid_hid = int( hid )
            except:
                raise AssertionError, "Invalid hid (%s) created when uploading file %s" % ( hid, filename )
        # Wait for upload processing to finish (TODO: this should be done in each test case instead)
        self.wait()
    def upload_url_paste( self, url_paste, ftype='auto', dbkey='unspecified (?)' ):
        """Pasted data in the upload utility"""
        self.visit_page( "tool_runner/index?tool_id=upload1" )
        try: 
            tc.fv( "1", "file_type", ftype )
            tc.fv( "1", "dbkey", dbkey )
            tc.fv( "1", "url_paste", url_paste )
            tc.submit( "runtool_btn" )
            self.home()
        except Exception, e:
            errmsg = "Problem executing upload utility using url_paste: %s" % str( e )
            raise AssertionError( e )
        # Make sure every history item has a valid hid
        hids = self.get_hids_in_history()
        for hid in hids:
            try:
                valid_hid = int( hid )
            except:
                raise AssertionError, "Invalid hid (%s) created when pasting %s" % ( hid, url_paste )
        # Wait for upload processing to finish (TODO: this should be done in each test case instead)
        self.wait()

    # Functions associated with histories
    def check_history_for_errors( self ):
        """Raises an exception if there are errors in a history"""
        self.home()
        self.visit_page( "history" )
        page = self.last_page()
        if page.find( 'error' ) > -1:
            raise AssertionError('Errors in the history for user %s' % self.user )
    def check_history_for_string( self, patt, show_deleted=False ):
        """Looks for 'string' in history page"""
        self.home()
        if show_deleted:
            self.visit_page( "history?show_deleted=True" )
        else:
            self.visit_page( "history" )
        for subpatt in patt.split():
            try:
                tc.find( subpatt )
            except:
                fname = self.write_temp_file( tc.browser.get_html() )
                errmsg = "no match to '%s'\npage content written to '%s'" % ( subpatt, fname )
                raise AssertionError( errmsg )
        self.home()
    def clear_history( self ):
        """Empties a history of all datasets"""
        self.visit_page( "clear_history" )
        self.check_history_for_string( 'Your history is empty' )
        self.home()
    def delete_history( self, id ):
        """Deletes one or more histories"""
        history_list = self.get_histories_as_data_list()
        self.assertTrue( history_list )
        num_deleted = len( id.split( ',' ) )
        self.home()
        self.visit_page( "history/list?operation=delete&id=%s" % ( id ) )
        check_str = 'Deleted %d %s' % ( num_deleted, iff( num_deleted != 1, "histories", "history" ) )
        self.check_page_for_string( check_str )
        self.home()
    def delete_current_history( self, strings_displayed=[] ):
        """Deletes the current history"""
        self.home()
        self.visit_page( "history/delete_current" )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def get_histories_as_data_list( self ):
        """Returns the data elements of all histories"""
        tree = self.histories_as_xml_tree()
        data_list = [ elem for elem in tree.findall("data") ]
        return data_list
    def get_history_as_data_list( self, show_deleted=False ):
        """Returns the data elements of a history"""
        tree = self.history_as_xml_tree( show_deleted=show_deleted )
        data_list = [ elem for elem in tree.findall("data") ]
        return data_list
    def history_as_xml_tree( self, show_deleted=False ):
        """Returns a parsed xml object of a history"""
        self.home()
        self.visit_page( 'history?as_xml=True&show_deleted=%s' % show_deleted )
        xml = self.last_page()
        tree = ElementTree.fromstring(xml)
        return tree
    def histories_as_xml_tree( self ):
        """Returns a parsed xml object of all histories"""
        self.home()
        self.visit_page( 'history/list_as_xml' )
        xml = self.last_page()
        tree = ElementTree.fromstring(xml)
        return tree
    def history_options( self, user=False, active_datasets=False, activatable_datasets=False, histories_shared_by_others=False ):
        """Mimics user clicking on history options link"""
        self.home()
        self.visit_page( "root/history_options" )
        if user:
            self.check_page_for_string( 'Previously</a> stored histories' )
            if active_datasets:
                self.check_page_for_string( 'Create</a> a new empty history' )
                self.check_page_for_string( 'Construct workflow</a> from current history' )
                self.check_page_for_string( 'Clone</a> current history' ) 
            self.check_page_for_string( 'Share</a> current history' )
            self.check_page_for_string( 'Change default permissions</a> for current history' )
            if histories_shared_by_others:
                self.check_page_for_string( 'Histories</a> shared with you by others' )
        if activatable_datasets:
            self.check_page_for_string( 'Show deleted</a> datasets in current history' )
        self.check_page_for_string( 'Rename</a> current history' )
        self.check_page_for_string( 'Delete</a> current history' )
        self.home()
    def new_history( self, name=None ):
        """Creates a new, empty history"""
        self.home()
        if name:
            self.visit_url( "%s/history_new?name=%s" % ( self.url, name ) )
        else:
            self.visit_url( "%s/history_new" % self.url )
        self.check_history_for_string('Your history is empty')
        self.home()
    def rename_history( self, id, old_name, new_name ):
        """Rename an existing history"""
        self.home()
        self.visit_page( "history/rename?id=%s&name=%s" %( id, new_name ) )
        check_str = 'History: %s renamed to: %s' % ( old_name, urllib.unquote( new_name ) )
        self.check_page_for_string( check_str )
        self.home()
    def set_history( self ):
        """Sets the history (stores the cookies for this run)"""
        if self.history_id:
            self.home()
            self.visit_page( "history?id=%s" % self.history_id )
        else:
            self.new_history()
        self.home()
    def share_current_history( self, email, strings_displayed=[], strings_displayed_after_submit=[],
                               action='', action_strings_displayed=[], action_strings_displayed_after_submit=[] ):
        """Share the current history with different users"""
        self.visit_url( "%s/history/share" % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( 'share', 'email', email )
        tc.submit( 'share_button' )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        if action:
            # If we have an action, then we are sharing datasets with users that do not have access permissions on them
            for check_str in action_strings_displayed:
                self.check_page_for_string( check_str )
            tc.fv( 'share_restricted', 'action', action )
            tc.submit( "share_restricted_button" )
            for check_str in action_strings_displayed_after_submit:
                self.check_page_for_string( check_str )
        self.home()
    def share_histories_with_users( self, ids, emails, strings_displayed=[], strings_displayed_after_submit=[],
                                    action=None, action_strings_displayed=[] ):
        """Share one or more histories with one or more different users"""
        self.visit_url( "%s/history/list?id=%s&operation=Share" % ( self.url, ids ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( 'share', 'email', emails )
        tc.submit( 'share_button' )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        if action:
            # If we have an action, then we are sharing datasets with users that do not have access permissions on them
            tc.fv( 'share_restricted', 'action', action )
            tc.submit( "share_restricted_button" )
            for check_str in action_strings_displayed:
                self.check_page_for_string( check_str )
        self.home()
    def unshare_history( self, history_id, user_id, strings_displayed=[] ):
        """Unshare a history that has been shared with another user"""
        self.visit_url( "%s/history/list?id=%s&operation=share+or+publish" % ( self.url, history_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.visit_url( "%s/history/sharing?unshare_user=%s&id=%s" % ( self.url, user_id, history_id ) )
        self.home()
    def switch_history( self, id='', name='' ):
        """Switches to a history in the current list of histories"""
        self.visit_url( "%s/history/list?operation=switch&id=%s" % ( self.url, id ) )
        if name:
            self.check_history_for_string( escape( name ) )
        self.home()
    def view_stored_active_histories( self, strings_displayed=[] ):
        self.home()
        self.visit_page( "history/list" )
        self.check_page_for_string( 'Saved Histories' )
        self.check_page_for_string( '<input type="checkbox" name="id" value=' )
        self.check_page_for_string( 'operation=Rename' )
        self.check_page_for_string( 'operation=Switch' )
        self.check_page_for_string( 'operation=Delete' )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def view_stored_deleted_histories( self, strings_displayed=[] ):
        self.home()
        self.visit_page( "history/list?f-deleted=True" )
        self.check_page_for_string( 'Saved Histories' )
        self.check_page_for_string( '<input type="checkbox" name="id" value=' )
        self.check_page_for_string( 'operation=Undelete' )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def view_shared_histories( self, strings_displayed=[] ):
        self.home()
        self.visit_page( "history/list_shared" )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def clone_history( self, history_id, clone_choice, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.home()
        self.visit_page( "history/clone?id=%s" % history_id )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( '1', 'clone_choice', clone_choice )
        tc.submit( 'clone_choice_button' )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def make_accessible_via_link( self, history_id, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.home()
        self.visit_page( "history/list?operation=share+or+publish&id=%s" % history_id )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        # twill barfs on this form, possibly because it contains no fields, but not sure.
        # In any case, we have to mimic the form submission
        self.home()
        self.visit_page( 'history/sharing?id=%s&make_accessible_via_link=True' % history_id )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def disable_access_via_link( self, history_id, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.home()
        self.visit_page( "history/list?operation=share+or+publish&id=%s" % history_id )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        # twill barfs on this form, possibly because it contains no fields, but not sure.
        # In any case, we have to mimic the form submission
        self.home()
        self.visit_page( 'history/sharing?id=%s&disable_link_access=True' % history_id )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def import_history_via_url( self, history_id, email, strings_displayed_after_submit=[] ):
        self.home()
        self.visit_page( "history/imp?&id=%s" % history_id )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()

    # Functions associated with datasets (history items) and meta data
    def get_job_stderr( self, id ):
        self.visit_page( "dataset/stderr?id=%s" % id )
        return self.last_page()

    def _assert_dataset_state( self, elem, state ):
        if elem.get( 'state' ) != state:
            errmsg = "Expecting dataset state '%s', but state is '%s'. Dataset blurb: %s\n\n" % ( state, elem.get('state'), elem.text.strip() )
            errmsg += "---------------------- >> begin tool stderr << -----------------------\n"
            errmsg += self.get_job_stderr( elem.get( 'id' ) ) + "\n"
            errmsg += "----------------------- >> end tool stderr << ------------------------\n"
            raise AssertionError( errmsg )

    def check_metadata_for_string( self, patt, hid=None ):
        """Looks for 'patt' in the edit page when editing a dataset"""
        data_list = self.get_history_as_data_list()
        self.assertTrue( data_list )
        if hid is None: # take last hid
            elem = data_list[-1]
            hid = int( elem.get('hid') )
        self.assertTrue( hid )
        self.visit_page( "dataset/edit?hid=%s" % hid )
        for subpatt in patt.split():
            tc.find(subpatt)
    def delete_history_item( self, hda_id, strings_displayed=[] ):
        """Deletes an item from a history"""
        try:
            hda_id = int( hda_id )
        except:
            raise AssertionError, "Invalid hda_id '%s' - must be int" % hda_id
        self.visit_url( "%s/datasets/%s/delete?show_deleted_on_refresh=False" % ( self.url, self.security.encode_id( hda_id ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def undelete_history_item( self, hda_id, strings_displayed=[] ):
        """Un-deletes a deleted item in a history"""
        try:
            hda_id = int( hda_id )
        except:
            raise AssertionError, "Invalid hda_id '%s' - must be int" % hda_id
        self.visit_url( "%s/datasets/%s/undelete" % ( self.url, self.security.encode_id( hda_id ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def display_history_item( self, hda_id, strings_displayed=[] ):
        """Displays a history item - simulates eye icon click"""
        self.visit_url( '%s/datasets/%s/display/' % ( self.url, self.security.encode_id( hda_id ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def view_history( self, history_id, strings_displayed=[] ):
        """Displays a history for viewing"""
        self.visit_url( '%s/history/view?id=%s' % ( self.url, self.security.encode_id( history_id ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def edit_hda_attribute_info( self, hda_id, new_name='', new_info='', new_dbkey='', new_startcol='',
                                 strings_displayed=[], strings_not_displayed=[] ):
        """Edit history_dataset_association attribute information"""
        self.home()
        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
        submit_required = False
        self.check_page_for_string( 'Edit Attributes' )
        if new_name:
            tc.fv( 'edit_attributes', 'name', new_name )
            submit_required = True
        if new_info:
            tc.fv( 'edit_attributes', 'info', new_info )
            submit_required = True
        if new_dbkey:
            tc.fv( 'edit_attributes', 'dbkey', new_dbkey )
            submit_required = True
        if new_startcol:
            tc.fv( 'edit_attributes', 'startCol', new_startcol )
            submit_required = True
        if submit_required:
            tc.submit( 'save' )
            self.check_page_for_string( 'Attributes updated' )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed on Edit Attributes page." % check_str
            except:
                pass
        self.home()
    def check_hda_attribute_info( self, hda_id, strings_displayed=[] ):
        """Edit history_dataset_association attribute information"""
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def auto_detect_metadata( self, hda_id ):
        """Auto-detect history_dataset_association metadata"""
        self.home()
        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
        self.check_page_for_string( 'This will inspect the dataset and attempt' )
        tc.fv( 'auto_detect', 'detect', 'Auto-detect' )
        tc.submit( 'detect' )
        try:
            self.check_page_for_string( 'Attributes have been queued to be updated' )
            self.wait()
        except AssertionError:
            self.check_page_for_string( 'Attributes updated' )
        #self.check_page_for_string( 'Attributes updated' )
        self.home()
    def convert_format( self, hda_id, target_type ):
        """Convert format of history_dataset_association"""
        self.home()
        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
        self.check_page_for_string( 'This will inspect the dataset and attempt' )
        tc.fv( 'convert_data', 'target_type', target_type )
        tc.submit( 'convert_data' )
        self.check_page_for_string( 'The file conversion of Convert BED to GFF on data' )
        self.wait() #wait for the format convert tool to finish before returning
        self.home()
    def change_datatype( self, hda_id, datatype ):
        """Change format of history_dataset_association"""
        self.home()
        self.visit_url( "%s/datasets/%s/edit" % ( self.url, self.security.encode_id( hda_id ) ) )
        self.check_page_for_string( 'This will change the datatype of the existing dataset but' )
        tc.fv( 'change_datatype', 'datatype', datatype )
        tc.submit( 'change' )
        self.check_page_for_string( 'Changed the type of dataset' )
        self.home()
    def copy_history_item( self, source_dataset_id=None, target_history_id=None, all_target_history_ids=[],
                           deleted_history_ids=[] ):
        """
        Copy 1 history_dataset_association to 1 history (Limited by twill since it doesn't support multiple
        field names, such as checkboxes
        """
        self.home()
        self.visit_url( "%s/dataset/copy_datasets?source_dataset_ids=%s" % ( self.url, source_dataset_id ) )
        self.check_page_for_string( 'Source History:' )
        # Make sure all of users active histories are displayed
        for id in all_target_history_ids:
            self.check_page_for_string( id )
        # Make sure only active histories are displayed
        for id in deleted_history_ids:
            try:
                self.check_page_for_string( id )
                raise AssertionError, "deleted history id %d displayed in list of target histories" % id
            except:
                pass
        
        tc.fv( '1', 'target_history_id', target_history_id )
        tc.submit( 'do_copy' )
        check_str = '1 dataset copied to 1 history'
        self.check_page_for_string( check_str )
        self.home()
    def get_hids_in_history( self ):
        """Returns the list of hid values for items in a history"""
        data_list = self.get_history_as_data_list()
        hids = []
        for elem in data_list:
            hid = elem.get('hid')
            hids.append(hid)
        return hids
    def get_hids_in_histories( self ):
        """Returns the list of hids values for items in all histories"""
        data_list = self.get_histories_as_data_list()
        hids = []
        for elem in data_list:
            hid = elem.get('hid')
            hids.append(hid)
        return hids

    def makeTfname(self, fname=None):
        """safe temp name - preserve the file extension for tools that interpret it"""
        suffix = os.path.split(fname)[-1] # ignore full path
        fd,temp_prefix = tempfile.mkstemp(prefix='tmp',suffix=suffix)
        return temp_prefix

    def verify_dataset_correctness( self, filename, hid=None, wait=True, maxseconds=120, attributes=None ):
        """Verifies that the attributes and contents of a history item meet expectations"""
        if wait:
            self.wait( maxseconds=maxseconds ) #wait for job to finish
        data_list = self.get_history_as_data_list()
        self.assertTrue( data_list )
        if hid is None: # take last hid
            elem = data_list[-1]
            hid = str( elem.get('hid') )
        else:
            hid = str( hid )
            elems = [ elem for elem in data_list if elem.get('hid') == hid ]
            self.assertTrue( len(elems) == 1 )
            elem = elems[0]
        self.assertTrue( hid )
        self._assert_dataset_state( elem, 'ok' )
        if self.is_zipped( filename ):
            errmsg = 'History item %s is a zip archive which includes invalid files:\n' % hid
            zip_file = zipfile.ZipFile( filename, "r" )
            name = zip_file.namelist()[0]
            test_ext = name.split( "." )[1].strip().lower()
            if not ( test_ext == 'scf' or test_ext == 'ab1' or test_ext == 'txt' ):
                raise AssertionError( errmsg )
            for name in zip_file.namelist():
                ext = name.split( "." )[1].strip().lower()
                if ext != test_ext:
                    raise AssertionError( errmsg )
        else:
            local_name = self.get_filename( filename )
            temp_name = self.makeTfname(fname = filename)
            self.home()
            self.visit_page( "display?hid=" + hid )
            data = self.last_page()
            file( temp_name, 'wb' ).write(data)
            if self.keepOutdir > '':
                ofn = os.path.join(self.keepOutdir,os.path.basename(local_name))
                shutil.copy(temp_name,ofn)
                log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % (self.keepOutdir,ofn))
            try:
                # have to nest try-except in try-finally to handle 2.4
                try:
                    if attributes is None:
                        attributes = {}
                    compare = attributes.get( 'compare', 'diff' )
                    if attributes.get( 'ftype', None ) == 'bam':
                        local_fh, temp_name = self._bam_to_sam( local_name, temp_name )
                        local_name = local_fh.name
                    extra_files = attributes.get( 'extra_files', None )
                    if compare == 'diff':
                        self.files_diff( local_name, temp_name, attributes=attributes )
                    elif compare == 're_match':
                        self.files_re_match( local_name, temp_name, attributes=attributes )
                    elif compare == 're_match_multiline':
                        self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
                    elif compare == 'sim_size':
                        delta = attributes.get('delta','100')
                        s1 = len(data)
                        s2 = os.path.getsize(local_name)
                        if abs(s1-s2) > int(delta):
                           raise Exception, 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name,s1,local_name,s2,delta)
                    elif compare == "contains":
                        self.files_contains( local_name, temp_name, attributes=attributes )
                    else:
                        raise Exception, 'Unimplemented Compare type: %s' % compare
                    if extra_files:
                        self.verify_extra_files_content( extra_files, elem.get( 'id' ) )
                except AssertionError, err:
                    errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare )
                    errmsg += str( err )
                    raise AssertionError( errmsg )
            finally:
                os.remove( temp_name )

    def _bam_to_sam( self, local_name, temp_name ):
        temp_local = tempfile.NamedTemporaryFile( suffix='.sam', prefix='local_bam_converted_to_sam_' )
        fd, temp_temp = tempfile.mkstemp( suffix='.sam', prefix='history_bam_converted_to_sam_' )
        os.close( fd )
        p = subprocess.Popen( args="samtools view -h %s -o %s" % ( local_name, temp_local.name ), shell=True )
        assert not p.wait(), 'Converting local (test-data) bam to sam failed'
        p = subprocess.Popen( args="samtools view -h %s -o %s" % ( temp_name, temp_temp ), shell=True )
        assert not p.wait(), 'Converting history bam to sam failed'
        os.remove( temp_name )
        return temp_local, temp_temp

    def verify_extra_files_content( self, extra_files, hda_id ):
        files_list = []
        for extra_type, extra_value, extra_name, extra_attributes in extra_files:
            if extra_type == 'file':
                files_list.append( ( extra_name, extra_value, extra_attributes ) )
            elif extra_type == 'directory':
                for filename in os.listdir( self.get_filename( extra_value ) ):
                    files_list.append( ( filename, os.path.join( extra_value, filename ), extra_attributes ) )
            else:
                raise ValueError, 'unknown extra_files type: %s' % extra_type
        for filename, filepath, attributes in files_list:
            self.verify_composite_datatype_file_content( filepath, hda_id, base_name = filename, attributes = attributes )
        
    def verify_composite_datatype_file_content( self, file_name, hda_id, base_name = None, attributes = None ):
        local_name = self.get_filename( file_name )
        if base_name is None:
            base_name = os.path.split(file_name)[-1]
        temp_name = self.makeTfname(fname = base_name)
        self.visit_url( "%s/datasets/%s/display/%s" % ( self.url, self.security.encode_id( hda_id ), base_name ) )
        data = self.last_page()
        file( temp_name, 'wb' ).write( data )
        if self.keepOutdir > '':
            ofn = os.path.join(self.keepOutdir,base_name)
            shutil.copy(temp_name,ofn)
            log.debug('## GALAXY_TEST_SAVE=%s. saved %s' % (self.keepOutdir,ofn))
        try:
            # have to nest try-except in try-finally to handle 2.4
            try:
                if attributes is None:
                    attributes = {}
                compare = attributes.get( 'compare', 'diff' )
                if compare == 'diff':
                    self.files_diff( local_name, temp_name, attributes=attributes )
                elif compare == 're_match':
                    self.files_re_match( local_name, temp_name, attributes=attributes )
                elif compare == 're_match_multiline':
                    self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
                elif compare == 'sim_size':
                    delta = attributes.get('delta','100')
                    s1 = len(data)
                    s2 = os.path.getsize(local_name)
                    if abs(s1-s2) > int(delta):
                       raise Exception, 'Files %s=%db but %s=%db - compare (delta=%s) failed' % (temp_name,s1,local_name,s2,delta)
                else:
                    raise Exception, 'Unimplemented Compare type: %s' % compare
            except AssertionError, err:
                errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare )
                errmsg += str( err )
                raise AssertionError( errmsg )
        finally:
            os.remove( temp_name )

    def is_zipped( self, filename ):
        if not zipfile.is_zipfile( filename ):
            return False
        return True

    def is_binary( self, filename ):
        temp = open( filename, "U" ) # why is this not filename? Where did temp_name come from
        lineno = 0
        for line in temp:
            lineno += 1
            line = line.strip()
            if line:
                for char in line:
                    if ord( char ) > 128:
                        return True
            if lineno > 10:
                break
        return False

    def verify_genome_build( self, dbkey='hg17' ):
        """Verifies that the last used genome_build at history id 'hid' is as expected"""
        data_list = self.get_history_as_data_list()
        self.assertTrue( data_list )
        elems = [ elem for elem in data_list ]
        elem = elems[-1]
        genome_build = elem.get('dbkey')
        self.assertTrue( genome_build == dbkey )

    # Functions associated with user accounts
    def create( self, cntrller='user', email='test@bx.psu.edu', password='testuser', username='admin-user', webapp='galaxy', referer='' ):
        # HACK: don't use panels because late_javascripts() messes up the twill browser and it 
        # can't find form fields (and hence user can't be logged in).
        self.visit_url( "%s/user/create?cntrller=%s&use_panels=False" % ( self.url, cntrller ) )
        tc.fv( '1', 'email', email )
        tc.fv( '1', 'webapp', webapp )
        tc.fv( '1', 'referer', referer )
        tc.fv( '1', 'password', password )
        tc.fv( '1', 'confirm', password )
        tc.fv( '1', 'username', username )
        tc.submit( 'create_user_button' )
        previously_created = False
        username_taken = False
        invalid_username = False
        try:
            self.check_page_for_string( "Created new user account" )
        except:
            try:
                # May have created the account in a previous test run...
                self.check_page_for_string( "User with that email already exists" )
                previously_created = True
            except:
                try:
                    self.check_page_for_string( 'This user name is not available' )
                    username_taken = True
                except:
                    try:
                        # Note that we're only checking if the usr name is >< 4 chars here...
                        self.check_page_for_string( 'User name must be at least 4 characters in length' )
                        invalid_username = True
                    except:
                        pass
        return previously_created, username_taken, invalid_username
    def create_user_with_info( self, email, password, username, user_info_values, user_type_fd_id='', cntrller='user',
                               strings_displayed=[], strings_displayed_after_submit=[] ):
        # This method creates a new user with associated info
        self.visit_url( "%s/user/create?cntrller=%s&use_panels=False" % ( self.url, cntrller ) )
        tc.fv( "1", "email", email )
        tc.fv( "1", "password", password )
        tc.fv( "1", "confirm", password )
        tc.fv( "1", "username", username )
        if user_type_fd_id:
            # The user_type_fd_id SelectField requires a refresh_on_change
            self.refresh_form( 'user_type_fd_id', user_type_fd_id )
            for index, ( field_name, info_value ) in enumerate( user_info_values ):
                tc.fv( "1", field_name, info_value )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str)
        tc.submit( "create_user_button" )
    def edit_user_info( self, cntrller='user', id='', new_email='', new_username='', password='', new_password='',
                        info_values=[], strings_displayed=[], strings_displayed_after_submit=[] ):
        if cntrller == 'admin':
            url = "%s/admin/users?id=%s&operation=information" % ( self.url, id )
        else:  # cntrller == 'user:
            # The user is editing his own info, so the user id is gotten from trans.user.
            url = "%s/user/manage_user_info?cntrller=user" % self.url
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        if new_email or new_username:
            if new_email:
                tc.fv( "login_info", "email", new_email )
            if new_username:
                tc.fv( "login_info", "username", new_username )
            tc.submit( "login_info_button" )
        if password and new_password:
            tc.fv( "change_password", "current", password )
            tc.fv( "change_password", "password", new_password )
            tc.fv( "change_password", "confirm", new_password )
            tc.submit( "change_password_button" )
        if info_values:
            for index, ( field_name, info_value ) in enumerate( info_values ):
                field_index = index + 1
                tc.fv( "user_info", field_name, info_value )
            tc.submit( "edit_user_info_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def user_set_default_permissions( self, cntrller='user', permissions_out=[], permissions_in=[], role_id='2' ):
        # role.id = 2 is Private Role for test2@bx.psu.edu 
        # NOTE: Twill has a bug that requires the ~/user/permissions page to contain at least 1 option value 
        # in each select list or twill throws an exception, which is: ParseError: OPTION outside of SELECT
        # Due to this bug, we'll bypass visiting the page, and simply pass the permissions on to the 
        # /user/set_default_permissions method.
        url = "user/set_default_permissions?cntrller=%s&update_roles_button=Save&id=None" % cntrller
        for po in permissions_out:
            key = '%s_out' % po
            url ="%s&%s=%s" % ( url, key, str( role_id ) )
        for pi in permissions_in:
            key = '%s_in' % pi
            url ="%s&%s=%s" % ( url, key, str( role_id ) )
        self.visit_url( "%s/%s" % ( self.url, url ) )
        self.check_page_for_string( 'Default new history permissions have been changed.' )
        self.home()
    def history_set_default_permissions( self, permissions_out=[], permissions_in=[], role_id=3 ): # role.id = 3 is Private Role for test3@bx.psu.edu 
        # NOTE: Twill has a bug that requires the ~/user/permissions page to contain at least 1 option value 
        # in each select list or twill throws an exception, which is: ParseError: OPTION outside of SELECT
        # Due to this bug, we'll bypass visiting the page, and simply pass the permissions on to the 
        # /user/set_default_permissions method.
        url = "root/history_set_default_permissions?update_roles_button=Save&id=None&dataset=True"
        for po in permissions_out:
            key = '%s_out' % po
            url ="%s&%s=%s" % ( url, key, str( role_id ) )
        for pi in permissions_in:
            key = '%s_in' % pi
            url ="%s&%s=%s" % ( url, key, str( role_id ) )
        self.home()
        self.visit_url( "%s/%s" % ( self.url, url ) )
        self.check_page_for_string( 'Default history permissions have been changed.' )
        self.home()
    def login( self, email='test@bx.psu.edu', password='testuser', username='admin-user', webapp='galaxy', referer='' ):
        # test@bx.psu.edu is configured as an admin user
        previously_created, username_taken, invalid_username = \
            self.create( email=email, password=password, username=username, webapp=webapp, referer=referer )
        if previously_created:
            # The acount has previously been created, so just login.
            # HACK: don't use panels because late_javascripts() messes up the twill browser and it 
            # can't find form fields (and hence user can't be logged in).
            self.visit_url( "%s/user/login?use_panels=False" % self.url )
            tc.fv( '1', 'email', email )
            tc.fv( '1', 'webapp', webapp )
            tc.fv( '1', 'referer', referer )
            tc.fv( '1', 'password', password )
            tc.submit( 'login_button' )
    def logout( self ):
        self.home()
        self.visit_page( "user/logout" )
        self.check_page_for_string( "You have been logged out" )
        self.home()
    
    # Functions associated with browsers, cookies, HTML forms and page visits

    def check_page_for_string( self, patt ):
        """Looks for 'patt' in the current browser page"""        
        page = self.last_page()
        if page.find( patt ) == -1:
            fname = self.write_temp_file( page )
            errmsg = "no match to '%s'\npage content written to '%s'" % ( patt, fname )
            raise AssertionError( errmsg )
        
    def check_string_count_in_page( self, patt, min_count ):
        """Checks the number of 'patt' occurrences in the current browser page"""        
        page = self.last_page()
        patt_count = page.count( patt )
        # The number of occurrences of patt in the page should be at least min_count
        # so show error if patt_count is less than min_count
        if patt_count < min_count:
            fname = self.write_temp_file( page )
            errmsg = "%i occurrences of '%s' found instead of %i.\npage content written to '%s' " % ( min_count, patt, patt_count, fname )
            raise AssertionError( errmsg )
            
    def check_string_not_in_page( self, patt ):
        """Checks to make sure 'patt' is NOT in the page."""        
        page = self.last_page()
        if page.find( patt ) != -1:
            fname = self.write_temp_file( page )
            errmsg = "string (%s) incorrectly displayed in page.\npage content written to '%s'" % ( patt, fname )
            raise AssertionError( errmsg )
        
    def check_page(self, strings_displayed, strings_displayed_count, strings_not_displayed):
        """Checks a page for strings displayed, not displayed and number of occurrences of a string"""
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str, count in strings_displayed_count:
            self.check_string_count_in_page( check_str, count )
        for check_str in strings_not_displayed:
            self.check_string_not_in_page( check_str )

    
    def write_temp_file( self, content, suffix='.html' ):
        fd, fname = tempfile.mkstemp( suffix=suffix, prefix='twilltestcase-' )
        f = os.fdopen( fd, "w" )
        f.write( content )
        f.close()
        return fname

    def clear_cookies( self ):
        tc.clear_cookies()

    def clear_form( self, form=0 ):
        """Clears a form"""
        tc.formclear(str(form))

    def home( self ):
        self.visit_url( self.url )

    def last_page( self ):
        return tc.browser.get_html()

    def load_cookies( self, file ):
        filename = self.get_filename(file)
        tc.load_cookies(filename)

    def reload_page( self ):
        tc.reload()
        tc.code(200)

    def show_cookies( self ):
        return tc.show_cookies()

    def showforms( self ):
        """Shows form, helpful for debugging new tests"""
        return tc.showforms()

    def submit_form( self, form_no=0, button="runtool_btn", **kwd ):
        """Populates and submits a form from the keyword arguments."""
        # An HTMLForm contains a sequence of Controls.  Supported control classes are:
        # TextControl, FileControl, ListControl, RadioControl, CheckboxControl, SelectControl,
        # SubmitControl, ImageControl
        for i, f in enumerate( self.showforms() ):
            if i == form_no:
                break
        # To help with debugging a tool, print out the form controls when the test fails
        print "form '%s' contains the following controls ( note the values )" % f.name
        controls = {}
        hc_prefix = '<HiddenControl('
        for i, control in enumerate( f.controls ):
           print "control %d: %s" % ( i, str( control ) )
           if not hc_prefix in str( control ):
              try:
                #check if a repeat element needs to be added
                if control.name not in kwd and control.name.endswith( '_add' ):
                    #control name doesn't exist, could be repeat
                    repeat_startswith = control.name[0:-4]
                    if repeat_startswith and not [ c_name for c_name in controls.keys() if c_name.startswith( repeat_startswith ) ] and [ c_name for c_name in kwd.keys() if c_name.startswith( repeat_startswith ) ]:
                        tc.submit( control.name )
                        return self.submit_form( form_no=form_no, button=button, **kwd )
                # Check for refresh_on_change attribute, submit a change if required
                if hasattr( control, 'attrs' ) and 'refresh_on_change' in control.attrs.keys():
                    changed = False
                    item_labels = [ item.attrs[ 'label' ] for item in control.get_items() if item.selected ] #For DataToolParameter, control.value is the HDA id, but kwd contains the filename.  This loop gets the filename/label for the selected values.
                    for value in kwd[ control.name ]:
                        if value not in control.value and True not in [ value in item_label for item_label in item_labels ]:
                            changed = True
                            break
                    if changed:
                        # Clear Control and set to proper value
                        control.clear()
                        # kwd[control.name] should be a singlelist
                        for elem in kwd[ control.name ]:
                            tc.fv( f.name, control.name, str( elem ) )
                        # Create a new submit control, allows form to refresh, instead of going to next page
                        control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name':'refresh_grouping'} )
                        control.add_to_form( f )
                        control.fixup()
                        # Submit for refresh
                        tc.submit( '___refresh_grouping___' )
                        return self.submit_form( form_no=form_no, button=button, **kwd )
              except Exception, e:
                log.debug( "In submit_form, continuing, but caught exception: %s" % str( e ) )
                continue
              controls[ control.name ] = control
        # No refresh_on_change attribute found in current form, so process as usual
        for control_name, control_value in kwd.items():
            if control_name not in controls:
                continue # these cannot be handled safely - cause the test to barf out
            if not isinstance( control_value, list ):
                control_value = [ control_value ]
            control = controls[ control_name ]
            control.clear()
            if control.is_of_kind( "text" ):
                tc.fv( f.name, control.name, ",".join( control_value ) )
            elif control.is_of_kind( "list" ):
                try:
                    if control.is_of_kind( "multilist" ):
                        if control.type == "checkbox":
                            def is_checked( value ):
                                # Copied from form_builder.CheckboxField
                                if value == True:
                                    return True
                                if isinstance( value, list ):
                                    value = value[0]
                                return isinstance( value, basestring ) and value.lower() in ( "yes", "true", "on" )
                            try:
                                checkbox = control.get()
                                checkbox.selected = is_checked( control_value )
                            except Exception, e1:
                                print "Attempting to set checkbox selected value threw exception: ", e1
                                # if there's more than one checkbox, probably should use the behaviour for
                                # ClientForm.ListControl ( see twill code ), but this works for now...
                                for elem in control_value:
                                    control.get( name=elem ).selected = True
                        else:
                            for elem in control_value:
                                control.get( name=elem ).selected = True
                    else: # control.is_of_kind( "singlelist" )
                        for elem in control_value:
                            try:
                                tc.fv( f.name, control.name, str( elem ) )
                            except Exception, e2:
                                print "Attempting to set control '", control.name, "' to value '", elem, "' threw exception: ", e2
                                # Galaxy truncates long file names in the dataset_collector in ~/parameters/basic.py
                                if len( elem ) > 30:
                                    elem_name = '%s..%s' % ( elem[:17], elem[-11:] )
                                else:
                                    elem_name = elem
                                tc.fv( f.name, control.name, str( elem_name ) )
                except Exception, exc:
                    errmsg = "Attempting to set field '%s' to value '%s' in form '%s' threw exception: %s\n" % ( control_name, str( control_value ), f.name, str( exc ) )
                    errmsg += "control: %s\n" % str( control )
                    errmsg += "If the above control is a DataToolparameter whose data type class does not include a sniff() method,\n"
                    errmsg += "make sure to include a proper 'ftype' attribute to the tag for the control within the <test> tag set.\n"
                    raise AssertionError( errmsg )
            else:
                # Add conditions for other control types here when necessary.
                pass
        tc.submit( button )
    def refresh_form( self, control_name, value, form_no=0, **kwd ):
        """Handles Galaxy's refresh_on_change for forms without ultimately submitting the form"""
        # control_name is the name of the form field that requires refresh_on_change, and value is
        # the value to which that field is being set.
        for i, f in enumerate( self.showforms() ):
            if i == form_no:
                break
        try:
            control = f.find_control( name=control_name )
        except:
            # This assumes we always want the first control of the given name, which may not be ideal...
            control = f.find_control( name=control_name, nr=0 )
        # Check for refresh_on_change attribute, submit a change if required
        if 'refresh_on_change' in control.attrs.keys():
            # Clear Control and set to proper value
            control.clear()
            tc.fv( f.name, control.name, value )
            # Create a new submit control, allows form to refresh, instead of going to next page
            control = ClientForm.SubmitControl( 'SubmitControl', '___refresh_grouping___', {'name':'refresh_grouping'} )
            control.add_to_form( f )
            control.fixup()
            # Submit for refresh
            tc.submit( '___refresh_grouping___' )
    def visit_page( self, page ):
        # tc.go("./%s" % page)
        if not page.startswith( "/" ):
            page = "/" + page 
        tc.go( self.url + page )
        tc.code( 200 )

    def visit_url( self, url ):
        tc.go("%s" % url)
        tc.code( 200 )

    """Functions associated with Galaxy tools"""
    def run_tool( self, tool_id, repeat_name=None, **kwd ):
        tool_id = tool_id.replace(" ", "+")
        """Runs the tool 'tool_id' and passes it the key/values from the *kwd"""
        self.visit_url( "%s/tool_runner/index?tool_id=%s" % (self.url, tool_id) )
        if repeat_name is not None:
            repeat_button = '%s_add' % repeat_name
            # Submit the "repeat" form button to add an input)
            tc.submit( repeat_button )
            print "button '%s' clicked" % repeat_button
        tc.find( 'runtool_btn' )
        self.submit_form( **kwd )

    def run_ucsc_main( self, track_params, output_params ):
        """Gets Data From UCSC"""
        tool_id = "ucsc_table_direct1"
        track_string = urllib.urlencode( track_params )
        galaxy_url = urllib.quote_plus( "%s/tool_runner/index?" % self.url )
        self.visit_url( "http://genome.ucsc.edu/cgi-bin/hgTables?GALAXY_URL=%s&hgta_compressType=none&tool_id=%s&%s" % ( galaxy_url, tool_id, track_string ) )
        tc.fv( "mainForm", "hgta_doTopSubmit", "get output" )
        self.submit_form( button="get output" )#, **track_params )
        tc.fv( 2, "hgta_doGalaxyQuery", "Send query to Galaxy" )
        self.submit_form( button="Send query to Galaxy" )#, **output_params ) #AssertionError: Attempting to set field 'fbQual' to value '['whole']' in form 'None' threw exception: no matching forms! control: <RadioControl(fbQual=[whole, upstreamAll, endAll])>

    def wait( self, maxseconds=120 ):
        """Waits for the tools to finish"""
        sleep_amount = 0.1
        slept = 0
        self.home()
        while slept <= maxseconds:
            self.visit_page( "history" )
            page = tc.browser.get_html()
            if page.find( '<!-- running: do not change this comment, used by TwillTestCase.wait -->' ) > -1:
                time.sleep( sleep_amount )
                slept += sleep_amount
                sleep_amount *= 2
                if slept + sleep_amount > maxseconds:
                    sleep_amount = maxseconds - slept # don't overshoot maxseconds
            else:
                break
        assert slept < maxseconds

    # Dataset Security stuff
    # Tests associated with users
    def create_new_account_as_admin( self, email='test4@bx.psu.edu', password='testuser',
                                     username='regular-user4', webapp='galaxy', referer='' ):
        """Create a new account for another user"""
        # HACK: don't use panels because late_javascripts() messes up the twill browser and it 
        # can't find form fields (and hence user can't be logged in).
        self.visit_url( "%s/user/create?cntrller=admin" % self.url )
        tc.fv( '1', 'email', email )
        tc.fv( '1', 'webapp', webapp )
        tc.fv( '1', 'referer', referer )
        tc.fv( '1', 'password', password )
        tc.fv( '1', 'confirm', password )
        tc.fv( '1', 'username', username )
        tc.submit( 'create_user_button' )
        previously_created = False
        username_taken = False
        invalid_username = False
        try:
            self.check_page_for_string( "Created new user account" )
        except:
            try:
                # May have created the account in a previous test run...
                self.check_page_for_string( "User with that email already exists" )
                previously_created = True
            except:
                try:
                    self.check_page_for_string( 'This user name is not available' )
                    username_taken = True
                except:
                    try:
                        # Note that we're only checking if the usr name is >< 4 chars here...
                        self.check_page_for_string( 'User name must be at least 4 characters in length' )
                        invalid_username = True
                    except:
                        pass
        return previously_created, username_taken, invalid_username
    def reset_password_as_admin( self, user_id, password='testreset' ):
        """Reset a user password"""
        self.home()
        self.visit_url( "%s/admin/reset_user_password?id=%s" % ( self.url, user_id ) )
        tc.fv( "1", "password", password )
        tc.fv( "1", "confirm", password )
        tc.submit( "reset_user_password_button" )
        self.check_page_for_string( "Passwords reset for 1 users" )
        self.home()
    def mark_user_deleted( self, user_id, email='' ):
        """Mark a user as deleted"""
        self.home()
        self.visit_url( "%s/admin/users?operation=delete&id=%s" % ( self.url, user_id ) )
        check_str = "Deleted 1 users"
        self.check_page_for_string( check_str )
        self.home()
    def undelete_user( self, user_id, email='' ):
        """Undelete a user"""
        self.home()
        self.visit_url( "%s/admin/users?operation=undelete&id=%s" % ( self.url, user_id ) )
        check_str = "Undeleted 1 users"
        self.check_page_for_string( check_str )
        self.home()
    def purge_user( self, user_id, email ):
        """Purge a user account"""
        self.home()
        self.visit_url( "%s/admin/users?operation=purge&id=%s" % ( self.url, user_id ) )
        check_str = "Purged 1 users"
        self.check_page_for_string( check_str )
        self.home()
    def manage_roles_and_groups_for_user( self, user_id, in_role_ids=[], out_role_ids=[],
                                          in_group_ids=[], out_group_ids=[], strings_displayed=[] ):
        self.home()
        url = "%s/admin/manage_roles_and_groups_for_user?id=%s" % ( self.url, user_id )
        if in_role_ids:
            url += "&in_roles=%s" % ','.join( in_role_ids )
        if out_role_ids:
            url += "&out_roles=%s" % ','.join( out_role_ids )
        if in_group_ids:
            url += "&in_groups=%s" % ','.join( in_group_ids )
        if out_group_ids:
            url += "&out_groups=%s" % ','.join( out_group_ids )
        if in_role_ids or out_role_ids or in_group_ids or out_group_ids:
            url += "&user_roles_groups_edit_button=Save"
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()

    # Tests associated with roles
    def browse_roles( self, strings_displayed=[] ):
        self.visit_url( '%s/admin/roles' % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def create_role( self,
                     name='Role One',
                     description="This is Role One",
                     in_user_ids=[],
                     in_group_ids=[],
                     create_group_for_role='',
                     private_role='',
                     strings_displayed=[] ):
        """Create a new role"""
        url = "%s/admin/roles?operation=create&create_role_button=Save&name=%s&description=%s" % \
            ( self.url, name.replace( ' ', '+' ), description.replace( ' ', '+' ) )
        if in_user_ids:
            url += "&in_users=%s" % ','.join( in_user_ids )
        if in_group_ids:
            url += "&in_groups=%s" % ','.join( in_group_ids )
        if create_group_for_role == 'yes':
            url += '&create_group_for_role=yes&create_group_for_role=yes'
        self.home()
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        if private_role:
            # Make sure no private roles are displayed
            try:
                self.check_page_for_string( private_role )
                errmsg = 'Private role %s displayed on Roles page' % private_role
                raise AssertionError( errmsg )
            except AssertionError:
                # Reaching here is the behavior we want since no private roles should be displayed
                pass
        self.home()
        self.visit_url( "%s/admin/roles" % self.url )
        self.check_page_for_string( name )
        self.home()
    def rename_role( self, role_id, name='Role One Renamed', description='This is Role One Re-described' ):
        """Rename a role"""
        self.home()
        self.visit_url( "%s/admin/roles?operation=rename&id=%s" % ( self.url, role_id ) )
        self.check_page_for_string( 'Change role name and description' )
        tc.fv( "1", "name", name )
        tc.fv( "1", "description", description )
        tc.submit( "rename_role_button" )
        self.home()
    def mark_role_deleted( self, role_id, role_name ):
        """Mark a role as deleted"""
        self.home()
        self.visit_url( "%s/admin/roles?operation=delete&id=%s" % ( self.url, role_id ) )
        check_str = "Deleted 1 roles:  %s" % role_name
        self.check_page_for_string( check_str )
        self.home()
    def undelete_role( self, role_id, role_name ):
        """Undelete an existing role"""
        self.home()
        self.visit_url( "%s/admin/roles?operation=undelete&id=%s" % ( self.url, role_id ) )
        check_str = "Undeleted 1 roles:  %s" % role_name
        self.check_page_for_string( check_str )
        self.home()
    def purge_role( self, role_id, role_name ):
        """Purge an existing role"""
        self.home()
        self.visit_url( "%s/admin/roles?operation=purge&id=%s" % ( self.url, role_id ) )
        check_str = "Purged 1 roles:  %s" % role_name
        self.check_page_for_string( check_str )
        self.home()
    def associate_users_and_groups_with_role( self, role_id, role_name, user_ids=[], group_ids=[] ):
        self.home()
        url = "%s/admin/role?id=%s&role_members_edit_button=Save" % ( self.url, role_id )
        if user_ids:
            url += "&in_users=%s" % ','.join( user_ids )
        if group_ids:
            url += "&in_groups=%s" % ','.join( group_ids )
        self.visit_url( url )
        check_str = "Role '%s' has been updated with %d associated users and %d associated groups" % ( role_name, len( user_ids ), len( group_ids ) )
        self.check_page_for_string( check_str )
        self.home()

    # Tests associated with groups
    def create_group( self, name='Group One', in_user_ids=[], in_role_ids=[], create_role_for_group='', strings_displayed=[] ):
        """Create a new group"""
        url = "%s/admin/groups?operation=create&create_group_button=Save&name=%s" % ( self.url, name.replace( ' ', '+' ) )
        if in_user_ids:
            url += "&in_users=%s" % ','.join( in_user_ids )
        if in_role_ids:
            url += "&in_roles=%s" % ','.join( in_role_ids )
        if create_role_for_group == 'yes':
            url += '&create_role_for_group=yes&create_role_for_group=yes'
        self.home()
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
        self.visit_url( "%s/admin/groups" % self.url )
        self.check_page_for_string( name )
        self.home()
    def browse_groups( self, strings_displayed=[] ):
        self.visit_url( '%s/admin/groups' % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def rename_group( self, group_id, name='Group One Renamed' ):
        """Rename a group"""
        self.home()
        self.visit_url( "%s/admin/groups?operation=rename&id=%s" % ( self.url, group_id ) )
        self.check_page_for_string( 'Change group name' )
        tc.fv( "1", "name", name )
        tc.submit( "rename_group_button" )
        self.home()
    def associate_users_and_roles_with_group( self, group_id, group_name, user_ids=[], role_ids=[] ):
        self.home()
        url = "%s/admin/manage_users_and_roles_for_group?id=%s&group_roles_users_edit_button=Save" % ( self.url, group_id )
        if user_ids:
            url += "&in_users=%s" % ','.join( user_ids )
        if role_ids:
            url += "&in_roles=%s" % ','.join( role_ids )
        self.visit_url( url )
        check_str = "Group '%s' has been updated with %d associated roles and %d associated users" % ( group_name, len( role_ids ), len( user_ids ) )
        self.check_page_for_string( check_str )
        self.home()
    def mark_group_deleted( self, group_id, group_name ):
        """Mark a group as deleted"""
        self.home()
        self.visit_url( "%s/admin/groups?operation=delete&id=%s" % ( self.url, group_id ) )
        check_str = "Deleted 1 groups:  %s" % group_name
        self.check_page_for_string( check_str )
        self.home()
    def undelete_group( self, group_id, group_name ):
        """Undelete an existing group"""
        self.home()
        self.visit_url( "%s/admin/groups?operation=undelete&id=%s" % ( self.url, group_id ) )
        check_str = "Undeleted 1 groups:  %s" % group_name
        self.check_page_for_string( check_str )
        self.home()
    def purge_group( self, group_id, group_name ):
        """Purge an existing group"""
        self.home()
        self.visit_url( "%s/admin/groups?operation=purge&id=%s" % ( self.url, group_id ) )
        check_str = "Purged 1 groups:  %s" % group_name
        self.check_page_for_string( check_str )
        self.home()

    # Form stuff
    def create_form( self, name, description, form_type, field_type='TextField', form_layout_name='',
                     num_fields=1, num_options=0, field_name='1_field_name', strings_displayed=[],
                     strings_displayed_after_submit=[] ):
        """Create a new form definition."""
        self.visit_url( "%s/forms/create_form_definition" % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "name", name )
        tc.fv( "1", "description", description )
        tc.fv( "1", "form_type_select_field", form_type )
        tc.submit( "create_form_button" )
        if form_type == "Sequencing Sample Form":
            tc.submit( "add_layout_grid" )
            tc.fv( "1", "grid_layout0", form_layout_name )
        # if not adding any fields at this time, remove the default empty field
        if num_fields == 0:
            tc.submit( "remove_button" )
        # Add fields to the new form definition
        for index1 in range( num_fields ):
            field_label = 'field_label_%i' % index1
            field_contents = field_type
            field_help_name = 'field_helptext_%i' % index1
            field_help_contents = 'Field %i help' % index1
            field_default = 'field_default_0'
            field_default_contents = '%s default contents' % form_type
            tc.fv( "1", field_label, field_contents )
            tc.fv( "1", field_help_name, field_help_contents )
            if field_type == 'SelectField':
                # SelectField field_type requires a refresh_on_change
                self.refresh_form( 'field_type_0', field_type )
                # Add options so our select list is functional
                if num_options == 0:
                    # Default to 2 options
                    num_options = 2
                for index2 in range( 1, num_options+1 ):
                    tc.submit( "addoption_0" )
                # Add contents to the new options fields
                for index2 in range( num_options ):
                    option_field_name = 'field_0_option_%i' % index2
                    option_field_value = 'Option%i' % index2
                    tc.fv( "1", option_field_name, option_field_value )
            else:
                tc.fv( "1", "field_type_0", field_type )
            tc.fv( "1", 'field_name_0', field_name )
            tc.fv( "1", field_default, field_default_contents )
        # All done... now save
        tc.submit( "save_changes_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def edit_form( self, id, form_type='', new_form_name='', new_form_desc='', field_dicts=[], field_index=0,
                   strings_displayed=[], strings_not_displayed=[], strings_displayed_after_submit=[] ):
        """Edit form details; name and description"""
        self.home()
        self.visit_url( "%s/forms/edit_form_definition?id=%s" % ( self.url, id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        if new_form_name:
            tc.fv( "1", "name", new_form_name )
        if new_form_desc:
            tc.fv( "1", "description", new_form_desc )
        for i, field_dict in enumerate( field_dicts ):
            index = i + field_index
            tc.submit( "add_field_button" )
            field_label = "field_label_%i" % index
            field_label_value = field_dict[ 'label' ]
            field_help = "field_helptext_%i" % index
            field_help_value = field_dict[ 'desc' ]
            field_type = "field_type_%i" % index
            field_type_value = field_dict[ 'type' ]
            field_required = "field_required_%i" % index
            field_required_value = field_dict[ 'required' ]
            field_name = "field_name_%i" % index
            field_name_value = field_dict.get( 'name', '%i_field_name' % index )
            tc.fv( "1", field_label, field_label_value )
            tc.fv( "1", field_help, field_help_value )
            tc.fv( "1", field_required, field_required_value )
            tc.fv( "1", field_name, field_name_value )
            if field_type_value.lower() == 'selectfield':
                # SelectFields require a refresh_on_change
                self.refresh_form( field_type, field_type_value )
                for option_index, option in enumerate( field_dict[ 'selectlist' ] ):
                    tc.submit( "addoption_%i" % index )
                    tc.fv( "1", "field_%i_option_%i" % ( index, option_index ), option )
            else:
                tc.fv( "1", field_type, field_type_value )
        tc.submit( "save_changes_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def view_form( self, id, form_type='', form_name='', form_desc='', form_layout_name='', field_dicts=[] ):
        '''View form details'''
        self.home()
        self.visit_url( "%s/forms/view_latest_form_definition?id=%s" % ( self.url, id ) )
        #self.check_page_for_string( form_type )
        self.check_page_for_string( form_name )
        #self.check_page_for_string( form_desc )
        self.check_page_for_string( form_layout_name )
        for i, field_dict in enumerate( field_dicts ):
            self.check_page_for_string( field_dict[ 'label' ] )
            self.check_page_for_string( field_dict[ 'desc' ] )
            self.check_page_for_string( field_dict[ 'type' ] )
            if field_dict[ 'type' ].lower() == 'selectfield':
                for option_index, option in enumerate( field_dict[ 'selectlist' ] ):
                    self.check_page_for_string( option )
        self.home()
    def mark_form_deleted( self, form_id ):
        """Mark a form_definition as deleted"""
        self.home()
        url = "%s/forms/delete_form_definition?id=%s" % ( self.url, form_id )
        self.visit_url( url )
        check_str = "1 forms have been deleted."
        self.check_page_for_string( check_str )
        self.home()
    
    # External services stuff
    def reload_external_service( self, external_service_type_id, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( '%s/external_service/reload_external_service_types' % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "external_service_type_id", external_service_type_id )
        tc.submit( "reload_external_service_type_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def create_external_service( self, name, description, version, external_service_type_id, field_values={}, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( '%s/external_service/create_external_service' % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "name", name )
        tc.fv( "1", "description", description )
        tc.fv( "1", "version", version )
        self.refresh_form( "external_service_type_id", external_service_type_id )
        for field, value in field_values.items():
            tc.fv( "1", field, value )
        tc.submit( "create_external_service_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def view_external_service( self, external_service_id, strings_displayed=[] ):
        self.visit_url( '%s/external_service/view_external_service?id=%s' % ( self.url, external_service_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def edit_external_service( self, external_service_id, field_values={}, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( '%s/external_service/edit_external_service?id=%s' % ( self.url, external_service_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for field, value in field_values.items():
            tc.fv( "1", field, value )
        tc.submit( "edit_external_service_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    # Sample tracking stuff
    def check_request_grid( self, cntrller, state, deleted=False, strings_displayed=[] ):
        self.visit_url( '%s/%s/browse_requests?sort=create_time&f-state=%s&f-deleted=%s' % \
                        ( self.url, cntrller, state.replace( ' ', '+' ), str( deleted ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def create_request_type( self, name, desc, request_form_id, sample_form_id, states, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.home()
        self.visit_url( "%s/request_type/create_request_type" % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "name", name )
        tc.fv( "1", "desc", desc )
        tc.fv( "1", "request_form_id", request_form_id )
        tc.fv( "1", "sample_form_id", sample_form_id )
        for index, state in enumerate(states):
            tc.fv("1", "state_name_%i" % index, state[0])
            tc.fv("1", "state_desc_%i" % index, state[1])
            tc.submit( "add_state_button" )
        tc.submit( "create_request_type_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def request_type_permissions( self, request_type_id, request_type_name, role_ids_str, permissions_in, permissions_out ):
        # role_ids_str must be a comma-separated string of role ids
        url = "request_type/request_type_permissions?id=%s&update_roles_button=Save" % ( request_type_id )
        for po in permissions_out:
            key = '%s_out' % po
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        for pi in permissions_in:
            key = '%s_in' % pi
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        self.home()
        self.visit_url( "%s/%s" % ( self.url, url ) )
        check_str = "Permissions updated for request type '%s'" % request_type_name
        self.check_page_for_string( check_str )
        self.home()
    def view_request_type( self, request_type_id, request_type_name, sample_states, strings_displayed=[] ):
        '''View request_type details'''
        self.home()
        self.visit_url( "%s/request_type/view_request_type?id=%s" % ( self.url, request_type_id ) )
        self.check_page_for_string( '"%s" request type' % request_type_name )
        for name, desc in sample_states:
            self.check_page_for_string( name )
            self.check_page_for_string( desc )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def create_request( self, cntrller, request_type_id, name, desc, field_value_tuples, other_users_id='',
                        strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( "%s/requests_common/create_request?cntrller=%s" % ( self.url, cntrller ) )
        # The request_type SelectList requires a refresh_on_change
        self.refresh_form( 'request_type_id', request_type_id )
        if cntrller == 'requests_admin' and other_users_id:
            # The admin is creating a request on behalf of another user
            # The user_id SelectField requires a refresh_on_change so that the selected
            # user's addresses will be populated in the AddressField widget
            self.refresh_form( "user_id", other_users_id )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "name", name )
        tc.fv( "1", "desc", desc )
        for index, field_value_tuple in enumerate( field_value_tuples ):
            field_index = index + 1
            field_name, field_value, refresh_on_change = field_value_tuple
            if refresh_on_change:
                # Only the AddressField type has a refresh on change setup on selecting an option
                address_option = field_value[0]
                address_value = field_value[1]
                self.refresh_form( field_name, address_option )
                if address_option == 'new':
                    # handle new address
                    self.check_page_for_string( 'Short address description' )
                    for address_field, value in address_value.items():
                        tc.fv( "1", field_name+'_'+address_field, value )
                else:
                    # existing address
                    tc.fv( "1", field_name, address_value )
            else:
                tc.fv( "1", field_name, field_value )
        tc.submit( "create_request_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def view_request( self, cntrller, request_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
        self.visit_url( "%s/%s/browse_requests?operation=view_request&id=%s" % ( self.url, cntrller, request_id ) )
        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
    def view_request_history( self, cntrller, request_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
        self.visit_url( "%s/requests_common/view_request_history?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
    def view_sample_history( self, cntrller, sample_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
        self.visit_url( "%s/requests_common/view_sample_history?cntrller=%s&sample_id=%s" % ( self.url, cntrller, sample_id ) )
        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
    def view_sample_dataset( self, sample_dataset_id, strings_displayed=[], strings_displayed_count=[], strings_not_displayed=[] ):
        self.visit_url( "%s/requests_admin/manage_datasets?operation=view&id=%s" % ( self.url, sample_dataset_id ) )
        self.check_page( strings_displayed, strings_displayed_count, strings_not_displayed )
    def edit_basic_request_info( self, cntrller, request_id, name, new_name='', new_desc='', new_fields=[],
                                 strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( "%s/requests_common/edit_basic_request_info?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        if new_name:
            tc.fv( "1", "name", new_name )
        if new_desc:
            tc.fv( "1", "desc", new_desc )
        for index, ( field_name, field_value ) in enumerate( new_fields ):
            field_name_index = index + 1
            tc.fv( "1", field_name, field_value )
        tc.submit( "edit_basic_request_info_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def edit_request_email_settings( self, cntrller, request_id, check_request_owner=True, additional_emails='', 
                                     check_sample_states=[], strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( "%s/requests_common/edit_basic_request_info?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "2", "email_address", check_request_owner )
        tc.fv( "2", "additional_email_addresses", additional_emails )
        for state_name, state_id, is_checked in check_sample_states:
            tc.fv( "2", "sample_state_%i" % state_id, is_checked )
        tc.submit( "edit_email_settings_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def add_samples( self, cntrller, request_id, sample_value_tuples, folder_options=[], strings_displayed=[], strings_displayed_after_submit=[] ):
        url = "%s/requests_common/add_sample?cntrller=%s&request_id=%s&add_sample_button=Add+sample" % ( self.url, cntrller, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_index, ( sample_name, target_library_info, sample_field_values ) in enumerate( sample_value_tuples ):
            tc.fv( "1", "sample_%i_name" % sample_index, sample_name )
            tc.fv( "1", "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
            self.refresh_form( "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
            # check if the folder selectfield has been correctly populated
            for check_str in folder_options:
                self.check_page_for_string( check_str )
            tc.fv( "1", "sample_%i_folder_id" % sample_index, target_library_info[ 'folder' ] )
            for field_index, field_value in enumerate( sample_field_values ):
                tc.fv( "1", "sample_%i_field_%i" % ( sample_index, field_index ), field_value )
            # Do not click on Add sample button when all the sample have been added 
            if sample_index < len( sample_value_tuples ) - 1:
                tc.submit( "add_sample_button" )
        # select the correct form before submitting it
        tc.fv( "1", "copy_sample_index", "-1" )
        tc.submit( "save_samples_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def edit_samples( self, cntrller, request_id, sample_value_tuples, strings_displayed=[], strings_displayed_after_submit=[] ):
        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_index, ( sample_name, target_library_info, sample_field_values ) in enumerate( sample_value_tuples ):
            tc.fv( "1", "sample_%i_name" % sample_index, sample_name )
            tc.fv( "1", "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
            self.refresh_form( "sample_%i_library_id" % sample_index, target_library_info[ 'library' ] )
            tc.fv( "1", "sample_%i_folder_id" % sample_index, target_library_info[ 'folder' ] )
            for field_index, field_value in enumerate( sample_field_values ):
                tc.fv( "1", "sample_%i_field_%i" % ( sample_index, field_index ), field_value )
        tc.submit( "save_samples_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def add_bar_codes( self, cntrller, request_id, bar_codes, strings_displayed=[], strings_displayed_after_submit=[] ):
        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_index, bar_code in enumerate( bar_codes ):
            tc.fv( "1", "sample_%i_bar_code" % sample_index, bar_code )
        tc.submit( "save_samples_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def submit_request( self, cntrller, request_id, request_name, strings_displayed_after_submit=[] ):
        self.visit_url( "%s/requests_common/submit_request?cntrller=%s&id=%s" % ( self.url, cntrller, request_id ) )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def reject_request( self, request_id, request_name, comment, strings_displayed=[], strings_displayed_after_submit=[] ):
        self.visit_url( "%s/requests_admin/reject_request?id=%s" % ( self.url, request_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.fv( "1", "comment", comment )
        tc.submit( "reject_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def change_sample_state( self, request_id, sample_ids, new_sample_state_id, comment='', strings_displayed=[], strings_displayed_after_submit=[] ):
        url = "%s/requests_common/edit_samples?cntrller=requests_admin&id=%s" % ( self.url, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_id in sample_ids:
            tc.fv( "1", "select_sample_%i" % sample_id, True )
        tc.fv( "1", "sample_operation", 'Change state' )
        # refresh on change to show the sample states selectfield
        self.refresh_form( "sample_operation", 'Change state' )
        self.check_page_for_string( "Change current state" )
        tc.fv( "1", "sample_state_id", new_sample_state_id )
        tc.fv( "1", "sample_event_comment", comment )
        tc.submit( "save_samples_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def change_sample_target_data_library( self, cntrller, request_id, sample_ids, new_library_id, new_folder_id, folder_options=[], comment='', strings_displayed=[], strings_displayed_after_submit=[] ):
        url = "%s/requests_common/edit_samples?cntrller=%s&id=%s" % ( self.url, cntrller, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_id in sample_ids:
            tc.fv( "1", "select_sample_%i" % sample_id, True )
        tc.fv( "1", "sample_operation", 'Select data library and folder' )
        # refresh on change to show the data libraries selectfield
        self.refresh_form( "sample_operation", 'Select data library and folder' )
        self.check_page_for_string( "Select data library:" )
        tc.fv( "1", "sample_operation_library_id", new_library_id )
        # refresh on change to show the selectfield with the list of 
        # folders in the selected data library above
        self.refresh_form( "sample_operation_library_id", new_library_id )
        self.check_page_for_string( "Select folder:" )
        # check if the folder selectfield has been correctly populated
        for check_str in folder_options:
            self.check_page_for_string( check_str )
        tc.fv( "1", "sample_operation_folder_id", new_folder_id )
        tc.submit( "save_samples_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def add_datasets_to_sample( self, request_id, sample_id, sample_datasets, strings_displayed=[], strings_displayed_after_submit=[] ):
        # visit the dataset selection page
        url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s" % ( self.url, sample_id, request_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        # Datasets are associated with the given by the building the appropriate url
        # and calling it as the dataset selection UI is a javascript dynatree
        url = "%s/requests_admin/select_datasets_to_transfer?cntrller=requests_admin&sample_id=%s&request_id=%s" % ( self.url, sample_id, request_id )
        url += '&select_datasets_to_transfer_button=Select%20datasets'
        url += '&selected_datasets_to_transfer=%s' % ','.join( sample_datasets )
        self.visit_url( url )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def rename_sample_datasets( self, sample_id, sample_dataset_ids, new_sample_dataset_names, strings_displayed=[], strings_displayed_after_submit=[] ):
        sample_dataset_ids_string = ','.join( sample_dataset_ids )
        url = "%s/requests_admin/manage_datasets?operation=rename&id=%s" % ( self.url, sample_dataset_ids_string )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for sample_dataset_id, ( prefix, new_name ) in zip( sample_dataset_ids, new_sample_dataset_names ):
            tc.fv( "1", 'rename_datasets_for_sample_%s' % sample_dataset_id, prefix )
            tc.fv( "1", 'new_name_%s' % sample_dataset_id, new_name )
        tc.submit( "rename_datasets_button" )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
    def delete_sample_datasets( self, sample_id, sample_dataset_ids, strings_displayed=[], strings_displayed_after_submit=[], strings_not_displayed=[] ):
        url = '%s/requests_admin/manage_datasets?cntrller=requests_admin&sample_id=%s' % ( self.url, sample_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        # simulate selecting datasets and clicking the delete button on the sample datasets grid
        sample_dataset_ids_string = ','.join( sample_dataset_ids )
        url = "%s/requests_admin/manage_datasets?operation=delete&id=%s" % ( self.url, sample_dataset_ids_string )
        self.visit_url( url )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            self.check_string_not_in_page( check_str )
    def start_sample_datasets_transfer( self, sample_id, sample_dataset_ids, strings_displayed=[], strings_displayed_after_submit=[], strings_displayed_count=[], strings_not_displayed=[] ):
        url = '%s/requests_admin/manage_datasets?cntrller=requests_admin&sample_id=%s' % ( self.url, sample_id )
        self.visit_url( url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        # simulate selecting datasets and clicking the transfer button on the sample datasets grid
        sample_dataset_ids_string = ','.join( sample_dataset_ids )
        url = "%s/requests_admin/manage_datasets?operation=transfer&id=%s" % ( self.url, sample_dataset_ids_string )
        self.visit_url( url )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            self.check_string_not_in_page( check_str )
        for check_str, count in strings_displayed_count:
            self.check_string_count_in_page( check_str, count )
    def add_user_address( self, user_id, address_dict ):
        self.home()
        self.visit_url( "%s/user/new_address?cntrller=user&user_id=%s" % ( self.url, user_id ) )
        self.check_page_for_string( 'Add new address' )
        for field_name, value in address_dict.items():
            tc.fv( "1", field_name, value )
        tc.submit( "new_address_button" )
        self.check_page_for_string( 'Address (%s) has been added' % address_dict[ 'short_desc' ] )
        
    # Library stuff
    def add_template( self, cntrller, item_type, form_type, form_id, form_name,
                      library_id=None, folder_id=None, ldda_id=None, request_type_id=None, sample_id=None ):
        """
        Add a new template to an item - for library items, the template will ALWAYS BE SET TO INHERITABLE here.  If you want to
        dis-inherit your template, call the manage_library_template_inheritance() below immediately after you call this
        method in your test code.  Templates added to Requesttype objects are always inherited to samples.
        """
        self.home()
        if item_type == 'library':
            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s" % \
            ( self.url, cntrller, item_type, form_type, library_id )
        elif item_type == 'folder':
            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s&folder_id=%s" % \
            ( self.url, cntrller, item_type, form_type, library_id, folder_id )
        elif item_type == 'ldda':
            url = "%s/library_common/add_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s&folder_id=%s&ldda_id=%s" % \
            ( self.url, cntrller, item_type, form_type, library_id, folder_id, ldda_id )
        self.visit_url( url )
        self.check_page_for_string ( "Select a template for the" )
        self.refresh_form( "form_id", form_id )
        # For some unknown reason, twill barfs if the form number ( 1 ) is used in the following
        # rather than the form anme ( select_template ), so we have to use the form name.
        tc.fv( "select_template", "inheritable", '1' )
        tc.submit( "add_template_button" )
        self.check_page_for_string = 'A template based on the form "%s" has been added to this' % form_name
        self.home()
    def manage_library_template_inheritance( self, cntrller, item_type, library_id, folder_id=None, ldda_id=None, inheritable=True ):
        # If inheritable is True, the item is currently inheritable.
        self.home()
        if item_type == 'library':
            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s" % \
            ( self.url, cntrller, item_type, library_id )
        elif item_type == 'folder':
            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s" % \
            ( self.url, cntrller, item_type, library_id, folder_id )
        elif item_type == 'ldda':
            url = "%s/library_common/manage_template_inheritance?cntrller=%s&item_type=%s&library_id=%s&folder_id=%s&ldda_id=%s" % \
            ( self.url, cntrller, item_type, library_id, folder_id, ldda_id )
        self.visit_url( url )
        if inheritable:
            self.check_page_for_string = 'will no longer be inherited to contained folders and datasets'
        else:
            self.check_page_for_string = 'will now be inherited to contained folders and datasets'
        self.home()
    def browse_libraries_admin( self, deleted=False, strings_displayed=[], strings_not_displayed=[] ):
        self.visit_url( '%s/library_admin/browse_libraries?sort=name&f-description=All&f-name=All&f-deleted=%s' % ( self.url, str( deleted ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed when browing library." % check_str
            except:
                pass
    def browse_libraries_regular_user( self, strings_displayed=[], strings_not_displayed=[] ):
        self.visit_url( '%s/library/browse_libraries' % self.url )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed when browing library." % check_str
            except:
                pass
    def browse_library( self, cntrller, library_id, show_deleted=False, strings_displayed=[], strings_not_displayed=[] ):
        self.visit_url( '%s/library_common/browse_library?cntrller=%s&id=%s&show_deleted=%s' % ( self.url, cntrller, library_id, str( show_deleted ) ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed when browing library." % check_str
            except:
                pass
    def create_library( self, name='Library One', description='This is Library One', synopsis='Synopsis for Library One' ):
        """Create a new library"""
        self.visit_url( "%s/library_admin/create_library" % self.url )
        self.check_page_for_string( 'Create a new data library' )
        tc.fv( "1", "name", name )
        tc.fv( "1", "description", description )
        tc.fv( "1", "synopsis", synopsis )
        tc.submit( "create_library_button" )
        check_str = "The new library named '%s' has been created" % name
        self.check_page_for_string( check_str )
        self.home()
    def edit_template( self, cntrller, item_type, form_type, library_id, field_type, field_label_1, field_helptext_1, field_default_1,
                       folder_id='', ldda_id='', action='add_field'  ):
        """Edit the form fields defining a library template"""
        self.visit_url( "%s/library_common/edit_template?cntrller=%s&item_type=%s&form_type=%s&library_id=%s" % \
                        ( self.url, cntrller, item_type, form_type, library_id ) )
        self.check_page_for_string( "Edit form definition" )
        if action == 'add_field':
            tc.submit( "add_field_button" )
            tc.fv( "edit_form", "field_label_1", field_label_1 )
            tc.fv( "edit_form", "field_helptext_1", field_helptext_1 )
            if field_type == 'SelectField':
                # Performs a refresh_on_change in this case
                self.refresh_form( "field_type_1", field_type )
            else:
                tc.fv( "edit_form", "field_type_1", field_type )
            tc.fv( "edit_form", "field_default_1", field_default_1 )
        tc.submit( 'save_changes_button' )
        self.check_page_for_string( "The template for this data library has been updated with your changes." )
    def library_info( self, cntrller, library_id, library_name='', new_name='', new_description='', new_synopsis='', 
                      template_fields=[], strings_displayed=[] ):
        """Edit information about a library, optionally using an existing template with up to 2 elements"""
        self.visit_url( "%s/library_common/library_info?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        if new_name and new_description and new_synopsis:
            tc.fv( '1', 'name', new_name )
            tc.fv( '1', 'description', new_description )
            tc.fv( '1', 'synopsis', new_synopsis )
            tc.submit( 'library_info_button' )
            self.check_page_for_string( "Information updated for library" )
        if template_fields:
            for field_name, field_value in template_fields:
                # The 2nd form on the page contains the template, and the form is named edit_info.
                # Set the template field value
                tc.fv( "edit_info", field_name, field_value )
            tc.submit( 'edit_info_button' )
        self.home()
    def library_permissions( self, library_id, library_name, role_ids_str, permissions_in, permissions_out, cntrller='library_admin' ):
        # role_ids_str must be a comma-separated string of role ids
        url = "library_common/library_permissions?id=%s&cntrller=%s&update_roles_button=Save" % ( library_id, cntrller )
        for po in permissions_out:
            key = '%s_out' % po
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        for pi in permissions_in:
            key = '%s_in' % pi
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        self.home()
        self.visit_url( "%s/%s" % ( self.url, url ) )
        check_str = "Permissions updated for library '%s'." % library_name
        self.check_page_for_string( check_str )
        self.home()
    def make_library_item_public( self, library_id, id, cntrller='library_admin', item_type='library',
                                  contents=False, library_name='', folder_name='', ldda_name='' ):
        url = "%s/library_common/make_library_item_public?cntrller=%s&library_id=%s&item_type=%s&id=%s&contents=%s" % \
            ( self.url, cntrller, library_id, item_type, id, str( contents ) )
        self.visit_url( url )
        if item_type == 'library':
            if contents:
                check_str = "The data library (%s) and all it's contents have been made publicly accessible." % library_name
            else:
                check_str = "The data library (%s) has been made publicly accessible, but access to it's contents has been left unchanged." % library_name
        elif item_type == 'folder':
            check_str = "All of the contents of folder (%s) have been made publicly accessible." % folder_name
        elif item_type == 'ldda':
            check_str = "The libary dataset (%s) has been made publicly accessible." % ldda_name
        self.check_page_for_string( check_str )

    # Library folder stuff
    def add_folder( self, cntrller, library_id, folder_id, name='Folder One', description='This is Folder One' ):
        """Create a new folder"""
        url = "%s/library_common/create_folder?cntrller=%s&library_id=%s&parent_id=%s" % ( self.url, cntrller, library_id, folder_id )
        self.visit_url( url )
        self.check_page_for_string( 'Create a new folder' )
        tc.fv( "1", "name", name )
        tc.fv( "1", "description", description )
        tc.submit( "new_folder_button" )
        check_str = "The new folder named '%s' has been added to the data library." % name
        self.check_page_for_string( check_str )
        self.home()
    def folder_info( self, cntrller, folder_id, library_id, name='', new_name='', description='', template_refresh_field_name='1_field_name',
                     template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[],
                     strings_displayed_after_submit=[], strings_not_displayed_after_submit=[] ):
        """Add information to a library using an existing template with 2 elements"""
        self.visit_url( "%s/library_common/folder_info?cntrller=%s&id=%s&library_id=%s" % \
                        ( self.url, cntrller, folder_id, library_id ) )
        if name and new_name and description:
            tc.fv( '1', "name", new_name )
            tc.fv( '1', "description", description )
            tc.submit( 'rename_folder_button' )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed." % check_str
            except:
                pass
        if template_refresh_field_contents:
            # A template containing an AddressField is displayed on the form, so we need to refresh the form 
            # with the received template_refresh_field_contents.  There are 2 forms on the folder_info page
            # when in edit mode, and the 2nd one is the one we want.
            self.refresh_form( template_refresh_field_name, template_refresh_field_contents, form_no=2 )
        if template_fields:
            # We have an information template associated with the folder, so
            # there are 2 forms on this page and the template is the 2nd form
            for field_name, field_value in template_fields:
                tc.fv( "edit_info", field_name, field_value )
            tc.submit( 'edit_info_button' )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed_after_submit:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) incorrectly displayed." % check_str
            except:
                pass
        self.home()

    # Library dataset stuff
    def upload_library_dataset( self, cntrller, library_id, folder_id, filename='', server_dir='', replace_id='',
                                upload_option='upload_file', file_type='auto', dbkey='hg18', space_to_tab='',
                                link_data_only='copy_files', preserve_dirs='Yes', filesystem_paths='', roles=[],
                                ldda_message='', hda_ids='', template_refresh_field_name='1_field_name',
                                template_refresh_field_contents='', template_fields=[], show_deleted='False', strings_displayed=[] ):
        """Add datasets to library using any upload_option"""
        # NOTE: due to the library_wait() method call at the end of this method, no tests should be done
        # for strings_displayed_after_submit.
        url = "%s/library_common/upload_library_dataset?cntrller=%s&library_id=%s&folder_id=%s" % \
            ( self.url, cntrller, library_id, folder_id )
        if replace_id:
            # If we're uploading a new version of a library dataset, we have to include the replace_id param in the
            # request because the form field named replace_id will not be displayed on the upload form if we dont.
            url += "&replace_id=%s" % replace_id
        self.visit_url( url )
        if template_refresh_field_contents:
            # A template containing an AddressField is displayed on the upload form, so we need to refresh the form 
            # with the received template_refresh_field_contents.
            self.refresh_form( template_refresh_field_name, template_refresh_field_contents )
        for tup in template_fields:
            tc.fv( "1", tup[0], tup[1] )
        tc.fv( "1", "library_id", library_id )
        tc.fv( "1", "folder_id", folder_id )
        tc.fv( "1", "show_deleted", show_deleted )
        tc.fv( "1", "ldda_message", ldda_message )
        tc.fv( "1", "file_type", file_type )
        tc.fv( "1", "dbkey", dbkey )
        if space_to_tab:
            tc.fv( "1", "space_to_tab", space_to_tab )
        for role_id in roles:
            tc.fv( "1", "roles", role_id )
        # Refresh the form by selecting the upload_option - we do this here to ensure
        # all previously entered form contents are retained.
        self.refresh_form( 'upload_option', upload_option )
        if upload_option == 'import_from_history':
            for check_str in strings_displayed:
                self.check_page_for_string( check_str )
            if hda_ids:
                # Twill cannot handle multi-checkboxes, so the form can only have 1 hda_ids checkbox
                try:
                    tc.fv( "add_history_datasets_to_library", "hda_ids", hda_ids )
                except:
                    tc.fv( "add_history_datasets_to_library", "hda_ids", '1' )
            tc.submit( 'add_history_datasets_to_library_button' )
        else:
            if upload_option in [ 'upload_paths', 'upload_directory' ]:
                tc.fv( "1", "link_data_only", link_data_only )
            if upload_option == 'upload_paths':
                tc.fv( "1", "filesystem_paths", filesystem_paths )
            if upload_option == 'upload_directory' and server_dir:
                tc.fv( "1", "server_dir", server_dir )
            if upload_option == 'upload_file':
                if filename:
                    filename = self.get_filename( filename )
                    tc.formfile( "1", "files_0|file_data", filename )
            for check_str in strings_displayed:
                self.check_page_for_string( check_str )
            tc.submit( "runtool_btn" )
        # Give the files some time to finish uploading
        self.library_wait( library_id )
        self.home()
    def ldda_permissions( self, cntrller, library_id, folder_id, id, role_ids_str,
                          permissions_in=[], permissions_out=[], strings_displayed=[], ldda_name='' ):
        # role_ids_str must be a comma-separated string of role ids
        url = "%s/library_common/ldda_permissions?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
            ( self.url, cntrller, library_id, folder_id, id )
        for po in permissions_out:
            key = '%s_out' % po
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        for pi in permissions_in:
            key = '%s_in' % pi
            url ="%s&%s=%s" % ( url, key, role_ids_str )
        if permissions_in or permissions_out:
            url += "&update_roles_button=Save"
            self.visit_url( url )
        if not strings_displayed:
            strings_displayed = [ "Permissions updated for dataset '%s'." % ldda_name ]
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        self.home()
    def ldda_info( self, cntrller, library_id, folder_id, ldda_id, strings_displayed=[], strings_not_displayed=[] ):
        """View library_dataset_dataset_association information"""
        self.visit_url( "%s/library_common/ldda_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
                        ( self.url, cntrller, library_id, folder_id, ldda_id ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) should not have been displayed on ldda info page." % check_str
            except:
                pass
        self.home()
    def ldda_edit_info( self, cntrller, library_id, folder_id, ldda_id, ldda_name, new_ldda_name='', template_refresh_field_name='1_field_name',
                        template_refresh_field_contents='', template_fields=[], strings_displayed=[], strings_not_displayed=[] ):
        """Edit library_dataset_dataset_association information, optionally template element information"""
        self.visit_url( "%s/library_common/ldda_edit_info?cntrller=%s&library_id=%s&folder_id=%s&id=%s" % \
                        ( self.url, cntrller, library_id, folder_id, ldda_id ) )        
        check_str = 'Edit attributes of %s' % ldda_name
        self.check_page_for_string( check_str )
        if new_ldda_name:
            tc.fv( '1', 'name', new_ldda_name )
            tc.submit( 'save' )
            check_str = "Attributes updated for library dataset '%s'." % new_ldda_name
            self.check_page_for_string( check_str )
        if template_refresh_field_contents:
            # A template containing an AddressField is displayed on the upload form, so we need to refresh the form 
            # with the received template_refresh_field_contents.  There are 4 forms on this page, and the template is
            # contained in the 4th form named "edit_info".
            self.refresh_form( template_refresh_field_name, template_refresh_field_contents, form_no=4 )
        if template_fields:
            # We have an information template associated with the folder, so
            # there are 2 forms on this page and the template is the 2nd form
            for field_name, field_value in template_fields:
                tc.fv( "edit_info", field_name, field_value )
            tc.submit( 'edit_info_button' )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        for check_str in strings_not_displayed:
            try:
                self.check_page_for_string( check_str )
                raise AssertionError, "String (%s) should not have been displayed on ldda Edit Attributes page." % check_str
            except:
                pass
        self.home()
    def act_on_multiple_datasets( self, cntrller, library_id, do_action, ldda_ids='', strings_displayed=[] ):
        # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission
        # by going directly to the form action
        self.visit_url( '%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&ldda_ids=%s&do_action=%s' \
                        % ( self.url, cntrller, library_id, ldda_ids, do_action ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def import_datasets_to_histories( self, cntrller, library_id, ldda_ids='', new_history_name='Unnamed history', strings_displayed=[] ):
        # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission
        # by going directly to the form action
        self.visit_url( '%s/library_common/import_datasets_to_histories?cntrller=%s&library_id=%s&ldda_ids=%s&new_history_name=%s&import_datasets_to_histories_button=Import+library+datasets' \
                        % ( self.url, cntrller, library_id, ldda_ids, new_history_name ) )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
    def download_archive_of_library_files( self, cntrller, library_id, ldda_ids, format ):
        self.home()
        # Here it would be ideal to have twill set form values and submit the form, but
        # twill barfs on that due to the recently introduced page wrappers around the contents
        # of the browse_library.mako template which enable panel layout when visiting the
        # page from an external URL.  By "barfs", I mean that twill somehow loses hod on the 
        # cntrller param.  We'll just simulate the form submission by building the URL manually.
        # Here's the old, better approach...
        #self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
        #for ldda_id in ldda_ids:
        #    tc.fv( "1", "ldda_ids", ldda_id )
        #tc.fv( "1", "do_action", format )
        #tc.submit( "action_on_datasets_button" )
        # Here's the new approach...
        url = "%s/library_common/act_on_multiple_datasets?cntrller=%s&library_id=%s&do_action=%s" \
            % ( self.url, cntrller, library_id, format )
        for ldda_id in ldda_ids:
            url += "&ldda_ids=%s" % ldda_id
        self.visit_url( url )
        tc.code( 200 )
        archive = self.write_temp_file( self.last_page(), suffix='.' + format )
        self.home()
        return archive
    def check_archive_contents( self, archive, lddas ):
        def get_ldda_path( ldda ):
            path = ""
            parent_folder = ldda.library_dataset.folder
            while parent_folder is not None:
                if parent_folder.parent is None:
                    path = os.path.join( parent_folder.library_root[0].name, path )
                    break
                path = os.path.join( parent_folder.name, path )
                parent_folder = parent_folder.parent
            path += ldda.name
            return path
        def mkdir( file ):
            dir = os.path.join( tmpd, os.path.dirname( file ) )
            if not os.path.exists( dir ):
                os.makedirs( dir )
        tmpd = tempfile.mkdtemp()
        if tarfile.is_tarfile( archive ):
            t = tarfile.open( archive )
            for n in t.getnames():
                mkdir( n )
                t.extract( n, tmpd )
            t.close()
        elif zipfile.is_zipfile( archive ):
            z = zipfile.ZipFile( archive, 'r' )
            for n in z.namelist():
                mkdir( n )
                open( os.path.join( tmpd, n ), 'wb' ).write( z.read( n ) )
            z.close()
        else:
            raise Exception( 'Unable to read archive: %s' % archive )
        for ldda in lddas:
            orig_file = self.get_filename( ldda.name )
            downloaded_file = os.path.join( tmpd, get_ldda_path( ldda ) )
            assert os.path.exists( downloaded_file )
            try:
                self.files_diff( orig_file, downloaded_file )
            except AssertionError, err:
                errmsg = 'Library item %s different than expected, difference:\n' % ldda.name
                errmsg += str( err )
                errmsg += 'Unpacked archive remains in: %s\n' % tmpd
                raise AssertionError( errmsg )
        shutil.rmtree( tmpd )
    def move_library_item( self, cntrller, item_type, item_id, source_library_id, make_target_current,
                           target_library_id='', target_folder_id='', strings_displayed=[], strings_displayed_after_submit=[] ):
        self.home()
        self.visit_url( "%s/library_common/move_library_item?cntrller=%s&item_type=%s&item_id=%s&source_library_id=%s&make_target_current=%s" \
                        % ( self.url, cntrller, item_type, item_id, source_library_id, make_target_current ) )
        if target_library_id:
            self.refresh_form( 'target_library_id', target_library_id )
        if target_folder_id:
            tc.fv( '1', 'target_folder_id', target_folder_id )
        for check_str in strings_displayed:
            self.check_page_for_string( check_str )
        tc.submit( 'move_library_item_button' )
        for check_str in strings_displayed_after_submit:
            self.check_page_for_string( check_str )
        self.home()
    def delete_library_item( self, cntrller, library_id, item_id, item_name, item_type='library_dataset' ):
        """Mark a library item as deleted"""
        self.home()
        self.visit_url( "%s/library_common/delete_library_item?cntrller=%s&library_id=%s&item_id=%s&item_type=%s" \
                        % ( self.url, cntrller, library_id, item_id, item_type ) )
        if item_type == 'library_dataset':
            item_desc = 'Dataset'
        else:
            item_desc = item_type.capitalize()
        check_str = "%s '%s' has been marked deleted" % ( item_desc, item_name )
        self.check_page_for_string( check_str )
        self.home()
    def undelete_library_item( self, cntrller, library_id, item_id, item_name, item_type='library_dataset' ):
        """Mark a library item as deleted"""
        self.home()
        self.visit_url( "%s/library_common/undelete_library_item?cntrller=%s&library_id=%s&item_id=%s&item_type=%s" \
                        % ( self.url, cntrller, library_id, item_id, item_type ) )
        if item_type == 'library_dataset':
            item_desc = 'Dataset'
        else:
            item_desc = item_type.capitalize()
        check_str = "%s '%s' has been marked undeleted" % ( item_desc, item_name )
        self.check_page_for_string( check_str )
        self.home()
    def purge_library( self, library_id, library_name ):
        """Purge a library"""
        self.home()
        self.visit_url( "%s/library_admin/purge_library?id=%s" % ( self.url, library_id ) )
        check_str = "Library '%s' and all of its contents have been purged" % library_name
        self.check_page_for_string( check_str )
        self.home()
    def library_wait( self, library_id, cntrller='library_admin', maxiter=90 ):
        """Waits for the tools to finish"""
        count = 0
        sleep_amount = 1
        while count < maxiter:
            count += 1
            self.visit_url( "%s/library_common/browse_library?cntrller=%s&id=%s" % ( self.url, cntrller, library_id ) )
            page = tc.browser.get_html()
            if page.find( '<!-- running: do not change this comment, used by TwillTestCase.library_wait -->' ) > -1:
                time.sleep( sleep_amount )
                sleep_amount += 1
            else:
                break
        self.assertNotEqual(count, maxiter)

    # Tests associated with tags
    def add_tag( self, item_id, item_class, context, new_tag ):
        self.visit_url( "%s/tag/add_tag_async?item_id=%s&item_class=%s&context=%s&new_tag=%s" % \
                        ( self.url, item_id, item_class, context, new_tag ) )