Source

mct / mct_computer_admin / src / mct_computer_admin / cluster_control_fab.py

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
#!/usr/bin/env python
from __future__ import print_function
import roslib
roslib.load_manifest('mct_computer_admin')
import rospy

import sys
import time
import os.path
import tempfile
import subprocess
import mct_xml_tools
import mct_introspection
import mct_utilities
from mct_frame_drop_corrector import frame_drop_corrector

from fabric.api import *
from fabric.decorators import hosts
from fabric.contrib.files import exists

slave_list = mct_introspection.get_slave_hosts()
host_list = mct_introspection.get_hosts()
master = mct_introspection.get_master()

cmd_msgs = {
        'wakeup': 'wake up camera computers',
        'push_setup': 'push setup to slave computers',
        'shutdown': 'shut down camera computers',
        'list_slaves': 'listing slaves computers',
        'rospack_profile': 'run rospack profile on all slave computers', 
        'pull': 'pull latest version of the mct from repository to slave computers',
        'pull_master': 'pull lastest version of the mct from repository on master',
        'pull_all': 'pull latest version of the mct repository for all computers in current machine',
        'pull_from_master': 'pull latest version of the mct repository from the master to the slaves', 
        'clone': 'clone new version of mct from repository of slave computers',
        'clean': 'remove old version of mct',
        'rosmake': 'use rosmake to build mct on slave computers',
        'rosmake_preclean': 'use rosmake to build mct on slave computers',
        'update_machine_def': 'update the ROS xml machine launch file',
        'list_machine_def': 'list current machine definition',
        'list_cameras': 'list cameras',
        'list_camera_assignment': 'list camera assignment', 
        'rsync_camera_calibrations': 'rsync camera calibraitons',
        'clean_camera_calibrations': 'clean camera calibrations',
        'show_camera_info':  'launch camera_info viewers',
        'show_camera_info_header': 'launch camera_info/header viewers',
        'show_camera_info_header_seq': 'launch camera_info/header/seq viewers',
        'show_corrector_seq': 'launch seq_and_image_corr/seq viewers',
        'test': 'test command for development',
        'camera_assignment': 'start camera assigment application',
        'zoom_calibration': 'start zoom calibration application',
        'camera_calibration': 'start the camera calibration application',
        'homography_calibration': 'start the homography calibration application',
        'transform_2d_calibration': 'start the 2d transform calibration application',
        'tracking_2d': 'start the 2d tracking application',
        'frame_drop_test':  '2d tracking application w/ frame dropped frame logger',
        'frames_dropped': 'print a report of the frames dropped by the system',
        'help': 'Command line interface to MCT (Multi-Camera Tracker)',
        'list_all': 'list all available commands',
        }

fab_cmds = [ 
        'wakeup', 
        'push_setup', 
        'shutdown', 
        'list_slaves', 
        'rospack_profile', 
        'pull', 
        'pull_master',
        'pull_all',
        'pull_from_master',
        'clone',
        'clean',
        'rosmake',
        'rosmake_preclean',
        'clean_camera_calibrations',
        ]

def help():
    print('Usage: mct [command]')
    print()
    print('Commands:')
    print()
    print('  Basic Administration')
    print('  --------------------')
    print('    wakeup               - wake up slave computers')
    print('    shutdown             - shutdown slave computers ')
    print('    update_machine_def   - updates the machine definition file')
    print()
    print('  Calibration (in order of application)')
    print('  -------------------------------------')
    print('    camera_assignment          - run the camera assigment application')
    print('    zoom_calibration           - run the camera zoom calibration application')
    print('    camera_calibration         - run the camera calibration applicaiton')
    print('    homography_calibration     - run the homography calibration application')
    print('    transform_2d_calibration   - run the 2d transform calibration application')
    print()
    print('  Tracking')
    print('  --------')
    print('    tracking_2d   - 2D tracking applicaton')
    print()
    print('  Testing')
    print('  -------')
    print('    frame_drop_test - 2D tracking application w/ dropped frame logger')
    print('    frames_dropped  - prints a report listing the frames dropped by each camera')
    print()
    print('  Help')
    print('  ----')
    print('    help      - prints this menu')
    print('    list_all  - prints a list and description of all commands')
    print()

def list_all():
    """
    Prints list of all available commands.
    """
    max_len = max([len(k) for k in cmd_msgs])
    for k,v in sorted(cmd_msgs.items()):
        n = max_len - len(k)
        print('  ', k,' '*n, v)
    print()


def camera_assignment():
    """
    Starts the camera assignment application.
    """
    roslaunch('camera_assignment.launch')

def zoom_calibration():
    """
    Starts the zoom calibration tool.
    """
    roslaunch('zoom_calibration.launch')

def camera_calibration():
    """
    Starts the camera calibration application
    """
    roslaunch('camera_calibration.launch')

def homography_calibration():
    """
    Starts the homography calibration application
    """
    roslaunch('homography_calibration.launch')

def transform_2d_calibration():
    """
    Starts the 2d transformation calibration application
    """
    roslaunch('transform_2d_calibration.launch')

def tracking_2d():
    """
    Starts the tracking 2d application
    """
    roslaunch('tracking_2d.launch')

def frame_drop_test():
    """
    Starts the frame drop test application
    """
    roslaunch('frame_drop_test.launch')

def roslaunch(launch_file):
    """
    Runs a roslaunch file.
    """
    try:
        subprocess.call(['roslaunch', 'mct_launch', launch_file])
    except KeyboardInterrupt:
        return

def wakeup():
    """
    Wakes up camera computers using wake on lan. 
    """
    mac_and_iface_list = mct_introspection.get_slave_mac_and_iface()
    for mac, iface in mac_and_iface_list:
        local('sudo etherwake -i {0} {1}'.format(iface,mac))

@hosts(*slave_list)
def shutdown(): 
    """
    Shutdown camera computers
    """
    sudo('halt',shell=False)

@hosts(*slave_list)
def push_setup():
    """
    Pushes mct_setup.bash file to the slave computers
    """
    bin_dir = os.path.join(os.environ['HOME'],'bin')
    setup_file = os.path.join(bin_dir,'mct_setup.bash')
    if not exists(bin_dir):
        run('mkdir {0}'.format(bin_dir))
    with cd(bin_dir):
        put(setup_file,setup_file)

def _pull():
    """
    Pull lastest version of mct from the repository.
    """
    mct_name = os.environ['MCT_NAME']
    if exists(mct_name):
        with cd(mct_name):
            run('hg pull -u')
    else:
        print('ERROR: unable to pull and update {0} does not exits'.format(mct_name))

@hosts(*slave_list)
def pull():
    """
    Pull lastest version of mct from the repository on slave computers only.
    """
    _pull()

@hosts(master)
def pull_master():
    """
    Pull latest version of mct from repository on master computer only.
    """
    _pull()

@hosts(*host_list)
def pull_all():
    """
    Pull latest version for mct repository on all computers in machine
    """
    _pull()

@hosts(*slave_list)
def pull_from_master():
    """
    Pull the latest version of the mct repository from the master computer.
    """
    mct_name = os.environ['MCT_NAME']
    mct_name_split = mct_name.split('/')
    mct_local = '/'.join(mct_name_split[-2:])
    machine_def = mct_introspection.get_machine_def()
    user = machine_def['user']
    address = machine_def['mct_master']['address']
    cmd = 'hg pull -u ssh://{0}@{1}/{2}'.format(user,address,mct_local)
    if exists(mct_name):
        with cd(mct_name):
            run(cmd)

def _rosmake(preclean=False):
    """
    Run rosmake on mct project directory
    """
    mct_name = os.environ['MCT_NAME']
    if exists(mct_name):
        with cd(mct_name):
            if preclean:
                run('source ~/bin/mct_setup.bash; rosmake --rosdep-install --pre-clean',shell=True)
            else:
                run('source ~/bin/mct_setup.bash; rosmake --rosdep-install',shell=True)

    else:
        print('ERROR: unable to run rosamke {0} does not exist'.format(mct_name))


@hosts(*slave_list)
def rosmake():
    """
    Run rosmake on mct project directory on remote computers
    """
    _rosmake(preclean=False)

@hosts(*slave_list)
def rosmake_preclean():
    """
    Run rosmake on mct project directory on remote computers - preclean before makeing.
    """
    _rosmake(preclean=True)

@hosts(*slave_list)
def clone():
    """
    Clone a new versoin of mct on the slave computers.

    Note, may want to move bitbucket address to configuration file.
    """
    ros_local = os.environ['ROS_LOCAL']
    mct_name = os.environ['MCT_NAME']
    if not exists(ros_local):
        run('mkdir {0}'.format(ros_local))
    with cd(ros_local):
        if exists(mct_name):
            run('rm -r {0}'.format(mct_name))
        run('hg clone http://bitbucket.org/iorodeo/mct')

@hosts(*slave_list)
def clean():
    """
    Remove any old versions of mct, the contents of the .mct directory, and the
    bash setup file mct_setup.bash.
    """
    mct_name = os.environ['MCT_NAME']
    if exists(mct_name):
        run('rm -r {0}'.format(mct_name))
    mct_resources = os.environ['MCT_RESOURCES'] 
    if exists(mct_resources):
        run('rm -r {0}'.format(mct_resources))
    bin_dir = os.path.join(os.environ['HOME'],'bin')
    setup_file = os.path.join(bin_dir,'mct_setup.bash')
    run('rm {0}'.format(setup_file))

@hosts(*slave_list)
def rospack_profile():
    """
    Run rospack profile on all slave computers.
    """
    run('source ~/bin/mct_setup.bash; rospack profile',shell=True)

def list_slaves():
    """
    List slaves names and mac addresses
    """
    slave_info = mct_introspection.get_slave_info()
    for slave, info in slave_info.iteritems():
        print(slave,info)

def update_machine_def():
    """
    Updates the ROS xml machine launch file using the the current machine_def.yaml
    machine definition file.
    """
    machine_def = mct_introspection.get_machine_def()
    machine_dir = mct_utilities.file_tools.machine_dir
    mct_machine_file = os.path.join(machine_dir, 'mct.machine')
    mct_xml_tools.launch.create_machine_launch(mct_machine_file, machine_def)

def list_machine_def():
    """
    Prints the current machine definition from the machine_def.yaml file.
    """
    machine_def = mct_introspection.get_machine_def()
    print('user', machine_def['user'])
    print('master', machine_def['mct_master'])

    slave_keys = machine_def.keys()
    slave_keys.remove('user')
    slave_keys.remove('mct_master')
    slave_keys.sort()
    for key in slave_keys:
        print(key,machine_def[key])

def list_cameras():
    """
    Lists all comands currently connected to the system.
    """
    inspector_popen = subprocess.Popen(['roslaunch', 'mct_camera_tools', 'camera1394_inspector_master.launch'])
    time.sleep(3.0) # Wait fo processes to start - kludgey need a better approach
    camera_dict = mct_introspection.find_cameras()
    print('\nfound {0} cameras\n'.format(len(camera_dict)))
    for k,v in camera_dict.iteritems():
        print('guid {0}, {1}'.format(k,v))
    print('\n')
    inspector_popen.send_signal(subprocess.signal.SIGINT) 

def list_camera_assignment():
    """
    List the current camera assignment as given in the camera_assignment.yaml file.
    """
    camera_assignment = mct_introspection.get_camera_assignment()
    if camera_assignment is not None:
        camera_nums = camera_assignment.keys()

        # Sort cameras in numerical order
        def cmp_func(x,y):
            num_x = int(x.replace('camera_', ''))
            num_y = int(y.replace('camera_', ''))
            if num_x > num_y:
                return 1
            elif num_x < num_y:
                return -1
            else:
                return 0
        camera_nums.sort(cmp=cmp_func) 
        # Display camera assignment
        for cam in camera_nums: 
            print(cam)
            for k,v in camera_assignment[cam].iteritems():
                print('  {0}: {1}'.format(k,v))
        print()
    else:
        print('\n camera assignment is None\n')

def rsync_camera_calibrations():
    """
    Use rsync to send camera calibrations to camera computers
    """
    mct_utilities.file_tools.rsync_camera_calibrations(verbose=True)

@hosts(*host_list)
def clean_camera_calibrations():
    """
    Clean any existing old camera calibration files.
    """
    ros_camera_info_dir = mct_utilities.file_tools.ros_camera_info_dir
    if exists(ros_camera_info_dir):
        with cd(ros_camera_info_dir):
            run('rm *.yaml')

def show_camera_info():
    """
    Launch rostopic echo for all camera info topics.
    """
    camera_info_topics = mct_introspection.find_camera_info_topics()
    popen_list = []
    filename_list = []
    tmp_dir = tempfile.gettempdir()
    pos_x_init = 10
    pos_y_init = 50
    step_x = 160 
    step_y = 520 
    pos_x = pos_x_init 
    pos_y = pos_y_init
    for i, topic in enumerate(camera_info_topics):
        filename = os.path.join(tmp_dir, 'camera_info_{0}.bash'.format(i))
        if i>0:
            pos_x += step_x
            if i%4 == 0:
                pos_x = pos_x_init
                pos_y += step_y
        with open(filename, 'w') as f:
            f.write('rostopic echo {0}\n'.format(topic))
        os.chmod(filename,0755)
        geometry = '{0}x{1}+{2}+{3}'.format(80,28,pos_x,pos_y)
        popen = subprocess.Popen(['gnome-terminal', '--geometry', geometry, '-x', filename])
        popen_list.append(popen)
        filename_list.append(filename)
        time.sleep(0.1)
    # Wait a bit before deleting files
    time.sleep(5)
    for filename in filename_list:
        os.remove(filename)

def show_camera_info_header():
    """
    Launch rostopic echo for all camera_info/header topics. Note, This function
    should be unified with show_camera_info above.
    """
    camera_info_topics = mct_introspection.find_camera_info_topics()
    popen_list = []
    filename_list = []
    tmp_dir = tempfile.gettempdir()
    pos_x_init = 10
    pos_y_init = 50
    step_x = 160 
    step_y = 520 
    pos_x = pos_x_init 
    pos_y = pos_y_init
    for i, topic in enumerate(camera_info_topics):
        filename = os.path.join(tmp_dir, 'camera_info_{0}.bash'.format(i))
        if i>0:
            pos_x += step_x
            if i%4 == 0:
                pos_x = pos_x_init
                pos_y += step_y
        with open(filename, 'w') as f:
            f.write('rostopic echo {0}/header\n'.format(topic))
        os.chmod(filename,0755)
        geometry = '{0}x{1}+{2}+{3}'.format(80,28,pos_x,pos_y)
        popen = subprocess.Popen(['gnome-terminal', '--geometry', geometry, '-x', filename])
        popen_list.append(popen)
        filename_list.append(filename)
        time.sleep(0.1)
    # Wait a bit before deleting files
    time.sleep(5)
    for filename in filename_list:
        os.remove(filename)

def show_camera_info_header_seq():
    """
    Launch rostopic echo for all camera_info/header topics. Note, This function
    should be unified with show_camera_info above.
    """
    camera_info_topics = mct_introspection.find_camera_info_topics()
    popen_list = []
    filename_list = []
    tmp_dir = tempfile.gettempdir()
    pos_x_init = 10
    pos_y_init = 50
    step_x = 75
    step_y = 520 
    pos_x = pos_x_init 
    pos_y = pos_y_init

    for i, topic in enumerate(camera_info_topics):
        filename = os.path.join(tmp_dir, 'camera_info_{0}.bash'.format(i))
        if i>0:
            pos_x += step_x
            if i%6 == 0:
                pos_x = pos_x_init
                pos_y += step_y
        with open(filename, 'w') as f:
            f.write('rostopic echo {0}/header/seq\n'.format(topic))
        os.chmod(filename,0755)
        geometry = '{0}x{1}+{2}+{3}'.format(80,28,pos_x,pos_y)
        popen = subprocess.Popen(['gnome-terminal', '--geometry', geometry, '-x', filename])
        popen_list.append(popen)
        filename_list.append(filename)
        time.sleep(0.1)
    # Wait a bit before deleting files
    time.sleep(5)
    for filename in filename_list:
        os.remove(filename)

def show_corrector_seq():
    """
    Launch rostopic echo for all seq_and_image_corr topics - showing just the
    corrected sequences.     
    """
    seq_and_image_topics = mct_introspection.find_topics_w_ending('seq_and_image_corr')
    popen_list = []
    filename_list = []
    tmp_dir = tempfile.gettempdir()
    pos_x_init = 10
    pos_y_init = 50
    step_x = 75 
    step_y = 520 
    pos_x = pos_x_init 
    pos_y = pos_y_init
    for i, topic in enumerate(seq_and_image_topics):
        filename = os.path.join(tmp_dir, 'corrector_seq_{0}.bash'.format(i))
        if i>0:
            pos_x += step_x
            if i%6 == 0:
                pos_x = pos_x_init
                pos_y += step_y
        with open(filename, 'w') as f:
            f.write('rostopic echo {0}/seq\n'.format(topic))
        os.chmod(filename,0755)
        geometry = '{0}x{1}+{2}+{3}'.format(80,28,pos_x,pos_y)
        popen = subprocess.Popen(['gnome-terminal', '--geometry', geometry, '-x', filename])
        popen_list.append(popen)
        filename_list.append(filename)
        time.sleep(0.1)
    # Wait a bit before deleting files
    time.sleep(5)
    for filename in filename_list:
        os.remove(filename)

def frames_dropped():
    """
    Prints a simple report of the frames dropped by the system. 
    """
    info_dict = frame_drop_corrector.info_all()
    max_name_len = max([len(name) for name in info_dict])
    print('  ', 'camera', ' '*(max_name_len - len('camera')), '#','  seq list')
    print('  ', '-'*60)
    for name, seq_list in sorted(info_dict.items(),cmp=info_item_cmp):
        pad = ' '*(max_name_len - len(name))
        print('  ', name, pad, len(seq_list), '  ', seq_list)
    print()
    total = sum([len(x) for x in info_dict.values()])
    print('  ', 'total frames dropped: ', total)
    print()

def info_item_cmp(x,y):
    """
    Comparison function for items (k,v) in the dictionary returned
    byt the frame_drop_corrector.info_all() function. 
    """
    x_num = int(x[0].split('_')[1])
    y_num = int(y[0].split('_')[1])
    if x_num > y_num:
        return 1
    elif x_num < y_num:
        return -1
    else:
        return 0

def test(*args):
    print('test')

def main(argv):
    """
    Main function for mct_cluster_control when called as a commandline program.
    """
    # Get command line argument
    cmd = argv[1].lower()
    fabfile, ext = os.path.splitext(__file__)
    fabfile = '{0}.py'.format(fabfile)
    try: 
        msg = cmd_msgs[cmd]
        print('\n{0}\n'.format(msg))
    except KeyError:
        print('Error: unknown command {0}'.format(cmd))
        sys.exit(0)

    # Generate fab command and add any arguments
    if cmd in fab_cmds:
        fab_cmd = 'fab -f {0} {1}'.format(fabfile, cmd,)
        if argv[2:]:
            fab_cmd_args = ','.join(argv[2:])
            fab_cmd = '{0}:{1}'.format(fab_cmd,fab_cmd_args)

        subprocess.call(fab_cmd, shell=True)
    else:
        try:
            globals()[cmd]() 
        except KeyError:
            print('ERROR: command not found, {0}'.format(cmd))