Snippets

Brandon Nielsen Python3 parallel script runner

Created by Brandon Nielsen last modified
#!/usr/bin/env python3

import argparse
import multiprocessing
import subprocess

#Key is a 'friendly' script name, value is a tuple of command to run and arguments
SCRIPT_NAMES = {
    'test1': ('./test1.sh'),
    'test2': ('./test2.sh'),
    'test3': ('./test3.sh')
}

def main ():
    parser = argparse.ArgumentParser()

    #num_processes is the number of processes to split the scripts over, defaults to cpu count (1 to run serially)
    parser.add_argument('-n', '--num_processes', type=int, help='number of processes to split scripts across, default={0}'.format(multiprocessing.cpu_count()), default=multiprocessing.cpu_count())

    #scripts is the list of scripts to run, run all scripts by default
    parser.add_argument('scripts', type=str, nargs='*', help='the scripts to run, scripts available: {0}'.format(', '.join([*SCRIPT_NAMES])), default=[*SCRIPT_NAMES])

    args = parser.parse_args()

    #Build a multiprocessing pool with the requested process count
    pool = multiprocessing.Pool(args.num_processes)

    #Run in parallel, aggregating results
    results = pool.map(run_func, args.scripts)

    #Print the results, line by line
    for result in results:
        print(result)

def run_func(to_run):
    #Run the requested script, capturing standard out
    process = subprocess.Popen(SCRIPT_NAMES[to_run], stdout=subprocess.PIPE)

    #Bang the results into a string
    return process.communicate()[0].decode('utf-8')

if __name__ == '__main__':
    main()
1
2
3
4
#!/bin/sh

sleep 5
echo test 1
1
2
3
4
#!/bin/sh

sleep 5
echo test 2
1
2
3
4
#!/bin/sh

sleep 5
echo test 3

Comments (0)

HTTPS SSH

You can clone a snippet to your computer for local editing. Learn more.