how to use subprocess with multiple multiple stdin from zcat

Question

I would like to convert the following shell command into python code using subprocess. In particular, how to convert the multiple <(zcat ...) as stdin? Ideally, the code should use subprocess.call, but subprocess.Popen is fine, too.

rsem-calculate-expression \
-p 2 \
--some-other-option \
--paired-end \
<(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) \
<(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) \
~/index/hg19 \
output_dir \
1>stdin.log \
2>stderr.log

Answer 1

The easiest way is to just let bash handle the process substitutions:

subprocess.Popen(['/bin/bash', '-c', 'rsem-calculate-expression -p 2 \
    --some-other-option --paired-end \
    <(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) \
    <(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) \
    ~/index/hg19 output_dir'])

Answer 2

Another way to run the bash command is to use shell=True:

from subprocess import check_call

check_call('rsem-calculate-expression -p 2 --some-other-option '
           '--paired-end '
           '<(zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz) '
           '<(zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz) '
           '~/index/hg19 '
           'output_dir '
           '1>stdin.log '
           '2>stderr.log', shell=True, executable='/bin/bash')

For comparison, here's how to do it without running shell:

#!/usr/bin/env python3
import os
import shlex
from contextlib import ExitStack # $ pip install contextlib2 on Python 2
from shutil import rmtree
from subprocess import Popen
from tempfile import mkdtemp

# convert command-line into a list (program arguments)
args = 'rsem-calculate-expression -p2 --some-other-option --paired-end'.split()
zcat_args = [shlex.split('zcat a_1.fastq.gz b_1.fastq.gz c_1.fastq.gz'),
             shlex.split('zcat b_2.fastq.gz b_2.fastq.gz c_2.fastq.gz')]
npipes = len(zcat_args)
with ExitStack() as stack:
    # create named pipes
    pipenames = []
    dirname = mkdtemp() # create temporary directory for named pipes
    stack.callback(rmtree, dirname) # clean up at the end
    for i in range(npipes):
        pipename = os.path.join(dirname, 'zcat_named_pipe' + str(i))
        os.mkfifo(pipename)
        args.append(pipename) # append to the program args
        pipenames.append(pipename)

    # run rsem-calculate-expression that reads from the named pipes
    args.append(os.path.expanduser('~/index/hg19'))
    args.append('output_dir')
    with open('stdout.log', 'wb', 0) as f1, open('stderr.log', 'wb', 0) as f2:
        rsem = Popen(args, stdout=f1, stderr=f2)
    stack.callback(rsem.wait)

    # run zcat commands to populate the named pipes
    for pipename, cmd in zip(pipenames, zcat_args):
        if rsem.poll() is None: # it is still running and reading from pipes
            with open(pipename, 'wb', 0) as pipe: # may block if no readers
                stack.callback(Popen(cmd, stdout=pipe).wait)
exit(rsem.returncode != 0)