sh

bash in Python.

Nir Cohen @ Gigaspaces

@thinkops

Why sh?

• os.system()? Featureless!
• More human readable than subprocess
• Much less demanding than Fabric local()
• More Pythonic than Plumbum...
• Automatic executable mapping.
• Wonderfully documented (one pager! damn!).
• Mucho Fun to use.

Subprocess

import subprocess
import time


def run_command(cmd):
    print('Executing: {0}...'.format(cmd))
    pipe = subprocess.PIPE
    proc = subprocess.Popen(cmd, shell=True, stdout=pipe, stderr=pipe)
    proc.aggr_stdout = ''
    # while the process is still running, print output
    while proc.poll() is None:
        output = proc.stdout.readline()
        proc.aggr_stdout += output
        if len(output) > 0:
            print(output)
        time.sleep(0.2)
    output = proc.stdout.readline()
    proc.aggr_stdout += output
    if len(output) > 0:
        print(output)
    proc.aggr_stderr = proc.stderr.read()
    if len(proc.aggr_stderr) > 0:
        print(proc.aggr_stderr)
    return proc


output = run_command('pip install repex')
if output.returncode == 0:
    print('HOORAY!')
else:
    print('Docker.')

Fabric (local)

import fabric.api as fab


def run_command(cmd):
    print('Executing: {0}...'.format(cmd))
    # oi. either it'll print or return in proc.stdout. 
    # Not both..
    proc = fab.local(cmd, capture=False)
    return proc


output = run_command('pip install repex')
if output.succeeded:
    print('HOORAY!')
else:
    print('Docker.')

nicer!

import sh

o = sh.pip.install('repex')
print o
if o.exit_code == 0:
    print('HOORAY!')
else:
    print('Docker.')

execute. (with args)

# virtualenv test -p /usr/bin/python

import sh

vpath = 'test'
python = '/usr/bin/python'

# execute with arguments 
sh.virtualenv(vpath, '-p', python)
# or comfortably named arguments
sh.virtualenv(vpath, p=python)

premade exec object

# test/bin/pip install python-packer==0.0.1 --default-timeout 60 &

env["PIP_DEFAULT_TIMEOUT"] = "60"
bins_dir = 'scripts' if sys.platform == 'win32' else 'bin'

pip = sh.Command(os.path.join('test', bins_dir, 'pip'))

# You can iterate over the output, pass env, acceptable exit 
# codes and put a process in the background while 
# printing its output.
for line in pip.install(
    'python-packer==0.0.1',_iter=True, _env=env, 
    _ok_code=[0,19], _bg=True):
        print line

advanced processing

# test/bin/pip install repex==0.8 >> output.file
# test/bin/pip install feedr 1> output.file 

def _process(line):
    print(line)

# output can be directed to files
try:
    pip.install('repex==0.8', _out='output.file')
except sh.ErrorReturnCode_1 as ex:
    print('Could not install: {0}'.format('repex==0.8'))

# or file objects.
with open('output.file', 'w') as f:
    process = pip.install(
        'feedr', _out=f, _err=_process, _bg=True)
# backgrounds processes can be waited upon before acting.
process.wait()
...

piping

# test/bin/pip freeze | grep -v python-packer | wc -l

# you can use the output
modules = pip.freeze().splitlines()
print 'YAY!' if 'python-packer==0.0.1' in modules \
else 'DOCKER!'

# or you can pipe..
i = sh.wc(sh.grep(
    pip.freeze(), '-v', 'python-packer'), '-l'))
print('number of modules installed which '
      'are not python-packer: {0}'.format(i)
    

bake a cake

# pip wheel python-packer --verbose --find-links==dir
# mkdir -p my_wheels_dir
# mv dir/*.whl my_wheels_dir

from sh import glob as g

wheel = pip.wheel
# bake arguments for reuse
wheel = wheel.bake('--verbose', '--find-links==dir')
# now every `wheel` command will run with 
# `--verbose` and `--find-links`
wheel('feedr', _out=_process)
wheel('repex', _out=_process)

sh.mkdir('-p', 'my_wheels_dir')
# glob expansion
sh.mv(g('dir/*.whl'), 'my_wheels_dir')

context, advanced piping

# test/bin/mouth feed -t=File -f=ApacheAccess --gap=1 &
# sleep 2
# tail -f generated.log | grep GET

feedr = sh.Command('test/bin/mouth')
# run my program in the background and detach from shell
with sh.nohup:
  x = feedr.feed(
    t='File', t='ApacheAccess', g='1', _bg=True)
do_something_with(x)
time.sleep(2)
if os.path.isfile('generated.log'):
  # pass stdout from a long running process by piping 
  # its output while it runs
  sh.grep(sh.tail('-f', 'generated.log', _piped=True),
          'GET', _out=_process)