So, you've heard about STDIN, STDOUT and STDERR, but still don't know what the heck is that?

Long story short...

Every Unix process has —at least— those three file descriptors.

Redirection & Pipes

Redirecting STDOUT

~/Documents $ ls -Al
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf


# Don't wanna see that... save it to a file instead
~/Documents $ ls -Al > ls_output.txt    # Nothing shows :O
~/Documents $ cat ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 15:10 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf

but wait...

~/Documents $ cat ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 15:10 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf


~/Documents $ echo hello > ls_output.txt

# WTF??
~/Documents $ cat ls_output.txt
hello

meet noclobber

~/Documents $ ls -Al > ls_output.txt
~/Documents $ cat ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 15:10 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf


~/Documents $ set -o noclobber
~/Documents $ echo hello > ls_output.txt
-bash: ls_output.txt: cannot overwrite existing file

be careful, though...

~/Documents $ cat ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 15:10 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf


# '>>' appends output to the given file
~/Documents $ echo hello >> ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 15:10 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf
hello

Redirecting STDERR

~/Documents $ ls -Al NonExistent > ls_output.txt
ls: cannot access NonExistent: No such file or directory

~/Documents $ cat ls_output.txt
~/Documents $     # ls_output.txt is empty!


# That message you saw is supposed to go to STDERR
# Which happens to be the terminal emulator... let's fix that
~/Documents $ ls -Al NonExistent > ls_output.txt 2> ls_error.txt

# Empty again... 'ls' failed, so there's nothing to
# show in STDOUT
~/Documents $ cat ls_output.txt
~/Documnets $

# But...
~/Documents $ cat ls_error.txt
ls: cannot access NonExistent: No such file or directory

Redirecting STDIN

# We'll see a bit more about 'tr' later on
# For now, you only need to know that it expects two strings
# and input from STDIN
~/Documents $ cat ls_output.txt
total 20
-rwxrw-r--. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
-rw-rw-r--. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
-rw-rw-r--. 1 santiaro90 santiaro90    0 Sep 11 16:44 ls_output.txt
-rw-r--r--. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf

~/Documents $ tr - _ ls_output.txt
tr: extra operand ‘ls_output.txt’
Try 'tr --help' for more information.

# Check this out...
~/Documents $ tr - _ < ls_output.txt
total 20
_rwxrw_r__. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
_rw_rw_r__. 1 santiaro90 santiaro90    0 Sep 11 16:44 ls_output.txt
_rw_r__r__. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf

you can mix redirections, of course...

~/Documents $ tr - _ < ls_output.txt > tr_output.txt
~/Documents $ cat tr_output.txt
total 20
_rwxrw_r__. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
_rw_rw_r__. 1 santiaro90 santiaro90    0 Sep 11 16:44 ls_output.txt
_rw_r__r__. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf

Using Pipes

Pipes basically let you take the output from one command to feed another one.

Using Pipes

~/Documents $ ls -Al | tr - _
total 28
_rwxrw_r__. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 16:44 ls_output.txt
_rw_r__r__. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 17:12 tr_output.txt


# Chain as much as you need
~/Documents $ ls -Al | tr - _ | grep .txt | sort
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 16:44 ls_output.txt
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 17:12 tr_output.txt
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt


# Look mah, with redirection!
~/Documents $ ls -Al | tr - _ | grep .txt | sort > cool_piping.txt
~/Documents $ cat cool_piping.txt
_rw_rw_r__. 1 santiaro90 santiaro90    0 Sep 11 17:19 cool_piping.txt
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 16:44 ls_output.txt
_rw_rw_r__. 1 santiaro90 santiaro90  269 Sep 11 17:12 tr_output.txt
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt

tail

~/Documents $ cat /etc/passwd
# .... Loads of output above this
dbus:x:81:81:System message bus:/:/sbin/nologin
polkitd:x:997:995:User for polkitd:/:/sbin/nologin
tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin
postfix:x:89:89::/var/spool/postfix:/sbin/nologin
sshd:x:74:74:Privilege-separated SSH:/var/empty/sshd:/sbin/nologin
santiaro90:x:1000:1000:santiaro90:/home/santiaro90:/bin/bash
usbmuxd:x:113:113:usbmuxd user:/:/sbin/nologin
geoclue:x:996:994:User for geoclue:/var/lib/geoclue:/sbin/nologin
unbound:x:995:993:Unbound DNS resolver:/etc/unbound:/sbin/nologin
openvpn:x:994:992:OpenVPN:/etc/openvpn:/sbin/nologin
rtkit:x:172:172:RealtimeKit:/proc:/sbin/nologin
abrt:x:173:173::/etc/abrt:/sbin/nologin
pulse:x:171:171:PulseAudio System Daemon:/var/run/pulse:/sbin/nologin
lightdm:x:993:988::/var/lib/lightdm:/sbin/nologin
setroubleshoot:x:992:987::/var/lib/setroubleshoot:/sbin/nologin
nm-openvpn:x:991:986:Default user for running openvpn spawned by NetworkManager:/:/sbin/nologin
nm-openconnect:x:990:985:NetworkManager user for OpenConnect:/:/sbin/nologin
vboxadd:x:989:1::/var/run/vboxadd:/bin/false
ntp:x:38:38::/etc/ntp:/sbin/nologin
test:x:1001:1001::/home/test:/bin/bash
~/Documents $    # What the hell was that?

tail

# Much better...
~/Documents $ tail /etc/passwd    # last 10 lines by default
rtkit:x:172:172:RealtimeKit:/proc:/sbin/nologin
abrt:x:173:173::/etc/abrt:/sbin/nologin
pulse:x:171:171:PulseAudio System Daemon:/var/run/pulse:/sbin/nologin
lightdm:x:993:988::/var/lib/lightdm:/sbin/nologin
setroubleshoot:x:992:987::/var/lib/setroubleshoot:/sbin/nologin
nm-openvpn:x:991:986:Default user for running openvpn spawned by NetworkManager:/:/sbin/nologin
nm-openconnect:x:990:985:NetworkManager user for OpenConnect:/:/sbin/nologin
vboxadd:x:989:1::/var/run/vboxadd:/bin/false
ntp:x:38:38::/etc/ntp:/sbin/nologin
test:x:1001:1001::/home/test:/bin/bash

~/Documents $

tail -n

# Last 5 lines
~/Documents $ tail -n 5 /etc/passwd
nm-openvpn:x:991:986:Default user for running openvpn spawned by NetworkManager:/:/sbin/nologin
nm-openconnect:x:990:985:NetworkManager user for OpenConnect:/:/sbin/nologin
vboxadd:x:989:1::/var/run/vboxadd:/bin/false
ntp:x:38:38::/etc/ntp:/sbin/nologin
test:x:1001:1001::/home/test:/bin/bash


# From line No. 30 on...
~/Documents $ tail -n +30 /etc/passwd
lightdm:x:993:988::/var/lib/lightdm:/sbin/nologin
setroubleshoot:x:992:987::/var/lib/setroubleshoot:/sbin/nologin
nm-openvpn:x:991:986:Default user for running openvpn spawned by NetworkManager:/:/sbin/nologin
nm-openconnect:x:990:985:NetworkManager user for OpenConnect:/:/sbin/nologin
vboxadd:x:989:1::/var/run/vboxadd:/bin/false
ntp:x:38:38::/etc/ntp:/sbin/nologin
test:x:1001:1001::/home/test:/bin/bash

tail -f

head

# First 10 lines of the file
~/Documents $ head /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
operator:x:11:0:operator:/root:/sbin/nologin


# Ok, how about first 5...
~/Documents $ head -n 5 /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin

head

# Drop the last 20 lines
# Notice how the sign changes... :)
~/Documents $ head -n -20 /etc/passwd
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
operator:x:11:0:operator:/root:/sbin/nologin
games:x:12:100:games:/usr/games:/sbin/nologin
ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
nobody:x:99:99:Nobody:/:/sbin/nologin
avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin
systemd-bus-proxy:x:999:997:systemd Bus Proxy:/:/sbin/nologin
systemd-network:x:998:996:systemd Network Management:/:/sbin/nologin

wc

# How many chars are there in these files?
~/Documents $ wc -c /etc/passwd /etc/ntp.conf
1898 /etc/passwd
2000 /etc/ntp.conf
3898 total


# Noice! How about word count?
~/Documents $ wc -w /etc/passwd /etc/ntp.conf
  75 /etc/passwd
 273 /etc/ntp.conf
 348 total


# Words are meaningless to me...
# Wanna know how many lines are there
~/Documents $ wc -l /etc/passwd /etc/ntp.conf
  36 /etc/passwd
  58 /etc/ntp.conf
  94 total


# Can I get all numbers in one go?... You bet!
~/Documents $ wc /etc/passwd /etc/ntp.conf
  36   75 1898 /etc/passwd
  58  273 2000 /etc/ntp.conf
  94  348 3898 total

tee

# Try guessing what 'tee' does... have a look
~/Documents $ tr - _ < ls_output.txt | tee didnt_see_that_coming.txt
total 20
_rwxrw_r__. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
_rw_rw_r__. 1 santiaro90 santiaro90    0 Sep 11 16:44 ls_output.txt
_rw_r__r__. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf


~/Documents $ cat didnt_see_that_coming.txt
total 20
_rwxrw_r__. 1 santiaro90 santiaro90  168 Sep  4 22:19 awk_tuto
_rw_rw_r__. 1 santiaro90 santiaro90 9920 Sep  2 07:49 .hidden.txt
_rw_rw_r__. 1 santiaro90 santiaro90    0 Sep 11 16:44 ls_output.txt
_rw_r__r__. 1 santiaro90 santiaro90 2000 Sep  4 18:33 ntp.conf

sort

~/Documents $ sort /etc/passwd
abrt:x:173:173::/etc/abrt:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
# ... Couple more lines in here
test:x:1001:1001::/home/test:/bin/bash
tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin
unbound:x:995:993:Unbound DNS resolver:/etc/unbound:/sbin/nologin
usbmuxd:x:113:113:usbmuxd user:/:/sbin/nologin
vboxadd:x:989:1::/var/run/vboxadd:/bin/false

# You know what's comming, don't you?
~/Document $ sort -r /etc/passwd
vboxadd:x:989:1::/var/run/vboxadd:/bin/false
usbmuxd:x:113:113:usbmuxd user:/:/sbin/nologin
unbound:x:995:993:Unbound DNS resolver:/etc/unbound:/sbin/nologin
tss:x:59:59:Account used by the trousers package to sandbox the tcsd daemon:/dev/null:/sbin/nologin
test:x:1001:1001::/home/test:/bin/bash
# ... God, this is is long!
daemon:x:2:2:daemon:/sbin:/sbin/nologin
bin:x:1:1:bin:/bin:/sbin/nologin
avahi-autoipd:x:170:170:Avahi IPv4LL Stack:/var/lib/avahi-autoipd:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
abrt:x:173:173::/etc/abrt:/sbin/nologin

sort

sort

# '-n' => natural sorting... nice, innit?
~/Documents $ sort -n -t : -k 3,3 /etc/passwd | head -n 5
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin

Yeah, that was sweet... But too much information. I only need username and home directory...

 ¯\_(ツ)_/¯

cut

# '-d' => how fields are separated
# '-f' => which fields to take
~/Documents $ cut -d : -f 1,7 /etc/passwd | head
root:/bin/bash
bin:/sbin/nologin
daemon:/sbin/nologin
adm:/sbin/nologin
lp:/sbin/nologin
sync:/bin/sync
shutdown:/sbin/shutdown
halt:/sbin/halt
mail:/sbin/nologin
operator:/sbin/nologin


# That looks odd... make it prettier!
~/Documents $ cut -d : -f 1,7 --output-delimiter=' - ' /etc/passwd | head
root - /bin/bash
bin - /sbin/nologin
daemon - /sbin/nologin
adm - /sbin/nologin
lp - /sbin/nologin
sync - /bin/sync
shutdown - /sbin/shutdown
halt - /sbin/halt
mail - /sbin/nologin
operator - /sbin/nologin

tr (translate)

~/Documents $ echo "Change all my a's for x's please" | tr a x
Chxnge xll my x's for x's plexse


~/Documents $ echo "Now let's 'cypher' all those vowels" | tr aeiou x
Nxw lxt's 'cyphxr' xll thxsx vxwxls


~/Documents $ echo "A bit of Caesar's cypher on this string" | tr a-zA-Z cd-zabCD-ZAB
C dkv qh Ecguct'u earjgt qp vjku uvtkpi


~/Documents $ echo 'Trim my vowels... just consonants' | tr -d aeiou
Trm my vwls... jst cnsnnts


~/Documents $ echo 'Too     many     spaces  in    here,   mate!' | tr -s ' '
Too many spaces in here, mate!


# '-c' to complement a set (all characters but the ones listed)
# Notice the '\n' to keep the new line in the output
~/Documents $ echo "I only need vowels... just don't wanna list all consonants" | tr -d -c 'aeiouAEIOU\n '
I o ee oe u o aa i a ooa