All notes
Bash

My bashrc


## Suppress pushd output.
pushd () {
    command pushd "[email protected]" > /dev/null
}
popd () {
    command popd "[email protected]" > /dev/null
}

# bash checks the window size after each command and, if necessary, updates the values of LINES and COLUMNS.
shopt -s checkwinsize
shopt -s cmdhist
# Do not record duplicate commands, ls, bg, fg and exit.
export HISTIGNORE="&:ls:[bf]g:exit"

# The $PWD must be capitalized and the $ must be escaped, otherwise the $PWD will be evaluated where the PS1 is assigned, and then the PS1 will always show the home directory.
# export PS1="\h:\$PWD \u\$ "
export PS1="\[email protected]\h:\w "
# The last color-set sequence, \[\e[1;37m\], is not closed, so the remaining text will be white.
# The color setting may cause problem. Use in caution.
# export PS1='\[\e[0;32m\]\h\[\e[m\]:\[\e[1;34m\]\w\[\e[m\] \[\e[1;32m\]\u\$\[\e[m\] \[\e[1;37m\]'

export EDITOR=emacs

# alias open="xdg-open"
# export open

export PATH=$PATH:~/bin
export TERM='xterm-color'

cdls() { cd "[email protected]" && ls; }

alias ls="ls --color" # Linux
# alias ls="ls -G" # Mac
export ls
alias ll='ls -al'
export ll
alias rm='rm -ir'
export rm
alias rmf='rm -fr'
export rmf
alias cp='cp -ir'
export cp
alias mv='mv -i'
export mv
alias scp='scp -r'
export scp
alias grep="grep --color=always -n"
export grep

PS1

\u - Username. The original prompt also has \h, which prints the host name.
\w - Current absolute path. Use \W for current relative path.
\$ - The prompt character (eg. # for root, $ for regular users).
\[ and \] - These tags should be placed around color codes so bash knows how to properly place the cursor.

Colorful PS1

Ref.

Ref.

\e[ Indicates the beginning of color prompt
x;ym Indicates color code. Use the color code values mentioned below.
\e[m indicates the end of color prompt

Black 0;30
Blue 0;34
Green 0;32
Cyan 0;36
Red 0;31
Purple 0;35
Brown 0;33
[Note: Replace 0 with 1 for dark color]

Bash script recommendations

A function to print out error messages along with other status information is recommended. The following is from Google shell script guide.


err() {
	echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: [email protected]" >&2
}

Replacement of getopts

StackOverflow. It says:

Never use getopt. getopt cannot handle empty arguments strings, or arguments with embedded whitespace.
${i#*=} search for "Substring Removal" in this guide. It is functionally equivalent to `sed 's/[^=]*=//' <<< "$i"` which calls a needless subprocess or `echo "$i" | sed 's/[^=]*=//'` which calls two needless subprocesses.

for i in "[email protected]"
do
case $i in
    -e=*|--extension=*)
    EXTENSION="${i#*=}"
    shift # past argument=value
    ;;
    -s=*|--searchpath=*)
    SEARCHPATH="${i#*=}"
    shift # past argument=value
    ;;
    -l=*|--lib=*)
    LIBPATH="${i#*=}"
    shift # past argument=value
    ;;
    --default)
    DEFAULT=YES
    shift # past argument with no value
    ;;
    *)
            # unknown option
    ;;
esac
done
echo "FILE EXTENSION  = ${EXTENSION}"
echo "SEARCH PATH     = ${SEARCHPATH}"
echo "LIBRARY PATH    = ${LIBPATH}"
echo "Number files in SEARCH PATH with EXTENSION:" $(ls -1 "${SEARCHPATH}"/*."${EXTENSION}" | wc -l)
if [[ -n $1 ]]; then
    echo "Last line of file specified as non-opt/last argument:"
    tail -1 $1
fi

Straight Bash Space Separated

Usage: ./myscript.sh -e conf -s /etc -l /usr/lib /etc/hosts


#!/bin/bash
# Use > 1 to consume two arguments per pass in the loop (e.g. each
# argument has a corresponding value to go with it).
# Use > 0 to consume one or more arguments per pass in the loop (e.g.
# some arguments don't have a corresponding value to go with it such
# as in the --default example).
# note: if this is set to > 0 the /etc/hosts part is not recognized ( may be a bug )
while [[ $# > 1 ]]
do
key="$1"

case $key in
    -e|--extension)
    EXTENSION="$2"
    shift # past argument
    ;;
    -s|--searchpath)
    SEARCHPATH="$2"
    shift # past argument
    ;;
    -l|--lib)
    LIBPATH="$2"
    shift # past argument
    ;;
    --default)
    DEFAULT=YES
    ;;
    *)
            # unknown option
    ;;
esac
shift # past argument or value
done
echo FILE EXTENSION  = "${EXTENSION}"
echo SEARCH PATH     = "${SEARCHPATH}"
echo LIBRARY PATH    = "${LIBPATH}"
echo "Number files in SEARCH PATH with EXTENSION:" $(ls -1 "${SEARCHPATH}"/*."${EXTENSION}" | wc -l)
if [[ -n $1 ]]; then
    echo "Last line of file specified as non-opt/last argument:"
    tail -1 $1
fi

Cheet sheet

HereDoc

TLDP.org: here-docs.


# Basic.
interactive-program <<LimitString
command #1
command #2
...
LimitString

# Here variables are expanded.
cat <<Endofmessage
Hello, there, $NAME.
Greetings to you, $NAME, from $RESPONDENT.
# This comment shows up in the output (why?).
Endofmessage

# With single/double quotes or backslash escape, the variables will not expanded.
cat >> path/to/file/to/append-to.txt << "EOF"
export PATH=$HOME/jdk1.8.0_31/bin:$PATH
export JAVA_HOME=$HOME/jdk1.8.0_31/
EOF

# '-': Suppresses leading tabs (but not spaces)
cat <<-ENDOFMESSAGE
	This is line 1 of the message.
	This is line 2 of the message.
	This is line 3 of the message.
	This is line 4 of the message.
	This is the last line of the message.
ENDOFMESSAGE

# Set a variable from HereDoc.
variable=$(cat <<"SETVAR"
This variable
runs over multiple lines.
SETVAR
)
echo ""

# "anonymous" here document
# use : as a dummy command.
: <<TESTVARIABLES
${HOSTNAME?}${USER?}${MAIL?}  # Print error message if one of the variables not set.
TESTVARIABLES

Use heredoc for comments/documentations:


DOC_REQUEST=70

if [ "$1" = "-h"  -o "$1" = "--help" ]     # Request help.
then                                       # Use a "cat script" . . .
  cat <<DOCUMENTATIONXX
List the statistics of a specified directory in tabular format.
---------------------------------------------------------------
The command-line parameter gives the directory to be listed.
If no directory specified or directory specified cannot be read,
then list the current working directory.

DOCUMENTATIONXX
exit $DOC_REQUEST
fi

Expansions

The order of expansions is:

  1. brace expansion;
  2. tilde expansion;
  3. parameter and variable expansion;
  4. arithmetic expansion;
  5. command substitution (done in a left-to-right fashion);
  6. word splitting;
  7. filename expansion.

Brace expansion

LinuxJournal.


# Notice that there are no spaces inside the brackets or between the brackets and the adjoining strings.
echo {one,two,red,blue}fish
# onefish twofish redfish bluefish
echo {one, two, red, blue }fish
# {one, two, red, blue }fish
echo "{one,two,red,blue} fish"
# {one,two,red,blue} fish

# Spaces are OK if they're enclosed in quotes outside the braces or within an item in the comma-separated list
echo {"one ","two ","red ","blue "}fish
# one fish two fish red fish blue fish
echo {one,two,red,blue}" fish"
# one fish two fish red fish blue fish

cp /etc/httpd/conf/httpd.conf{,.bak}

diff /etc/httpd/conf/httpd.conf{.bak,}

Tilde expansion


# Get $PWD.
echo ~+
# Get $OLDPWD.
echo ~-

# The subdirectory `foo' of the home directory of the user `fred'.
echo ~fred/foo

Shell parameter expansion

If

tldp: comparison ops.

Ways to AND expressions:



if [ ! -z "$var" ] && [ -e "$var" ]; then
	# something ...
fi

if [[ -n "$var" && -e "$var" ]] ; then
    echo "'$var' is non-empty and the file exists"
fi

if [ -n "$var" -a -e "$var" ]; then
    do something ...
fi

# Note that the "<" needs to be escaped within a [ ] construct.
if [[ "$a" < "$b" ]]
if [ "$a" \< "$b" ]

if which ruby >/dev/null && which gem >/dev/null; then
    PATH="$(ruby -rubygems -e 'puts Gem.user_dir')/bin:$PATH"
fi
# /Users/me/.gem/ruby/2.1.0

## A one liner:
if [[ -d bin  ]]; then echo "haha"; fi

SO: check if a program exists:



command -v foo >/dev/null 2>&1 || { echo >&2 "I require foo but it's not installed.  Aborting."; exit 1; }
type foo >/dev/null 2>&1 || { echo >&2 "I require foo but it's not installed.  Aborting."; exit 1; }
hash foo 2>/dev/null || { echo >&2 "I require foo but it's not installed.  Aborting."; exit 1; }

hash wich
# 1
hash which
# 0

command -v which
# /usr/bin/which
command -v ls
# alias ls='ls -G'

For

TLDP.


for arg in "$var1" "$var2" "$var3" ... "$varN"  
# In pass 1 of the loop, arg = $var1	    
# In pass 2 of the loop, arg = $var2	    
# In pass 3 of the loop, arg = $var3	    
# ...
# In pass N of the loop, arg = $varN

# Arguments in [list] quoted to prevent possible word splitting.

FILES="/usr/sbin/accept
/usr/sbin/pwck
/usr/sbin/chroot
/usr/bin/fakefile
/sbin/badblocks
/sbin/ypbind"

echo

for file in $FILES
do
  if [ ! -e "$file" ]       # Check if file exists.
  then
    echo "$file does not exist."; echo
    continue                # On to next.
   fi

  ls -l $file | awk '{ print $8 "         file size: " $5 }'  # Print 2 fields.
  whatis `basename $file`   # File info.
  # Note that the whatis database needs to have been set up for this to work.
  # To do this, as root run /usr/bin/makewhatis.
  echo
done

for i in $(ls *.$SUFF)
do
  mv -f $i ${i%.$SUFF}.$suff
  #  Leave unchanged everything *except* the shortest pattern match
  #+ starting from the right-hand-side of the variable $i . . .
done ### This could be condensed into a "one-liner" if desired.

# List contents of all shells under current dir:
filename="*.sh"
for file in $filename
do
	echo "Contents of $file"
	echo "---"
	cat "$file"
	echo
done

# Removes only files beginning with "j" or "x" in $PWD.

for file in [jx]*
do
	rm -f $file
	echo "Removed file \"$file\"".
done

# Invoke this script both with and without arguments, and see what happens.

for a
do
# echo -n: Do not print the trailing newline character.
	echo -n "$a "
done

# The 'in list' missing, therefore the loop operates on '[email protected]' (command-line argument list, including whitespace).

echo

# Locates matching strings in a binary file.
# A "grep" replacement for binary files.
# Similar effect to "grep -a"

E_BADARGS=65
E_NOFILE=66

if [ $# -ne 2 ]
then
  echo "Usage: `basename $0` search_string filename"
  exit $E_BADARGS
fi

if [ ! -f "$2" ]
then
  echo "File \"$2\" does not exist."
  exit $E_NOFILE
fi  


IFS=$'\012'       # Per suggestion of Anton Filippov. was:  IFS="\n"
for word in $( strings "$2" | grep "$1" )
# The "strings" command lists strings in binary files.
# Output then piped to "grep", which tests for desired string.
do
  echo $word
done

# As S.C. points out, lines 23 - 30 could be replaced with the simpler: strings "$2" | grep "$1" | tr -s "$IFS" '[\n*]'

#  Try something like  "./bin-grep.sh mem /bin/ls"

exit 0

# The result of for loop could be piped to other commands.

for file in "$( find $directory -type l )"   # -type l = symbolic links
do
  echo "$file"
done | sort                                  # Otherwise file list is unsorted.
#  Strictly speaking, a loop isn't really necessary here,
#+ since the output of the "find" command is expanded into a single word.
#  However, it's easy to understand and illustrative this way.

# Iterates through numbers.

# Standard syntax.
for a in 1 2 3 4 5 6 7 8 9 10
do
  echo -n "$a "
done  

echo; echo

# +==========================================+

# Using "seq" ...
for a in `seq 10`
do
  echo -n "$a "
done  

echo; echo

# +==========================================+

# Using brace expansion ...
# Bash, version 3+.
for a in {1..10}
do
  echo -n "$a "
done  

echo; echo

# +==========================================+

# Now, let's do the same, using C-like syntax.

LIMIT=10
for ((a=1; a <= LIMIT ; a++))  # Double parentheses, and naked "LIMIT"
do
  echo -n "$a "
done

while


while true; do python test.py; done

Variables

These parameters may only be referenced; assignment to them is not allowed.

Variable contains space

Variable expansion in quotes and not in quotes makes difference:


a="d d"
set -x

# Without quotes.
echo $a
# + echo d d
# d d

# With quotes, the 'd d' is passed as a string.
echo test "$a"
# + echo 'd d'
# d d

Arrays

51cto.


# Assignment
array=(var1 var2 var3 ... varN)
# or
array=([0]=var1 [1]=var2 [2]=var3 ... [n]=varN)
# or
array[0]=var1
arrya[1]=var2
...
array[n]=varN

# Get element
echo ${array[0]}

# Erase element
unset array[2]
# Erase whole array
unset array
# Print the variable
set | grep arrayName

# Iterate
filename=(`ls`)
for var in ${filename[@]};do
	echo $var
done
  
# Get the total number:
echo ${#array[@]}
# or
echo ${#array[*]}

Grammer

function

TLDP.


function function_name {
	cmd $1 $2
}

########## C-like

# NOTE: the parenthesis can't contain any params.
# So "function_name (arg1, arg2)" will not work.
function_name () {
	cmd $1 $2
}

function_name "arg1" "arg2"

######## One liner

fun () { echo "This is a function"; echo; }

Export function

SO: call a function defined in bashrc.


function hello() {
   echo "Hello, $1!"
}

export -f hello

Case

TLDP.


case EXPRESSION in CASE1) COMMAND-LIST;; CASE2) COMMAND-LIST;; ... CASEN) COMMAND-LIST;; esac

case "$1" in
	start)
		start
		;;
	 
	stop)
		stop
		;;
	 
	status)
		status anacron
		;;

	restart)
		stop
		start
		;;

	condrestart)
		if test "x`pidof anacron`" != x; then
			stop
			start
		fi
		;;
	 
	*)
		echo $"Usage: $0 {start|stop|restart|condrestart|status}"
		exit 1
esac

Reference.


#!/bin/bash
# This script does a very simple test for checking disk space.

space=`df -h | awk '{print $5}' | grep % | grep -v Use | sort -n | tail -1 | cut -d "%" -f1 -`

case $space in
# 10%-69%
[1-6]*)
  Message="All is quiet."
  ;;
# 70%-89%
[7-8]*)
  Message="Start thinking about cleaning out some stuff.  There's a partition that is $space % full."
  ;;
# 91-98%
9[1-8])
  Message="Better hurry with that new disk...  One partition is $space % full."
  ;;
# 99%
99)
  Message="I'm drowning here!  There's a partition at $space %!"
  ;;
*)
  Message="I seem to be running with an nonexistent amount of disk space..."
  ;;
esac

echo $Message | mail -s "disk report `date`" anny

Use tail -16 /var/spool/mail/anny to see the email.

Subshell

Reference.

# var1 will have the result string.
var1=$(id user)

Commands

alias


alias ll="ls -l"
unalias ll

# Removes All aliases
unalias -a

ulimit


# List current limit for core dump.
ulimit -c

# Set unlimited for core dump.
ulimit -c unlimited

set

Print input lines: -v/-x

StackOverflow.


# -v, -o verbose	Print shell input lines as they are read.
# -x, -o xtrace		Print input lines with variables expanded. "+" is preceded before each line as an indicator.
set -x

Exit on error: -e


# The false command causes the subshell to exit without executing echo one;
# however, echo two is executed because the exit status of the pipeline (false; echo one) | cat is zero.
set -e; (false; echo one) | cat; echo two
# two

tldp.org.


for planet in "Mercury 36" "Venus 67" "Earth 93"  "Mars 142" "Jupiter 483"
do
  set -- $planet  #  Parses variable "planet" and sets positional parameters.
  #  The "--" prevents nasty surprises if $planet is null or begins with a dash.

  #  May need to save original positional parameters, since they get overwritten.
  #  One way of doing this is to use an array,
  #         original_params=("[email protected]")

  echo "$1		$2,000,000 miles from the sun"
  #-------two  tabs---concatenate zeroes onto parameter $2
done

echo

-n	do not print the trailing newline.
-e	enable interpretation of backslash escapes.

#Example
echo -e "This is a test.\nAnother line.\nNow comes the table:\ncol1\tcol2"

tee

SO: sudo tee.

SO: multiple lines.



# Make sure to avoid quotes inside quotes.
echo 'deb blah ... blah' | sudo tee --append /etc/apt/sources.list

# To avoid printing data back to the console:
echo 'deb blah ... blah' | sudo tee --append /etc/apt/sources.list > /dev/null

sudo tee -a /etc/profile.d/maven.sh > /dev/null << EOL
export M2_HOME=/opt/apache-maven-3.1.1
export M2=\$M2_HOME/bin
PATH=\$M2:\$PATH
EOL

export

Export to the environment of subsequently executed commands.

export [-fn] [name[=word]] ...
export -p

[-p]	display a list of exported names.
-f	The names refer to shell functions; otherwise the names refer to shell variables
-n	No longer mark each name for export.

Export not working to change the environment from a bash script?

StackOverflow. Child processes cannot alter their parent's environment. The parent has to choose to alter itself (either with source or eval). StackExchange.


# Method 1.
# sourcefile
export JAVA_HOME=/cygdrive/c/dev/Java/jdk1.5.0_22
export PATH=$JAVA_HOME/bin:$PATH
# and then source it
source sourcefile

# Method 2.
# evalfile.sh
echo "export JAVA_HOME=/cygdrive/c/dev/Java/jdk1.5.0_22"
echo "export PATH=$JAVA_HOME/bin:$PATH"
# and then evaluate it:
eval `evalfile.sh`

eval

eval takes its arguments, concatenates them separated by spaces, and executes the resulting string as Bash code in the current execution environment.
eval in Bash works in essentially the same way as most other languages that have an eval function.
Perhaps the easiest way to think about eval is that it works in the same way as running bash -c "bash code…" from a script, except in the case of eval, the given code is executed in the current shell environment rather than a child process.


for i in 2 3 4; do
 eval a$i=2
 eval echo "\$a$i" 
done
# Output:
# 2
# 2
# 2

for i in 2 3 4; do
 eval a$i=2
 eval echo "$a$i" 
done
# Output:
# $a2
# $a3
# $a4

Redirection

Use app > res.txt 2>&1 rather than app 2>&1 > res.txt. See man bash, redirection section for detail.


ls 2>&1 > dirlist

directs only the standard output to file dirlist, because the standard error was duplicated as standard output before the standard output was redirected to dirlist.


# Redirect stderr to stdout and then stdout to /dev/null:
command 2>&1 >/dev/null | grep 'something'

Mathematics

How to do mathematics in BASH?
For integers,

  1. Arithmetic expansion: \$((EXPR)). Example: num=\$((num+1)), or num=\$((num+=1)). This is a POSIX standard.
  2. Expr utility. Example: num=`expr \$num + 1`. It is a bit slower than the arithmetic expansion.
For floats: num=\$(echo \$num + 1 | bc).
Important notes:
  1. When doing assignment, there should be no whitespace around =. Otherwise the shell will interprete the first word as the name of application to run. Wrong: num= 1, or num =1. Right: num=1.
  2. Both bc and expr expect each number and operator as a separated argument, so whitespace is critical. Wrong: num=`expr \$num+1`. Right: num=`expr \$num + 1`.
Reference here.

Reference.


# Calculate the division and display the result with 5 digits after decimal point.
# Result: 1.25714
echo "scale=5; 0.88 / 0.7" | bc

Declare, typeset

These commands are used to constrain the attribute of variables, e.g. declare one as integer.

To find the doc for declare, you can go to MANpage of bash and read the section BUILTINS. declare -f func is used to print the function definition, if the function is un-defined, it returns error 1 (check with echo $?).

Other examples:

CNBlogs.


declare -i number
# 脚本余下的部分会把"number"当作整数看待.
number=3
echo "Number = $number"     # Number = 3
number=three
echo "Number = $number"     # Number = 0
# 脚本尝试把字符串"three"作为整数来求值(译者注:当然会失败,所以出现值为0).

# Do automatic arithmatics.
n=6/3
echo "n = $n"       # n = 6/3
declare -i n
n=6/3
echo "n = $n"       # n = 2

Variables

$?	Exit status.
$$	ProcessID of the script itself. The $$ variable often finds use in scripts to construct "unique" temp file names. This is usually simpler than invoking mktemp.

Ref: unset var.


var="hehe"

// Set to empty
var=""
// Another way to set it to empty.
var=

// Delete the var.
unset var

History


#!/bin/bash

history 100 # Show the last 100 commands in history.
!95	# Execute the 95th command in history.
!-5 # Execute the previously 5th command.
!! 	# Execute the last command. Same as !-1.
ls -la # a command example.
!l	# Search the command starting with "l". Will execute the ls -la.
!?ls? # Search the command containing the string "ls".
^str1^str2^ # Repeat the last command, replace str1 with str2. Equivalent to !!:s/str1/str2.
!ls:p	# :p is a modifier. This will only print the command but not execute it.
history | grep -i ls	# Combo with grep.

Another good way to use history productively is by typing Ctrl-r (search backwards in history) in the terminal, type and edit the command and then run. Reference in stackoverflow. For complete information, type man bash and find HISTORY EXPANSION.

This reference provides some customizations.

Jobs

Find out how to make detach from terminal.

Reference.


jobs -l # list all jobs.

run.sh &
disown -h # Disown all jobs and let them stand with SigHUP.
exit

nohup run.sh & # Run in background without worring about sigHUP.
exit

nohup nice -n -5 run.sh &

echo "run.sh" | at now + 1 minute # Make run.sh to queue later execution.

Conditional expressions

-b file	:	true if file is a block special file.
-c file	:	true if file is a char special file.
-d file	:	true if file is a dir.
-f file	:	true if file is a regular file.
-h/-L file	:	true if file is a symbolic link.
-p file	:	true if file is a named pipe (FIFO).
-t file	:	true if file descriptor is open and refers to a terminal.
-S file	:	true if file is a socket.

-g file	:	true if file is set-group-id.
-k file	:	true if file's sticky bit is set.
-u file	:	true if file's set-user-id bit is set.

-a/-e file	:	true if file exists.
-s file	:	true if file size is larger than 0.
-r file	:	true if file is readable.
-w file	:	true if file is writable.
-x file	:	true if file is executable.

file1 -nt file2	:	true if file1 is newer than file2 in mtime.
file1 -ef file2	:	true if file1 and file2 refer to the same device and inode numbers.

-z string	:	true if length of string is 0.
string		:	true if length of string is non-zero.
-n string	:	true if length of string is non-zero.
str1 == str2	:	true if the strings are equal.

This will return true only if a variable is set.


# The following two are equal.
[ ! -z "$VAR" ];
[ -n "$VAR" ];

Check whether user exists



if id -u $1 >/dev/null 2>&1; then
	echo "user exists"
else
	echo "user does not exist"
fi

FAQ

What is Login shell

SO: difference between login shell and non-login shell.

The login process tells the shell to behave as a login shell by prepending '-' before the shell's name, e.g. "-bash". So in a login shell, argv[0][0] == '-'.


echo $0
# On MacOS: -csh

A login shell can execute logout command. It is same as exit.

The difference between login shells and non-login ones is in the files they load at initial time.

Bourne shell: /etc/profile and ~/.profile
bash: ~/.bash_profile
zsh: /etc/zprofile and ~/.zprofile
csh: /etc/csh.login and ~/.login

See below for more cases.

Bashrc not loaded automatically

After ssh login, bashrc also not loaded

wcfNote: I just set source ~/.bashrc in .bash_profile. And it works!

Run as another user

StackOverflow.


su wangcf <<'EOF'
cur=${HOME}/gitRepo/ss/

pushd .
cd $cur
nohup python build/lib/ss/local.py &> ~/ss.log &
popd
EOF

Multiline string


str="hehe
haha"

## Here we must use quotes, otherwise the linebreaks will be gone.
echo ""
# hehe
# haha
## The linebreaks are gone...
echo 
# hehe haha

Dash as positional parameter

Note that in this context the "-" is not itself a Bash operator, but rather an option recognized by certain UNIX utilities that write to stdout, such as tar, cat, etc.

Where a filename is expected, - redirects output to stdout (sometimes seen with tar cf), or accepts input from stdin, rather than from a file. This is a method of using a file-oriented utility as a filter in a pipe.


file -
# Input: abc
# Input: Ctrl+D
# Output: standard input:  ASCII text
file -
# Input: #!/bin/bash
# Input: Ctrl+D
# Output: standard input:  Bourne-Again shell script text executable

echo "whatever" | cat -
nc -l -p 12345 | tar xvzf -

Generate random strings and numbers


# Generate a random number with 6 digits.
< /dev/urandom tr -dc 0-9 | head -c6 && echo

SSH and run synclient, we get "Failed to connect to X Server".

LinuxQuestions. For synclient to work, it must use some environment variables set by X. The author grabbed the variables using export -p, and found they are XAUTHORITY and DISPLAY. So set those two vars:


declare -x DISPLAY=":0"
declare -x XAUTHORITY="/home/bob/.Xauthority"

if [ $1 = "on" ]
then
  synclient TouchpadOff=0
fi

if [ $1 = "off" ]
then
  synclient TouchpadOff=1
fi

BASH terminal has unexpected line breaks

Solution

StackExchange.


# It is mostly to do with the size of the window assumed by the terminal is not the same as your actual window size. If you are using bash, you can try this.
shopt  | grep checkwinsize
# If you don't get
# checkwinsize    on
# Then activate it with
shopt -s checkwinsize 7

The issue is often caused by misconfiguring ~/.bashrc not to call /etc/bashrc. Normally, bash loads ~/.bashrc which is expected to call /etc/bashrc, which by default contains shopt -s checkwinsize.

Culprit 1

ArchLinuxBBS. It may be caused by your PS1 setting. Bash thinks you wrap non-printing characters in '\[' and '\]'. So avoid [ and ] in PS1.

Culprit 2

StackOverflow.

This issue happens generally when dumping binary data to the terminal "STDOUT" which when the escape codes received are processed can do anything from change the color of the text, disable echo, even change character set.

# Use this to reset all problematic bash
reset

Greps


# -E: --extended-regexp
# [:space:] stands for a space. And it must reside in another [], thus [[:space:]].
echo "No , come on." | grep -iE 'error|No[[:space:]]'

Bash script not recognizes aliases

StackExchange. look into the bash manpage you find:

Aliases are not expanded when the shell is not interactive, unless the expand_aliases shell option is set using shopt.

# If not running interactively, don't do anything
[ -z "$PS1" ] && return
shopt -s expand_aliases
source ~/.bash_aliases

Bash to parse config

StackOverflow.


[parameters.ini]
    database_user    = user
    database_version = 20110611142248

version=$(awk -F "=" '/database_version/ {print $2}' parameters.ini)

Dir name of script


## One-liner which will give you the full directory name of the script no matter where it is being called from.
# Take /a/b/test.sh for example, $BASH_SOURCE[0] is "/a/b/test.sh".
# "dirname /a/b/test.sh" gives "/a/b".
# dirname only returns parent dir: "dirname /a/b/" returns "/a"
cur=$( cd "$( dirname "" )" && pwd )

## Suppress pushd output.
pushd () {
    command pushd "[email protected]" > /dev/null
}
popd () {
    command popd "[email protected]" > /dev/null
}

Detect OS in bash

Stackoverflow.


###### OSTYPE var.

case $OSTYPE in darwin*) echo I am a Mac ;; esac

if [[ "$OSTYPE" == "linux-gnu" ]]; then
        # ...
elif [[ "$OSTYPE" == "darwin"* ]]; then
        # Mac OSX
elif [[ "$OSTYPE" == "cygwin" ]]; then
        # POSIX compatibility layer and Linux environment emulation for Windows
elif [[ "$OSTYPE" == "msys" ]]; then
        # Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
elif [[ "$OSTYPE" == "win32" ]]; then
        # I'm not sure this can happen.
elif [[ "$OSTYPE" == "freebsd"* ]]; then
        # ...
else
        # Unknown.
fi

### uname

platform='unknown'
unamestr=`uname`
if [[ "$unamestr" == 'Linux' ]]; then
   platform='linux'
elif [[ "$unamestr" == 'FreeBSD' ]]; then
   platform='freebsd'
fi

if [[ $platform == 'linux' ]]; then
   alias ls='ls --color=auto'
elif [[ $platform == 'freebsd' ]]; then
   alias ls='ls -G'
fi

-- dash

Show all directories

Ref.

for d in */ ; do
    echo "$d"
done

Clear all env variables

One quick and clean solution: env -i /path/to/main_script.sh

Ref.


unset $(/usr/bin/env | egrep '^(\w+)=(.*)$' | \
  egrep -vw 'PWD|USER|LANG' | /usr/bin/cut -d= -f1);

Read user input

StackOverflow.


## Equivalent to dos Pause
# -n1: wait for only one key.
# -r: raw mode, which don't allow combined characters like "\" or "^".
# -p: specifies the prompt.
# -p $'prompt': let spaces and escaped characters, rather than simple quote.
# -s: specifies silent mode, and because we don't need keyboard output.
read -n1 -rsp $"Press any key to continue...\n" key

read -sp "Password please: " pass
# Alternatively. Because the new line from Enter is hidden too, you've to print a newline.
stty -echo
read -p "Password please: " pass
stty echo
printf '\n'

# -t5: wait for 5 seconds.
read -t5 -n1 -r -p 'Press any key in the next five seconds...' key
if [ "$?" -eq "0" ]; then
	echo 'A key was pressed.'
else
	echo 'No key was pressed.'
fi

# -d $'\e' specifies escappe as delimiter charater
# -e specifies readline mode.
# -i $'Y' specifies Y as initial text in readline mode.

Execute command as somebody

su somebody <<'EOF'
command1 -p 'parameter with "quotes" inline'
command2 -p 'parameter with "quotes" inline'
command3 -p 'parameter with "quotes" inline'
EOF

sudo -u username command
If you encounter "su: must be run from a terminal", try sudo su somebody <<'EOF'.
If you use <<EOF instead of <<'EOF' then variables inside the here-doc will be expanded.

String operation

References:

String length

Three ways:

${#str}
expr length $str
expr "$str":'.*'
The first two are equivalent of strlen() in C.

stringZ=abcABC123ABCabc

echo ${#stringZ}                 # 15
echo `expr length $stringZ`      # 15
echo `expr "$stringZ" : '.*'`    # 15

Length of matching substring

expr match "$string" '$substring' $substring is a regular expression. expr "$string" : '$substring' $substring is a regular expression.

stringZ=abcABC123ABCabc
#       |------|
#       12345678

echo `expr match "$stringZ" 'abc[A-Z]*.2'`   # 8
echo `expr "$stringZ" : 'abc[A-Z]*.2'`       # 8

Substring Extraction

${string:position}	Extracts substring from $string at $position.
${string:position:length}
	Extracts $length characters of substring from $string at $position.

stringZ=abcABC123ABCabc
#       0123456789.....
#       0-based indexing.

echo ${stringZ:0}                            # abcABC123ABCabc
echo ${stringZ:1}                            # bcABC123ABCabc
echo ${stringZ:7}                            # 23ABCabc
echo ${stringZ:7:3}                          # 23A

# Is it possible to index from the right end of the string?
echo ${stringZ:-4}                           # abcABC123ABCabc
echo ${stringZ:-}                            # abcABC123ABCabc
# Defaults to full string, as in ${parameter:-default}.
# However . . .
echo ${stringZ:(-4)}                         # Cabc 
echo ${stringZ: -4}                          # Cabc
# Now, it works.
# Parentheses or added space "escape" the position parameter.

Generating an 8-character "random" string

if [ -n "$1" ]  #  If command-line argument present,
then            #+ then set start-string to it.
  str0="$1"
else            #  Else use PID of script as start-string.
  str0="$$"
fi

POS=2  # Starting from position 2 in the string.
LEN=8  # Extract eight characters.

str1=$( echo "$str0" | md5sum | md5sum )
#  Doubly scramble     ^^^^^^   ^^^^^^
#+ by piping and repiping to md5sum.

randstring="${str1:$POS:$LEN}"
# Can parameterize ^^^^ ^^^^

echo "$randstring"
exit $?

# ./rand-string.sh my-password
# 1bdd88c4

Substring Removal

${string#substring}
Deletes shortest match of $substring from front of $string.
${string##substring}
Deletes longest match of $substring from front of $string.

${string%substring}
Deletes shortest match of $substring from back of $string.
${string%%substring}
Deletes longest match of $substring from back of $string.

stringZ=abcABC123ABCabc
#       |----|          shortest
#       |----------|    longest

echo ${stringZ#a*C}      # 123ABCabc
# Strip out shortest match between 'a' and 'C'.

echo ${stringZ##a*C}     # abc
# Strip out longest match between 'a' and 'C'.

# You can parameterize the substrings.
X='a*C'
echo ${stringZ#$X}      # 123ABCabc
echo ${stringZ##$X}     # abc

stringZ=abcABC123ABCabc
#                    ||     shortest
#        |------------|     longest

echo ${stringZ%b*c}      # abcABC123ABCa
# Strip out shortest match between 'b' and 'c', from back of $stringZ.

echo ${stringZ%%b*c}     # a
# Strip out longest match between 'b' and 'c', from back of $stringZ.

# Rename all filenames in $PWD with "TXT" suffix to a "txt" suffix.
# For example, "file1.TXT" becomes "file1.txt" . . .

SUFF=TXT
suff=txt

for i in $(ls *.$SUFF)
do
  mv -f $i ${i%.$SUFF}.$suff
  #  Leave unchanged everything *except* the shortest pattern match
  #+ starting from the right-hand-side of the variable $i . . .
done ### This could be condensed into a "one-liner" if desired.

Substring Replacement

stringZ=abcABC123ABCabc

# Replaces first match of 'abc' with 'xyz'.
echo ${stringZ/abc/xyz}	# xyzABC123ABCabc
# Replaces all matches of 'abc' with # 'xyz'.
echo ${stringZ//abc/xyz}	# xyzABC123ABCxyz
# The string itself is not altered!
echo "$stringZ"               # abcABC123ABCabc

# Can the match and replacement strings be parameterized?
match=abc
repl=000
echo ${stringZ/$match/$repl}  # 000ABC123ABCabc
#              ^      ^         ^^^
echo ${stringZ//$match/$repl} # 000ABC123ABC000
# Yes!          ^      ^        ^^^         ^^^

# A simple deletion takes place if no $replacement string is supplied.
echo ${stringZ/abc}           # ABC123ABCabc
echo ${stringZ//abc}          # ABC123ABC

# Replaces front-end match of 'abc' with 'XYZ'.
echo ${stringZ/#abc/XYZ}          # XYZABC123ABCabc
# Replaces back-end match of 'abc' with 'XYZ'.
echo ${stringZ/%abc/XYZ}          # abcABC123ABCXYZ