Bash

Variables

  • No spaces between the variable name and the value to be assigned
DATE_STRING=$(date +"%Y-%m-%d.%0k.%M.%S")
echo "${DATE_STRING}" 
Bash parameters and parameter expansions
http://www-128.ibm.com/developerworks/linux/library/l-bash-parameters.html?ca=dgr-lnxw02LinuxBashParameters
Using wget in script
http://www.linux.com/articles/59457?tid=47&tid=13

Reference: http://wiki.bash-hackers.org/scripting/newbie_traps

Conditional Statements

NOTE: Spacing is very important on the IF statement line.

IF syntax

#!/bin/bash
if [ true ]; then
  echo "Always true"
fi
# -------------------- OR --------------------
# You can also move then down as follows.
if [ 1 ]
then
  echo "Always true"
fi


IF else syntax

#!/bin/bash
str_1="String 1"
str_2="String 2"
if [ "$str_1" = "$str_2" ]; then
  echo "Both strings are equal."
else
  echo "Both strings are not equal."
fi

Check file existence

# If file exists
if [ -f testfile ]
then
  echo testfile exists!
fi
 
# If file doesn't exist
if [ ! -f /tmp/foo.txt ]; then
  echo "File not found!"
fi
 
# For directory, us -d instead of -f.

Check for missing arguments

# If no argument, display error message and exit script.
if [ $# -eq 0 ]; then
  echo "ERROR: Year argument is missing! Supply year argument to the batch file."
  exit 1;
fi

IF condition using return value of commands

# Don't encapsulate IF condition within []
if echo `uname -m` | grep -q "x86_64" 
then
  echo "64-bit"
else
  echo "32-bit"  
fi
Bash File Testing
 
-b filename - Block special file
-c filename - Special character file
-d directoryname - Check for directory existence
-e filename - Check for file existence, regardless of type (node, directory, socket, etc.)
-f filename - Check for regular file existence not a directory
-G filename - Check if file exists and is owned by effective group ID.
-g filename - true if file exists and is set-group-id.
-k filename - Sticky bit
-L filename - Symbolic link
-n string is not null
-O filename - True if file exists and is owned by the effective user id.
-r filename - Check if file is a readable
-S filename - Check if file is socket
-s filename - Check if file is nonzero size
-u filename - Check if file set-user-id bit is set
-w filename - Check if file is writable
-x filename - Check if file is executable
 
http://www.tldp.org/LDP/abs/html/fto.html
http://www.tldp.org/LDP/abs/html/ops.html

Expressions table: TODO

Reference:

Extract filehosters links from webpage

The script below will extract all links of file hosters. Here is a breakdown of what it does:

  1. Break down string that starts with http as a new line.
  2. Pick up lines that contain http.
  3. Remove double quote and any character that follows.
  4. Remove space and any character that follows.
  5. Remove > symbol and any character that follows.
  6. Remove " and any character that follows.
  7. Remove ]] and any character that follows.
  8. Remove < and any character that follows.
#!/bin/bash
sed -e 's/http/\nhttp/gI' $1 \
 | grep -i http \
 | sed -e 's/".*//' \
 | sed -e 's/ .*//' \
 | sed -e 's/<.*//' \
 | sed -e 's/&quot;.*//' \
 | sed -e 's/]].*//' \
 | sed -e 's/&lt;.*//' \
 | grep -Ei "depositfiles|rapidshare|megaupload|qshare|uploadbox|letitbit|storage.to|shareflare|multiupload|mediafire|sendspace|kewlshare|uploading.com" \
 | uniq > $1.out
 
# Unwanted file hosters:
# filesonic|fileserve|hotfile|ul.to|uploaded|easy-share|oron.com|sharingmatrix|


Converted to run in DOS

REM DOS version
sed -e "s/http/\nhttp/gI" %1 ^
 | grep -i http ^
 | sed -e "s/\x22.*//" ^
 | sed -e "s/ .*//" ^
 | sed -e "s/\x3C.*//" ^
 | sed -e "s/&quot;.*//" ^
 | sed -e "s/]].*//" ^
 | sed -e "s/&lt;.*//" ^
 | grep -Ei "depositfiles|rapidshare|megaupload|qshare|uploadbox|letitbit|storage.to|shareflare|multiupload|mediafire|sendspace|kewlshare|uploading.com" ^
 | uniq > %1.out

REM filesonic|fileserve|hotfile|ul.to|uploaded|easy-share|oron.com|sharingmatrix|

Loops

For loop

#!/bin/bash
# Display each filename returned by ls.
for i in $( ls -r ); do
    echo "$i"
done
# -------------------- OR --------------------
# You can also move 'do' down
for i in $( ls -r )
do
    echo "$i"
done


C-style for loop

#!/bin/bash
# Display from 0 to 20.
MAX=20
for ((i=0; i <= MAX ; i++))  # Double parentheses, and "MAX" with no "$".
do
    echo $i
done


While loop

#!/bin/bash 
# Display 0 to 10.
i=0
while [  $i -lt 10 ]; do
    echo $i
    let i+=1 # Same as 'i=i+1'
done


Until loop

#!/bin/bash 
# Display 10 to 0.
i=10
until [  $i -lt 0 ]; do
    echo $i
    let i=i-1 # Same as 'i-=1'
done

For loop with file names with spaces

#!/bin/bash
 
# Changing internal field separator($IFS) to accept spaces
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
for f in $( ls -r )
do
  echo "$f"
done
 
# Restore $IFS
IFS=$SAVEIFS

Loop through array list

#!/bin/bash
 
# List of animal names
animalNames=( cat dog fish )
animalNames+=( bird )
for animal in "${animalNames[@]}"
do
  echo "********  ${animal}  ********"
done

Save data in array

LS_ARRAY=()
while IFS='' read -r LINE || [[ -n "$LINE" ]]; do
 
  LS_ARRAY+=("${LINE}")
  ((linecount++))
 
done < <( ls * )
echo "total number of lines: $linecount"
 
# http://mywiki.wooledge.org/BashFAQ/024

Use indirection to access argument values

for (( i=1; i<=$#; i++)); do
    echo "${!i}"
done
# http://unix.stackexchange.com/a/261186

Print the full path of the current directory

pwd

Read input from file

#!/bin/bash
cat myfilename.txt |     # Supply input from a file.
while read line          # As long as there is another line to read ...
do
        echo "$line"
done
 
# OR: Last line will be read.
cat myfilename.txt |     # Supply input from a file.
while IFS='' read -r LINE || [[ -n "$LINE" ]]; do
  echo ${LINE}
done

http://stackoverflow.com/a/10929511

Note: If you add a counter within the above loop will not work, see http://stackoverflow.com/questions/16854280/modifying-variable-inside-while-loop-is-not-remembered

Redirection

#Redirect stdout and stderr to a file
yourCommandName &> toFile.log
 
# Redirect and append both stdout and stderr to a file.
yourCommandName >> toFile.log 2>&1
 
 
cat > /tmp/somefile <<EOF
Write the following text below
that span multiple lines
in /tmp/somefile.
EOF

Remove duplicate lines in file

gawk " !x[$0]++" inputfilename.txt > outputfilename.txt

Set MP3 filename as the song title

  1. You have to install id3v2.
    aptitude -y install id3v2
     
  2. Run the following script in directories that contain MP3 files.
    #!/bin/bash
    # Description: For each MP3 file, set the filename as the song title
    for i in *.mp3; do
      SONG="$i"
      id3v2 --song "$SONG" "$i"
    done
     
  3. Or add also the order number in the comment.
    #!/bin/bash
    # Description: For each MP3 file, set the filename as the song title and add the order number in the comment.
    idx=1
    for i in *.mp3; do
      SONG="$i"
      id3v2 --song "$SONG" --comment "${idx}" "$i"
     
      let idx=idx+1
    done
     

Layout of ID3v1

Field Length Description
header 3 TAG
title 30 30 characters of the title
artist 30 30 characters of the artist name
album 30 30 characters of the album name
year 4 A four-digit year
comment 28 or 30 The comment
zero-byte 1 If a track number is stored, this byte contains a binary 0.
track 1 The number of the track on the album, or 0. Invalid, if previous byte is not a binary 0.
genre 1 Index in a list of genres, or 255

Sort directories by size

# (A)
# Commands used: du, sort, awk
# It is fast.
du --max-depth=1 -k * | sort -nr | awk '{ if($1>=1024*1024) {size=$1/1024/1024; unit="G"} else if($1>=1024) {size=$1/1024; unit="M"} else {size=$1; unit="K"}; if(size<10) format="%.1f%s"; else format="%.0f%s"; res=sprintf(format,size,unit); printf "%-8s %s\n",res,$2 }'
 
 
# (B)
# Commands used: du, sort, cut, xargs
# It is slow because it uses "du" twice.
du --max-depth=1 -k | sort -nr | cut -f2 | xargs -d '\n' du -sh 2>/dev/null
 
# (C)
# Commands used: du, sort
# It requires GNU coreutils >= 7.5 because of 'sort -h'
du --max-depth=4 --human-readable | sort -h -r

Reference: http://ubuntuforums.org/showthread.php?t=885344

or simply use the application, NCurses Disk Usage

String Manipulation

Length of string

#!/bin/bash
str="What is the length of this string?"
echo "The length is ${#str}"
# Output:
# The length is 34


Search and Replace string

#!/bin/bash
str="1st dog. 2nd dog. 3r dog"
 
#Replace the 1st occurrence: ${string/find/replace}
str=${str/dog/cat}
echo "${str}"
# Output:
# 1st cat. 2nd dog. 3r dog
 
#Replace all occurrences: ${string//find/replace}
str=${str//dog/cat}
echo "${str}"
# Output:
# 1st cat. 2nd cat. 3r cat
#http://www.arachnoid.com/linux/shell_programming.html

Test if file/directory exists

#!/bin/bash
 
# Create a file
touch myfile.txt
 
# Create a directory
mkdir mydirectory
 
 
if [ -f myfile.txt ]
then
  echo "myfile.txt exists!"
fi
 
if [ -d mydirectory ]
then
  echo "mydirectory exists!"
fi

Useful commands

ping -c 1 -w 1 -q
-c <x>: Stop sending after <x> packets.
-w <x>: Time to wait for a response, in seconds.
-q: Display summary line.

Check the internet connection

#!/bin/bash
# Description: Check every 5 minutes whether it is connected to the internet or not
#  and store the result into a file.
 
iconnectionDir=/root/iconnection
mkdir ${iconnectionDir}
 
let SecondsInOneDay=60*60*24
 
# Predefined startup variables.
TimeInSeconds=`date +%s`
CurrentDate=`date +%Y-%m-%d_%H.%M.%S`
InternetConnectionTest="InternetConnectionTest_"$CurrentDate".txt"
 
# Infinite loop
while( true )
do
  sleep 5m
 
  #---------------------------------------------------------------------------
  # Store the internet connection status of the whole day into 1 file per day.
  # Update the output filename if the number of elapsed seconds is greater than 1 day.
  #---------------------------------------------------------------------------
  CurrentTimeInSeconds=`date +%s`
  let DifferenceInSeconds=$CurrentTimeInSeconds-$TimeInSeconds
 
  # Change the output filename if the number of elapsed seconds is greater than 1 day.
  if [ $DifferenceInSeconds -gt $SecondsInOneDay ]
  then
    CurrentDate=`date +%Y-%m-%d_%H.%M.%S`
    InternetConnectionTest="InternetConnectionTest_"$CurrentDate".txt"
    TimeInSeconds=$CurrentTimeInSeconds
  fi
 
  #--------------------------------------------------
  # Test internet connection.
  #--------------------------------------------------
  # Ping timeout = 5 seconds
  if ping -c1 -w 5 -q 4.2.2.2
  then
    echo "`date +%Y-%m-%d_%H.%M.%S` Connected" >> ${iconnectionDir}/$InternetConnectionTest
  else
    echo "`date +%Y-%m-%d_%H.%M.%S` Disconnected" >> ${iconnectionDir}/$InternetConnectionTest
  fi
done
exit 0

Decompress all rar files and delete them if ran successfully

#!/bin/bash
# Description: Decompress all *.rar files and delete them if ran successfully.
#              Errors are logged in err.log.
# Author: Xuan Ngo
# Usage: sh thisScript.sh
# Warning: It doesn't handle multiple part rar files.
 
# Change $IFS so that it handles filename with spaces.
SAVEIFS=$IFS
IFS=$(echo -en "\n\b")
 
 
ERROR_LOG=err.log
 
for filename in $(ls *.rar)
do
 
  # Extract and send error text in ${ERROR_LOG}
  rar x -y "${filename}" 2> ${ERROR_LOG}
  ERROR_LOG_SIZE=`ls -sh ${ERROR_LOG} | gawk '{print $1}'`
 
  # If there is no error.
  if [ "${ERROR_LOG_SIZE}" == "0" ]; then
    rm -f "${filename}"
  fi
 
done
 
# Change $IFS back to the original value.
IFS=$SAVEIFS



For unzip, use the following command:

unzip -oq

Generate MD5 hash of all files under the provided path and all its subdirectories

#!/bin/bash
 
# Description: Generate MD5 hash of all files under the provided path and all its subdirectories.
# Author: Xuan Ngo
# Usage: thisScriptName.sh [path]
#        thisScriptName.sh .
#        thisScriptName.sh /some/absolute/path
#        thisScriptName.sh some/relative/path
# Output: YYYY-MM-DD HH:MM:SS | MD5 hash | Size in bytes | Filename
#############################################################################################
 
ProcessingPath="$1"
 
find $ProcessingPath -type f | while read filename ; do
 
  lFileSize=`du -sb "$filename" | gawk '{print $1}'`
  # Output: YYYY-MM-DD HH:MM:SS | MD5 hash | Size in bytes | Filename
  md5sum "$filename" | gawk -v hFileName="$filename" -v filesize="$lFileSize" '{print strftime("%Y-%m-%e %H:%M:%S"),"|", $1, "|", filesize, "|", hFileName}'
done

Get date and time

date_time=`date +%Y-%m-%d_%H.%M.%S`
echo ${date_time}

Get filename without extension

#!/bin/bash
set -e
# Description: Filename informations.
 
# Create test data.
TMP_DIR="./tmp some directory"
mkdir -p "${TMP_DIR}"
TMP_FILE="${TMP_DIR}/./tmp.some.file.txt"
echo "some text" > "${TMP_FILE}"
 
 
# Get different information of file.
FULL_PATH=$(realpath "${TMP_FILE}")
FILENAME=$(basename "${FULL_PATH}")
DIR_OF_FILE=$(dirname "${FULL_PATH}")
 
echo "Full path = ${FULL_PATH}"
echo "Filename  = ${FILENAME}"
echo "Directory = ${DIR_OF_FILE}"
echo "Don't process file extension. Otherwise, it will not be cross platform."
echo "File type = $(file "${FULL_PATH}")"
 
 
# Clean up.
rm -rf "${TMP_DIR}"


To clean up:

# http://tldp.org/LDP/GNU-Linux-Tools-Summary/html/mass-rename.html
for i in *.JPG; 
do mv $i `basename $i JPG`jpg; 
done 
 
# or
for i in *.JPG; 
do mv $i ${i%%.JPG}.jpg; 
done

Padding zeros using PRINTF command

# Description:
#   -Example below shows how to pad zeroes to a number using PRINTF command.
#   -It will pad zeroes to the number until the total length is 5.
SHORT_NUBMER=13
PADDING_RESULT=$(printf %05d ${SHORT_NUBMER})
echo ${PADDING_RESULT}
 
LONG_NUBMER=343
PADDING_RESULT=$(printf %05d ${LONG_NUBMER})
echo ${PADDING_RESULT}
 
# OUTPUT:
#  00013
#  00343

Reference: http://wiki.bash-hackers.org/commands/builtin/printf

Who Is Running The Script

#!/bin/bash
 
# Description: Who is running this script.
# Author: Xuan Ngo
#############################################
 
# Who Is Running The Script.
WHOAMI=`/usr/bin/whoami`
echo "$WHOAMI is running this script!"
 
if [ $WHOAMI != "root" ]; then
    echo
    echo "You must be root to run this!"
    echo
    exit 1
fi