Linux Shell Scripting
Table of Contents
- Shell Basics
- Variables
- Control Structures
- Functions
- String Operations
- File Operations
- Process Management
- Networking
- System Administration
- Common Patterns
- Processes, Signals & Process Lifecycle
- Bash Robustness & Scripting Pitfalls
- Files, Permissions & Filesystems
- Services, Scheduling & Logs
- Performance & Memory
- Senior and Principal Production Scenarios
Shell Basics
Shebang and Script Structure
#!/bin/bash
# ============================================
# Basic script structure
# ============================================
# Enable strict mode
set -euo pipefail
IFS=$'\n\t'
# Comments
echo "Hello, World!"
# Run script
# chmod +x script.sh
# ./script.sh
# bash script.sh
Variables
# ============================================
# VARIABLES
# ============================================
# Define variables (no spaces around =)
NAME="John"
AGE=30
PI=3.14
# Read-only variable
readonly CONSTANT="immutable"
# CONSTANT="changed" # Error!
# Environment variables
export APP_ENV="production"
echo $APP_ENV
# Array variables
FRUITS=("apple" "banana" "cherry")
echo "${FRUITS[0]}" # First element
echo "${FRUITS[@]}" # All elements
echo "${#FRUITS[@]}" # Array length
# Associative arrays (Bash 4+)
declare -A user_info
user_info["name"]="John"
user_info["email"]="john@example.com"
echo "${user_info[name]}"
Input/Output
# Read user input
echo "Enter your name:"
read NAME
echo "Hello, $NAME"
# Read with prompt
read -p "Enter password: " -s PASSWORD
echo "" # New line after password
# Command substitution
CURRENT_DATE=$(date +%Y-%m-%d)
CURRENT_TIME=$(date +%H:%M:%S)
echo "Date: $CURRENT_DATE Time: $CURRENT_TIME"
# Process substitution
diff <(sort file1.txt) <(sort file2.txt)
Control Structures
Conditionals
# ============================================
# IF-ELSE STATEMENTS
# ============================================
# Basic if
if [ "$AGE" -ge 18 ]; then
echo "Adult"
fi
# If-else
if [ "$AGE" -lt 18 ]; then
echo "Minor"
else
echo "Adult"
fi
# If-elif-else
if [ "$AGE" -lt 13 ]; then
echo "Child"
elif [ "$AGE" -lt 20 ]; then
echo "Teenager"
else
echo "Adult"
fi
# String comparisons
if [ "$NAME" == "John" ]; then
echo "Hello John!"
fi
if [ -z "$EMPTY_STRING" ]; then
echo "String is empty"
fi
if [ -n "$NON_EMPTY" ]; then
echo "String is not empty"
fi
# File tests
if [ -f "file.txt" ]; then
echo "Regular file exists"
fi
if [ -d "directory" ]; then
echo "Directory exists"
fi
if [ -r "file.txt" ]; then
echo "File is readable"
fi
if [ -w "file.txt" ]; then
echo "File is writable"
fi
if [ -x "script.sh" ]; then
echo "File is executable"
fi
# Logical operators
if [ "$AGE" -ge 18 ] && [ "$AGE" -lt 65 ]; then
echo "Working age adult"
fi
if [ "$STATUS" == "active" ] || [ "$STATUS" == "pending" ]; then
echo "Access granted"
fi
if [ ! "$DISABLED" == "true" ]; then
echo "Enabled"
fi
Case Statement
# ============================================
# CASE STATEMENTS
# ============================================
case "$OS" in
"linux")
echo "Linux detected"
;;
"darwin")
echo "macOS detected"
;;
"windows")
echo "Windows detected"
;;
*)
echo "Unknown OS"
;;
esac
# Pattern matching (+(...) needs extglob)
shopt -s extglob
case "$INPUT" in
[0-9])
echo "Single digit"
;;
+([0-9]))
echo "Multiple digits"
;;
+([a-zA-Z]))
echo "Letters only"
;;
*)
echo "Other"
;;
esac
Loops
# ============================================
# FOR LOOPS
# ============================================
# Basic for loop
for i in 1 2 3 4 5; do
echo "Number: $i"
done
# For with range
for i in {1..10}; do
echo $i
done
# For with step
for i in {0..10..2}; do # 0, 2, 4, 6, 8, 10
echo $i
done
# For with array
for fruit in "${FRUITS[@]}"; do
echo "Fruit: $fruit"
done
# C-style for loop
for ((i=0; i<10; i++)); do
echo $i
done
# For with command output (breaks on filenames with spaces; prefer the glob below)
for file in $(ls *.txt); do
echo "Processing: $file"
done
# For with globbing
for file in /path/to/*.txt; do
echo "File: $file"
done
# ============================================
# WHILE LOOPS
# ============================================
# Basic while
count=1
while [ $count -le 5 ]; do
echo "Count: $count"
((count++))
done
# While with read
while IFS= read -r line; do
echo "Line: $line"
done < "input.txt"
# While with pipe
cat "input.txt" | while read line; do
echo "Line: $line"
done
# Infinite while with break
while true; do
echo "Running..."
sleep 1
if [ "$STOP" == "true" ]; then
break
fi
done
# ============================================
# UNTIL LOOPS
# ============================================
count=1
until [ $count -gt 5 ]; do
echo "Count: $count"
((count++))
done
Functions
Function Definitions
# ============================================
# FUNCTIONS
# ============================================
# Basic function
hello() {
echo "Hello!"
}
# Function with parameters
greet() {
local name=$1
local greeting=${2:-"Hello"} # Default value
echo "$greeting, $name!"
}
# Function with return
is_even() {
local num=$1
if ((num % 2 == 0)); then
return 0 # Success (true)
else
return 1 # Failure (false)
fi
}
# Function with output
get_date() {
date +%Y-%m-%d
}
# Function with multiple outputs
get_stats() {
local min=$1
local max=$2
local sum=$((min + max))
local avg=$((sum / 2))
echo "$min $max $sum $avg"
}
# Calling functions
hello
greet "John"
greet "Jane" "Hi"
if is_even 4; then
echo "Even"
fi
CURRENT_DATE=$(get_date)
read min max sum avg <<< $(get_stats 10 20)
echo "Sum: $sum, Average: $avg"
Function Examples
# ============================================
# UTILITY FUNCTIONS
# ============================================
# Check if command exists
command_exists() {
command -v "$1" >/dev/null 2>&1
}
# Check if running as root
is_root() {
[ "$EUID" -eq 0 ]
}
# Confirm action
confirm() {
local prompt=${1:-"Continue?"}
local response
read -p "$prompt [y/N] " response
case "$response" in
[yY][eE][sS]|[yY])
return 0
;;
*)
return 1
;;
esac
}
# Print error and exit
error_exit() {
echo "ERROR: $1" >&2
exit 1
}
# Print with timestamp
log() {
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1"
}
# Usage
if ! command_exists docker; then
error_exit "Docker is required but not installed"
fi
log "Starting process..."
if confirm "Continue?"; then
echo "Continuing..."
fi
String Operations
# ============================================
# STRING OPERATIONS
# ============================================
STR="Hello, World!"
# String length
echo ${#STR} # 13
# Substring
echo ${STR:7} # World!
echo ${STR:7:5} # World
echo ${STR: -6} # World! (the space before -6 is required)
echo ${STR: -6:5} # World
# Find and replace
echo ${STR/World/Linux} # Hello, Linux!
echo ${STR//o/O} # HellO, WOrld!
echo ${STR/#Hello/Hi} # Hi, World!
# Remove pattern
echo ${STR%World*} # Hello,
echo ${STR%%,*} # Hello
# Case conversion
echo ${STR,,} # hello, world!
echo ${STR^^} # HELLO, WORLD!
# String splitting
IFS=' ' read -ra WORDS <<< "one two three"
for word in "${WORDS[@]}"; do
echo "$word"
done
# String concatenation
A="Hello"
B="World"
C="$A $B"
echo "$C" # Hello World
File Operations
# ============================================
# FILE OPERATIONS
# ============================================
# Read file line by line
while IFS= read -r line; do
echo "$line"
done < "file.txt"
# Read file into array
mapfile -t lines < "file.txt"
printf '%s\n' "${lines[@]}"
# Write to file
echo "Content" > file.txt
echo "Appended" >> file.txt
# Here documents
cat > config.txt << EOF
database_host=localhost
database_port=5432
database_name=mydb
EOF
# Check file properties
[ -f "file.txt" ] && echo "Regular file"
[ -d "dir" ] && echo "Directory"
[ -L "link" ] && echo "Symbolic link"
[ -s "file.txt" ] && echo "Non-empty file"
# Get file info
stat file.txt
file file.txt
wc -l file.txt
# Find files
find /path -name "*.txt" -type f
find /path -mtime -7 -type f # Modified in last 7 days
find /path -size +100M -type f # Larger than 100MB
find /path -empty -type f # Empty files
# xargs
find . -name "*.txt" | xargs rm
find . -name "*.txt" -print0 | xargs -0 rm
# Sed basics
sed 's/old/new/g' file.txt # Replace all
sed -i 's/old/new/g' file.txt # In-place edit
sed '1d' file.txt # Delete first line
sed -n '5p' file.txt # Print line 5
# Awk basics
awk '{print $1}' file.txt # Print first column
awk -F: '{print $1}' /etc/passwd # Use : as delimiter
awk 'NR==5' file.txt # Print line 5
awk '/pattern/ {print $0}' file.txt
Process Management
# ============================================
# PROCESS MANAGEMENT
# ============================================
# List processes
ps aux
ps -ef
top
htop
# Kill process
kill PID
kill -9 PID # Force kill
pkill process_name
# Background processes
./script.sh &
jobs
fg %1 # Bring to foreground
bg %1 # Send to background
# Process substitution
diff <(command1) <(command2)
# Subshell
(cd /path && ./script.sh)
# Nohup (run after logout)
nohup ./script.sh > output.log 2>&1 &
# Screen (persistent sessions)
screen -S mysession
screen -r mysession
screen -X -S mysession quit
# Tmux
tmux new -s mysession
tmux attach -t mysession
tmux kill-session -t mysession
# Check if process running
if pgrep -x "nginx" > /dev/null; then
echo "Nginx is running"
fi
# Wait for process
wait $PID
# Timeout
timeout 10 ./script.sh
Networking
# ============================================
# NETWORKING
# ============================================
# Check connectivity
ping -c 4 google.com
curl -I https://example.com
wget --spider https://example.com
# Download
wget https://example.com/file.zip
curl -O https://example.com/file.zip
curl -o output.zip https://example.com/file.zip
# HTTP requests
curl https://api.example.com
curl -X POST https://api.example.com \
-H "Content-Type: application/json" \
-d '{"key": "value"}'
curl -X PUT https://api.example.com \
-H "Authorization: Bearer token" \
-d '{"key": "new_value"}'
# DNS lookup
dig example.com
nslookup example.com
host example.com
# Port scanning
nc -zv host 80 443
nmap -p 80,443 host
# Network interfaces
ip addr
ifconfig
netstat -tuln
ss -tuln
# Firewall (iptables)
iptables -L
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -j DROP
System Administration
# ============================================
# SYSTEM ADMINISTRATION
# ============================================
# System info
uname -a
cat /etc/os-release
uptime
df -h
free -h
# Users
whoami
id
who
last
useradd username
usermod -aG sudo username
passwd username
# Permissions
chmod 755 file.sh
chmod +x file.sh
chown user:group file
chgrp group file
# Services (systemd)
systemctl status nginx
systemctl start nginx
systemctl stop nginx
systemctl restart nginx
systemctl enable nginx
systemctl disable nginx
journalctl -u nginx
journalctl -u nginx -f # Follow logs
# Disk usage
df -h
du -sh /var/log/*
du -ah /path | sort -rh | head -10
# Memory
free -h
vmstat
# CPU
top
htop
mpstat
lscpu
# Cron jobs
crontab -e
# Format: minute hour day month weekday command
# 0 3 * * * /backup.sh # Daily at 3 AM
# */15 * * * * /check.sh # Every 15 minutes
# Log rotation
cat /etc/logrotate.conf
Common Patterns
#!/bin/bash
# ============================================
# BACKUP SCRIPT EXAMPLE
# ============================================
set -euo pipefail
BACKUP_DIR="/backup"
SOURCE_DIR="/data"
DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_FILE="backup_${DATE}.tar.gz"
# Create backup
tar -czf "${BACKUP_DIR}/${BACKUP_FILE}" "$SOURCE_DIR"
# Keep only last 7 backups
cd "$BACKUP_DIR"
ls -t backup_*.tar.gz | tail -n +8 | xargs rm 2>/dev/null || true
echo "Backup created: $BACKUP_FILE"
# ============================================
# DEPLOYMENT SCRIPT EXAMPLE
# ============================================
set -euo pipefail
APP_DIR="/opt/myapp"
RELEASE_DIR="${APP_DIR}/releases/$(date +%Y%m%d%H%M%S)"
CURRENT_LINK="${APP_DIR}/current"
# Create release directory
mkdir -p "$RELEASE_DIR"
# Extract application
tar -xzf /tmp/myapp.tar.gz -C "$RELEASE_DIR"
# Create environment file
envsubst < "${RELEASE_DIR}/.env.template" > "${RELEASE_DIR}/.env"
# Switch symlink
ln -sfn "$RELEASE_DIR" "$CURRENT_LINK"
# Restart service
systemctl restart myapp
echo "Deployed to: $RELEASE_DIR"
# ============================================
# DOCKER CLEANUP SCRIPT
# ============================================
#!/bin/bash
set -euo pipefail
# Remove stopped containers
docker container prune -f
# Remove unused images
docker image prune -a -f
# Remove unused volumes
docker volume prune -f
# Remove unused networks
docker network prune -f
# Remove build cache
docker builder prune -a -f
echo "Docker cleanup complete"
Interview Questions
Q1: What is the difference between $@ and $*?
The difference only shows when they are quoted:
"$@"expands to each argument as a separate word, preserving spaces inside arguments. This is almost always what you want."$*"joins all arguments into a single word, separated by the first character ofIFS.- Unquoted,
$@and$*behave the same: both are split on whitespace and globbed.
Q2: How do you debug a shell script?
- Add
set -xto print commands before execution - Use
bash -x script.sh - Add
set -eto exit on error
Q3: What is the purpose of IFS?
IFS (Internal Field Separator) defines the delimiters used for word splitting. Default is space, tab, and newline.
Q4: How do you prevent variable expansion?
Use single quotes '$VAR' or escape the dollar sign \$VAR.
Q5: What is a here document?
A here document (<<EOF) allows inline file content in scripts, useful for multi-line strings or file generation.
Q6: What is the difference between &&, ||, and ; in bash?
&& runs the next command only if the previous succeeded (exit 0). || runs the next only if the previous failed. ; always runs the next command regardless. Use them for control flow: mkdir -p /tmp/x && echo ok || echo fail.
Q7: How do you handle errors in a shell script?
Use set -e (exit on error), set -u (error on unset variable), set -o pipefail (propagate failure through pipes). Trap EXIT/ERR for cleanup: trap 'cleanup' EXIT. Check exit codes explicitly for commands you must inspect.
Q8: What is the difference between single and double quotes?
Single quotes: literal, no expansion — '$HOME' stays $HOME. Double quotes: expand variables and commands but preserve spaces — "$HOME" becomes /home/user. Rule: quote variables always to avoid word-splitting issues.
Q9: What is command substitution and how does it work?
$(command) or backticks run a command and substitute its output: files=$(ls *.txt). Use $() not backticks (nesting, readability). Capture stderr with 2>&1 when needed.
Q10: How do you create and use functions in bash?
greet() {
local name="$1"
echo "hello $name"
}
greet "world"
local scopes variables; $1, $2 are positional args inside. Functions return via return N or echo output captured by callers.
Q11: What are exit codes and how do you use them?
0 = success, 1-255 = failure (convention: 2 usage, 126 not executable, 127 command not found, 130 SIGINT). Check with $? after a command, or if command; then. Always return meaningful codes from your own scripts.
Q12: How do you read a file line by line?
while IFS= read -r line; do
echo "LINE: $line"
done < input.txt
IFS= preserves leading/trailing whitespace, -r prevents backslash mangling. Avoid cat file | while (subshell loses variables).
Q13: How do you schedule tasks with cron?
crontab -e edits user crons; format: minute hour day month weekday command. Example 0 2 * * * /opt/backup.sh runs at 2am daily. Log output (>> /var/log/x.log 2>&1), set env inside scripts (cron runs minimal env), and watch for timezone.
Q14: What is a shebang and why does it matter?
#!/usr/bin/env bash tells the OS which interpreter runs the script. env finds bash in PATH (portable across systems), versus fixed #!/bin/bash. Scripts without shebang need explicit interpreter invocation.
Q15: How do you check if a command exists?
if command -v docker >/dev/null 2>&1; then
echo "docker installed"
else
echo "missing"
fi
command -v is POSIX-safe. Avoid which (non-standard output, failure semantics).
Q16: What are process management commands?
ps (snapshot), top/htop (live), kill (signal by PID), killall/pkill (by name), jobs/fg/bg (shell jobs), nohup (detach), & (background). Signals: TERM (graceful), KILL (force), HUP (reload configs).
Q17: How do you search and transform text with grep/sed/awk?
grep: filter lines (grep -E 'err|fail' log). sed: stream edit (sed -i 's/old/new/g' file). awk: field-based processing (awk '{print $1, $NF}'). Choose by job: filter → grep, replace → sed, column math/rearrange → awk.
Q18: What is a subshell and when does it matter?
( commands ) runs in a subshell — variable changes don't leak to parent. Pipes and $() also run in subshells. Matters for loops/while reading files: variables assigned inside won't persist outside without process substitution (< <(cmd)) or temp files.
Q19: How do you make a script idempotent and safe?
Check before acting: [ -f file ] || touch file, mkdir -p, command -v x || install. Quote all variables, fail loudly (set -e), never rm -rf without $VAR guard: rm -rf "${DIR:?}". Test in staging, keep backups.
Q20: What is process substitution?
<(cmd) gives a file-like stream: diff <(sort a.txt) <(sort b.txt). >(cmd) captures output to a command. Useful to avoid temp files and subshell variable-loss compared to pipes.
Processes, Signals & Process Lifecycle
Q21: What is the difference between a process and a thread on Linux?
A process has its own virtual address space, file descriptor table, credentials and signal handlers. A thread is an execution context inside a process: threads share the address space, open files and signal handlers, but each has its own stack, registers and thread ID (TID). On Linux both are created by the clone() system call with different sharing flags, and the kernel schedules each thread as its own task.
Practical consequences for backend engineers:
- A crash (for example a segfault) in one thread takes down the whole process.
- Threads communicate through shared memory (fast, but needs locks); processes communicate through pipes, sockets or shared memory segments (safer isolation, more overhead).
- Pre-fork servers (Gunicorn workers, PHP-FPM) use processes for isolation; Java and Go services use threads (or goroutines multiplexed onto threads).
# Show threads (LWP column = thread ID) for a process
ps -eLf | grep '[j]ava'
# Count threads of a process
ls /proc/1234/task | wc -l
# Per-thread CPU view of one process
top -H -p 1234
Follow-up to expect: "Why might a Python service use processes instead of threads?" Because the CPython GIL lets only one thread execute Python bytecode at a time, so CPU-bound work scales with processes, while I/O-bound work can still use threads.
Q22: Explain the common signals. What is the difference between SIGTERM and SIGKILL?
A signal is an asynchronous notification sent to a process. The ones you will be asked about:
| Signal | Number | Default action | Typical use |
|---|---|---|---|
| SIGHUP | 1 | Terminate | Terminal closed; many daemons reload config on it |
| SIGINT | 2 | Terminate | Ctrl-C |
| SIGQUIT | 3 | Core dump | Ctrl-\ ; JVM prints a thread dump |
| SIGKILL | 9 | Terminate | Force kill, cannot be caught or ignored |
| SIGTERM | 15 | Terminate | Polite shutdown request (default for kill) |
| SIGSTOP | 19 (x86) | Stop | Pause, cannot be caught |
| SIGCHLD | 17 (x86) | Ignore | A child exited or stopped |
SIGTERM can be handled: the application can stop accepting new work, finish in-flight requests, flush buffers and close connections. SIGKILL is handled by the kernel; the process gets no chance to clean up, so you can lose buffered writes, leave lock files behind, or abort transactions mid-flight.
kill 4321 # sends SIGTERM
kill -TERM 4321 # same, explicit
kill -HUP "$(cat /run/nginx.pid)" # ask nginx to reload config
kill -9 4321 # last resort
kill -l # list signal names and numbers
When a process dies from a signal, the shell reports exit code 128 + signal number: 130 for SIGINT, 137 for SIGKILL, 143 for SIGTERM. Seeing 137 in a container or CI job is a strong hint of an OOM kill or a forced kill after a timeout.
The correct shutdown order used by systemd, Docker and Kubernetes: send SIGTERM, wait a grace period, then send SIGKILL.
Q23: How do you use trap for cleanup and graceful shutdown in a script?
trap registers a command to run when the shell receives a signal or reaches a pseudo-signal (EXIT, ERR, DEBUG, RETURN). The EXIT trap is the most useful: it runs however the script ends (normal exit, exit 1, or set -e failure), so it is the right place for cleanup.
#!/usr/bin/env bash
set -euo pipefail
workdir=$(mktemp -d)
cleanup() {
rm -rf "$workdir"
}
trap cleanup EXIT
# Convert INT/TERM into a normal exit so the EXIT trap runs once
trap 'echo "interrupted" >&2; exit 130' INT
trap 'echo "terminated" >&2; exit 143' TERM
./long_running_job --out "$workdir" &
child=$!
wait "$child"
Why run the child in the background and wait? Bash does not run a trap while it is waiting for a foreground command to finish; the trap fires only after the command returns. When Bash is blocked in the wait builtin, a trapped signal makes wait return immediately (with a status above 128), so the trap runs right away.
If the script is only a launcher, use exec ./app as the last line instead: the shell is replaced by the app, which then receives signals directly.
Q24: What are zombie and orphan processes?
- Zombie (defunct): a child that has exited but whose parent has not yet called
wait()/waitpid()to collect its exit status. It holds no memory or CPU, only a process table entry (PID). It shows stateZinps. - Orphan: a child whose parent exited first. The kernel re-parents it to PID 1 (or to the nearest process marked as a "child subreaper", such as systemd user managers). A proper init reaps it when it exits.
You cannot kill a zombie, because it is already dead. You fix the parent: make it reap children (handle SIGCHLD or call wait), restart it, or kill it so the zombies are re-parented to init and reaped.
# Find zombies and their parents
ps -eo pid,ppid,stat,cmd | awk '$3 ~ /^Z/'
# Inspect the parent of zombie PID 5678 (the process failing to reap it)
ps -o pid,cmd -p "$(ps -o ppid= -p 5678)"
Why it matters: a few zombies are harmless, but a leak of thousands exhausts the PID space (kernel.pid_max, or a cgroup pids.max limit) and new fork() calls fail with "Resource temporarily unavailable". This happens often in containers where the app runs as PID 1 and never reaps the processes it spawns.
Q25: What does set -euo pipefail actually do, and where does it fall short?
set -e(errexit): exit when a command returns non-zero.set -u(nounset): treat expansion of an unset variable as an error.set -o pipefail: a pipeline's exit status is the status of the last command that failed, instead of only the last command.
It is a good default, but set -e has well-known gaps:
#!/usr/bin/env bash
set -euo pipefail
# 1. Ignored in conditions: if/while tests, left side of && and ||, after !
check() { false; echo "still running"; }
if check; then echo "ok"; fi # prints "still running" then "ok"
# 2. 'local' masks the exit status of a command substitution
f() { local out=$(false); echo "not stopped"; } # local returns 0
g() { local out; out=$(false); } # this one does fail
# 3. Arithmetic that evaluates to 0 returns status 1
count=0
((count++)) || true # without '|| true' this line would end the script
# 4. grep with no match returns 1 and kills the script
matches=$(grep -c "ERROR" app.log || true)
# 5. Unset variable with a safe default under set -u
echo "${OPTIONAL_FLAG:-}"
Senior answer: treat set -euo pipefail as a safety net, not error handling. Check critical commands explicitly, add an ERR trap that prints $LINENO and $BASH_COMMAND, and run shellcheck in CI.
Bash Robustness & Scripting Pitfalls
Q26: Why does quoting matter? Explain word splitting and globbing.
After expanding an unquoted variable or command substitution, Bash performs two more steps:
- Word splitting: the result is split on the characters in
IFS(space, tab, newline by default). - Pathname expansion (globbing): each resulting word containing
*,?or[is matched against filenames.
Double quotes suppress both steps while still expanding $var and $(cmd).
file="My Report.txt"
rm $file # tries to remove "My" and "Report.txt"
rm "$file" # removes the one file
pattern="*.log"
echo $pattern # expands to every .log file in the current directory
echo "$pattern" # prints *.log literally
# Pass all arguments through unchanged
run() { "$@"; } # each argument stays one word
# "$*" joins all arguments into a single word using the first char of IFS
# Arrays: always expand with quotes and [@]
files=("a b.txt" "c.txt")
for f in "${files[@]}"; do echo "$f"; done # two iterations, not three
# Loop over command output safely (instead of: for f in $(ls))
while IFS= read -r -d '' f; do
echo "found: $f"
done < <(find . -name '*.txt' -print0)
Notes:
- Inside
[[ ... ]]Bash does not word-split variables, but quoting is still a good habit; on the right side of==an unquoted variable is treated as a glob pattern. - Assignments (
a=$b) are not word-split, but quoting them costs nothing. shellcheckflags most quoting bugs (SC2086 is the classic "double quote to prevent globbing and word splitting").
Q27: How do you get the exit status of every command in a pipeline?
$? after a pipeline holds only the last command's status (or the last failing one under pipefail). Bash also records every stage in the PIPESTATUS array, which is overwritten by the next command, so copy it immediately.
curl -fsS https://example.com/data.json | jq '.items[]' | wc -l
status=("${PIPESTATUS[@]}")
echo "curl=${status[0]} jq=${status[1]} wc=${status[2]}"
if [[ ${status[0]} -ne 0 ]]; then
echo "download failed" >&2
exit 1
fi
A common real bug: mysqldump mydb | gzip > backup.sql.gz "succeeds" because gzip succeeded even though mysqldump failed, producing a tiny valid-looking archive. set -o pipefail or a PIPESTATUS check catches it.
Q28: Explain here-documents and here-strings, including quoting the delimiter.
name="api"
# Unquoted delimiter: variables and $(...) are expanded
cat <<EOF
service=$name
generated=$(date +%F)
EOF
# Quoted delimiter: body is literal (good for embedding scripts)
cat <<'EOF'
echo "$HOME is not expanded here"
EOF
# <<- strips leading TAB characters (not spaces) so you can indent
if true; then
cat <<-EOF
indented with tabs in the source
EOF
fi
# Writing a root-owned file: the redirection must run as root, so use tee
sudo tee /etc/myapp/app.conf > /dev/null <<'EOF'
listen = 0.0.0.0:8080
workers = 4
EOF
# Here-string: feed a single string to stdin
read -r user host <<< "deploy prod-01"
grep -c "ERROR" <<< "$log_output"
Interview trap: sudo cat > /etc/file <<EOF fails with "Permission denied", because the > redirection is performed by your unprivileged shell before sudo runs. The sudo tee pattern avoids that.
Q29: How do you use find and xargs safely and efficiently?
# Delete .log files older than 7 days (safe with spaces and newlines in names)
find /var/log/myapp -type f -name '*.log' -mtime +7 -print0 | xargs -0 -r rm -f
# Same without xargs: '+' batches many files per rm invocation
find /var/log/myapp -type f -name '*.log' -mtime +7 -exec rm -f {} +
# Built-in delete (GNU and BSD find)
find /var/log/myapp -type f -name '*.log' -mtime +7 -delete
# Run 4 jobs in parallel, one file per job
find ./images -name '*.png' -print0 | xargs -0 -n 1 -P 4 optipng -quiet
# Use the argument in the middle of a command
xargs -I {} curl -fsS -o /dev/null -w '%{http_code} {}\n' {} < urls.txt
# Prune a directory from the search
find . -path ./node_modules -prune -o -name '*.js' -print
Key points:
- Plain
find | xargs rmbreaks on filenames containing spaces or newlines;-print0withxargs -0uses NUL as the separator, which cannot appear in a filename. -r(--no-run-if-empty) stops GNU xargs from running the command once with no arguments.-exec ... {} \;runs one process per file (slow);-exec ... {} +batches.- Always run the
findwithout the action first to review what it will touch.
Q30: How would you find the top client IPs and the endpoints returning 5xx from an access log?
Assuming the nginx/Apache "combined" format, where field 1 is the client IP, field 7 is the path and field 9 is the status code:
# Top 10 client IPs
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -10
# Count of each status code
awk '{print $9}' access.log | sort | uniq -c | sort -rn
# Endpoints producing 5xx, most frequent first
awk '$9 ~ /^5/ {count[$7]++} END {for (p in count) print count[p], p}' access.log \
| sort -rn | head -20
# Requests per minute (timestamp is field 4, like [17/Nov/2026:10:15:32)
awk '{print substr($4, 2, 17)}' access.log | uniq -c | tail -20
# Only lines between two timestamps, then strip query strings from paths
sed -n '/17\/Nov\/2026:10:00/,/17\/Nov\/2026:10:30/p' access.log \
| awk '{split($7, a, "?"); print a[1]}' | sort | uniq -c | sort -rn | head
# Follow live and show only errors
tail -F access.log | grep --line-buffered -E '" 5[0-9]{2} '
What interviewers look for: knowing the sort | uniq -c | sort -rn idiom, using awk associative arrays instead of many greps, and knowing tail -F (follows the file name across log rotation) versus tail -f (follows the original file descriptor). Mention that for anything beyond a quick look you would query structured logs in your log platform.
Q31: How do you create temp files safely and stop two copies of a script running at once?
Temp files: never build names like /tmp/myjob.$$ by hand; they are predictable and can be pre-created by another user (symlink attacks). Use mktemp, which creates the file atomically with mode 0600, and remove it in an EXIT trap.
Single instance: use flock (from util-linux, available on most Linux distros, not installed on macOS by default). The kernel releases the lock automatically when the process exits, even after kill -9, unlike PID files that go stale.
#!/usr/bin/env bash
set -euo pipefail
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT
# Open file descriptor 9 on the lock file and try to take an exclusive lock
exec 9>/var/lock/report-job.lock
if ! flock -n 9; then
echo "another instance is running, exiting" >&2
exit 0
fi
# ... critical section: only one copy of the script gets here ...
generate_report > "$tmpdir/report.csv"
mv "$tmpdir/report.csv" /srv/reports/daily.csv
# crontab entry: wrap the whole command, no script changes needed
*/5 * * * * flock -n /var/lock/sync.lock /usr/local/bin/sync.sh >> /var/log/sync.log 2>&1
Also note the mv at the end: writing to a temp file and renaming it onto the target is atomic on the same filesystem, so readers never see a half-written file. flock only coordinates processes on one host; across several hosts you need a distributed lock (for example a database advisory lock).
Files, Permissions & Filesystems
Q32: Explain Linux permissions, umask, and the setuid, setgid and sticky bits.
Each file has an owner, a group and three permission triplets (user, group, other) of read (4), write (2) and execute (1).
For directories the bits mean something different: r lists names, w creates, renames or deletes entries, and x allows entering and accessing entries by name. Deleting a file needs write permission on the directory, not on the file.
umask removes bits from the default mode at creation time. Files start from 666 and directories from 777:
| umask | New file | New directory |
|---|---|---|
| 022 | 644 (rw-r--r--) | 755 (rwxr-xr-x) |
| 027 | 640 (rw-r-----) | 750 (rwxr-x---) |
| 077 | 600 (rw-------) | 700 (rwx------) |
Special bits:
- setuid (4000) on an executable: it runs with the file owner's privileges (
/usr/bin/passwdis setuid root so users can update/etc/shadow). Linux ignores setuid on interpreted scripts. - setgid (2000) on a directory: new files inherit the directory's group, which is useful for shared team folders. On an executable it runs with the file's group.
- sticky (1000) on a directory: only the file owner (or root) can delete or rename entries, which is why
/tmpisdrwxrwxrwt.
umask # show current mask, e.g. 0022
chmod 750 /opt/app
chmod u=rwx,g=rx,o= /opt/app # same thing, symbolic form
chmod g+s /srv/shared # setgid directory
chmod +t /srv/dropbox # sticky bit
ls -ld /tmp # drwxrwxrwt
# Security audit: find setuid/setgid binaries
find / -xdev -type f \( -perm -4000 -o -perm -2000 \) -ls 2>/dev/null
# Prefer a capability over setuid root to bind port 80 as a normal user
sudo setcap 'cap_net_bind_service=+ep' /usr/local/bin/myserver
getcap /usr/local/bin/myserver
Q33: What is an inode? Explain hard links versus symbolic links.
An inode stores a file's metadata: type, owner, permissions, size, timestamps, link count, and pointers to the data blocks. It does not store the name. A directory is a table mapping names to inode numbers.
- Hard link: another directory entry pointing to the same inode. All names are equal; there is no "original". The data is freed only when the link count reaches 0 and no process has the file open. Hard links cannot cross filesystems and normally cannot point to directories.
- Symbolic (soft) link: a small file whose content is a path. It can cross filesystems and point to directories, but it breaks (dangles) if the target is moved or deleted.
echo "hello" > original.txt
ln original.txt hard.txt # hard link
ln -s original.txt soft.txt # symlink
ls -li original.txt hard.txt soft.txt # same inode number for original and hard
stat -c '%h links, inode %i' original.txt
rm original.txt
cat hard.txt # still works: inode still has a link
cat soft.txt # fails: "No such file or directory"
Why it matters in backend work:
- Zero-downtime deploys switch a
currentsymlink to a new release directory (ln -sfn). mvwithin one filesystem is a rename: same inode, atomic. Across filesystems it is a copy then delete.- "Deleted" files still held open by a process keep consuming disk, because the inode is not freed (see the disk-full scenario later in this page).
Q34: What are file descriptors? Explain redirection order and ulimit -n.
A file descriptor (fd) is an integer index into a process's table of open files, sockets, pipes and devices. By convention 0 is stdin, 1 is stdout and 2 is stderr. You can list them under /proc/<pid>/fd.
# Redirections are applied left to right
cmd > out.log 2>&1 # stdout to file, then stderr to where stdout points: both in file
cmd 2>&1 > out.log # stderr to the terminal (old stdout), then stdout to file
cmd &> out.log # Bash shorthand for the first form
cmd 2>/dev/null # discard errors only
# Open, use and close a custom descriptor
exec 3> /tmp/trace.log
echo "debug info" >&3
exec 3>&-
# Inspect a running process
ls -l /proc/1234/fd | head
ls /proc/1234/fd | wc -l # number of open fds
grep 'open files' /proc/1234/limits # its soft and hard limit
# Limits for the current shell
ulimit -Sn # soft limit
ulimit -Hn # hard limit
ulimit -n 65536 # raise soft limit (up to the hard limit)
Every socket is a file descriptor, so a busy server with many keep-alive connections can hit the default soft limit of 1024 and fail with EMFILE: Too many open files.
Where limits come from: /etc/security/limits.conf is applied by PAM to login sessions; it does not apply to systemd services. For a service, set LimitNOFILE= in the unit file. The system-wide ceiling is fs.file-max.
Q35: The disk shows free space but writes fail with "No space left on device". Why?
Two different resources can run out:
- Blocks (bytes): check with
df -h. - Inodes (one per file or directory, fixed when an ext4 filesystem is created): check with
df -i.
Millions of tiny files (session files, cache entries, mail queues, unrotated per-request logs) can use up all inodes while plenty of bytes remain free. A third cause is ext4's reserved blocks (5% by default, kept for root), so a non-root process sees "full" before df reaches 100%.
df -h /var # bytes
df -i /var # inodes: look at IUse%
# Which directories hold the most inodes (GNU du)
sudo du --inodes -x / 2>/dev/null | sort -rn | head -20
# Count files per directory under a suspect path
sudo find /var/lib/php/sessions -xdev -type f | wc -l
# Clean up old small files in bulk
sudo find /var/lib/php/sessions -type f -mmin +1440 -delete
# Reserved blocks on ext4
sudo tune2fs -l /dev/nvme0n1p1 | grep -i 'reserved block count'
Fixes: delete or archive the small files, fix the application that creates them, and for workloads with many small files choose a filesystem or inode ratio sized for it (XFS allocates inodes dynamically). If df -h and df -i both look normal but df -h still disagrees with du, suspect deleted-but-open files.
Services, Scheduling & Logs
Q36: How do you run a backend service under systemd? Walk through a unit file.
# /etc/systemd/system/orders-api.service
[Unit]
Description=Orders API
After=network-online.target
Wants=network-online.target
StartLimitIntervalSec=60
StartLimitBurst=5
[Service]
Type=simple
User=orders
Group=orders
WorkingDirectory=/opt/orders
EnvironmentFile=/etc/orders/orders.env
ExecStart=/opt/orders/bin/orders-api --port 8080
Restart=on-failure
RestartSec=5
TimeoutStopSec=30
LimitNOFILE=65536
# Hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/lib/orders
[Install]
WantedBy=multi-user.target
sudo systemctl daemon-reload # after creating or editing unit files
sudo systemctl enable --now orders-api # start now and at boot
systemctl status orders-api
sudo systemctl edit orders-api # create a drop-in override instead of editing the unit
systemctl cat orders-api # show the unit plus all drop-ins
Points worth explaining:
Restart=on-failurewithStartLimitBurstgives crash recovery without an infinite restart loop.- On stop, systemd sends SIGTERM (
KillSignal=), waitsTimeoutStopSec, then sends SIGKILL, so the app must handle SIGTERM. ExecStartis not run by a shell: no pipes, redirections or$(...); wrap in/bin/sh -conly if you really need them.- Logs written to stdout/stderr go to the journal automatically, so the app does not need to manage log files.
- Resource controls such as
MemoryMax=andCPUQuota=use cgroups and can be added in a drop-in.
Q37: When would you use a systemd timer instead of cron?
Cron is simple and everywhere. systemd timers trade a bit more setup for better operations:
| Concern | cron | systemd timer |
|---|---|---|
| Logs | Whatever you redirect; otherwise mailed or lost | Captured in the journal per unit |
| Missed runs (host was off) | Skipped (unless using anacron) | Persistent=true runs it at next boot |
| Overlapping runs | Possible unless you add flock | A running oneshot service is not started again |
| Resource limits, sandboxing | Manual | Any [Service] directive (MemoryMax=, User=, ...) |
| Jitter across a fleet | Manual sleep | RandomizedDelaySec= |
| Observability | grep CRON /var/log/syslog | systemctl list-timers, systemctl status |
# /etc/systemd/system/db-backup.service
[Unit]
Description=Nightly database backup
[Service]
Type=oneshot
User=backup
ExecStart=/usr/local/bin/db-backup.sh
# /etc/systemd/system/db-backup.timer
[Unit]
Description=Run database backup nightly
[Timer]
OnCalendar=*-*-* 02:30:00
Persistent=true
RandomizedDelaySec=10m
[Install]
WantedBy=timers.target
sudo systemctl daemon-reload
sudo systemctl enable --now db-backup.timer
systemctl list-timers --all
systemd-analyze calendar '*-*-* 02:30:00' # check when the expression fires
sudo systemctl start db-backup.service # run it once right now
journalctl -u db-backup.service --since today
Q38: Which journalctl commands do you use most when debugging?
journalctl -u nginx # one unit
journalctl -u nginx -f # follow
journalctl -u nginx --since "1 hour ago" --until "10 min ago"
journalctl -u api -u worker --since today # several units interleaved by time
journalctl -p err -b # priority err and worse, current boot
journalctl -b -1 -e # previous boot, jump to the end (why did it reboot?)
journalctl -k # kernel messages (OOM killer, disk errors)
journalctl _PID=4123 # by field
journalctl -u api -o json-pretty -n 5 # structured output with all fields
journalctl --disk-usage
sudo journalctl --vacuum-size=1G # shrink stored journals
sudo journalctl --vacuum-time=14d
Things to know:
- On some distributions the journal is volatile (kept in
/run, lost on reboot) unless/var/log/journalexists orStorage=persistentis set injournald.conf. "I can't see logs from before the crash" is often this. - journald applies rate limiting (
RateLimitIntervalSec=,RateLimitBurst=), so a very chatty service can have messages suppressed; the journal records that it dropped them. -b -1combined with-kis the first stop after an unexpected reboot or kernel panic.
Performance & Memory
Q39: What does load average mean, and why can it be high while the CPU is idle?
Load average (from uptime, top, /proc/loadavg) is an exponentially damped average, over roughly 1, 5 and 15 minutes, of the number of tasks that are runnable (state R, running or waiting for a CPU) plus tasks in uninterruptible sleep (state D, usually waiting on disk or network filesystem I/O). Including D-state tasks is Linux-specific.
Interpretation:
- Compare it with the number of CPUs (
nproc). A load of 8 on a 16-core box is fine; on a 2-core box it means a queue. - High load and high CPU (
us/syinvmstat): CPU-bound. - High load and idle CPU and high
waor many processes in thebcolumn: tasks stuck in I/O (slow disk, saturated cloud volume, hung NFS mount). - High
st(steal) on a VM: the hypervisor is giving your CPU time to other tenants.
uptime # load averages
nproc # CPU count
vmstat 1 5 # r = runnable, b = blocked (D), us/sy/id/wa/st percentages
mpstat -P ALL 1 3 # per-CPU usage (sysstat package)
iostat -xz 1 3 # per-device %util and await (sysstat package)
pidstat -d 1 5 # per-process disk I/O (sysstat package)
ps -eo state,pid,cmd | awk '$1 == "D"' # who is stuck in uninterruptible sleep
Q40: How does Linux use memory? Explain page cache, "available" memory and the OOM killer.
Linux uses otherwise free RAM as page cache for file data. That memory shows as buff/cache and is mostly reclaimable, so "free" being low is normal. The number that matters is available: the kernel's estimate of memory that can be given to new allocations without swapping.
free -h
# total used free shared buff/cache available
# Mem: 15Gi 6.1Gi 410Mi 120Mi 9.0Gi 8.9Gi
cat /proc/meminfo | grep -E 'MemAvailable|Dirty|SwapTotal|SwapFree'
ps -eo pid,rss,vsz,cmd --sort=-rss | head # RSS = resident memory in KiB
Key concepts:
- RSS vs VSZ: VSZ is reserved virtual address space and is often huge and harmless; RSS is what is actually resident in RAM.
- Overcommit: Linux lets processes allocate more virtual memory than exists (
vm.overcommit_memory), betting that not all of it will be touched. - OOM killer: when memory truly runs out (system-wide, or inside a cgroup that hit its limit, such as a container or a systemd unit with
MemoryMax=), the kernel kills the process with the highestoom_score, which is mostly based on memory use and adjusted byoom_score_adj(-1000 to 1000; -1000 means never kill). - Dropping caches (
echo 3 > /proc/sys/vm/drop_caches) does not fix memory pressure; it just throws away useful cache. It is a benchmarking tool, not a production fix.
# Was something OOM-killed?
journalctl -k | grep -iE 'out of memory|killed process'
dmesg -T | grep -i oom
# Which processes would be chosen first
for p in /proc/[0-9]*; do
printf '%s %s %s\n' "$(cat "$p/oom_score" 2>/dev/null)" "${p#/proc/}" "$(tr '\0' ' ' < "$p/cmdline" 2>/dev/null | cut -c1-60)"
done | sort -rn | head
Senior and Principal Production Scenarios
Q41: Disk is 100% full, but du finds only half the usage. What is going on?
Level: Senior
Scenario: An API host alerts at 100% disk on /. Someone already ran rm /var/log/app/app.log (12 GB) to free space, but df -h still shows the disk full and writes keep failing. du -sh / adds up to far less than df reports.
Impact: The database client cannot write temp files, the app fails health checks, and logrotate and package updates fail.
Root Cause: Deleting a file removes only its directory entry. The app still has the log open, so the inode's link count is 0 but its open-file count is not, and the kernel cannot free the blocks. du walks directory entries (it cannot see the deleted file); df asks the filesystem for allocated blocks (it can).
Fix:
- Find deleted files that are still open.
- Free the space immediately by truncating through the process's file descriptor, or by restarting or reloading the process.
- Fix log handling so it does not happen again.
# 1. Deleted-but-open files (link count < 1)
sudo lsof -nP +L1
# or
sudo lsof -nP | grep '(deleted)'
# COMMAND PID USER FD TYPE DEVICE SIZE/OFF NLINK NODE NAME
# app 4123 app 7w REG 259,1 12884901888 0 1311 /var/log/app/app.log (deleted)
# 2a. Truncate via /proc without restarting (PID 4123, fd 7 from the output above)
sudo truncate -s 0 /proc/4123/fd/7
# 2b. Or make the app reopen its logs / restart it
sudo systemctl restart app
# Confirm
df -h /
# 3. Prevention: logrotate config that does not leave open handles behind
sudo tee /etc/logrotate.d/app > /dev/null <<'EOF'
/var/log/app/*.log {
daily
rotate 7
compress
delaycompress
missingok
notifempty
copytruncate
}
EOF
copytruncate copies the log then truncates the original in place, so the app keeps writing to the same inode (a few lines can be lost between copy and truncate). If the app can reopen logs on a signal, prefer create with a postrotate script that sends that signal. Better still, log to stdout and let journald or the container runtime handle rotation.
Workflow Diagram:
df says 100% ──► du says 50% ──► mismatch = space held outside the directory tree
│
▼
lsof +L1 (deleted but open)
│
┌─────────────────────┴─────────────────────┐
▼ ▼
truncate /proc/PID/fd/N restart / signal app to reopen
│ │
└──────────────► df back to normal ◄────────┘
│
▼
logrotate copytruncate / postrotate signal / log to stdout
Interviewer Notes: Strong candidates explain the inode link count versus open references, and know both lsof +L1 and the /proc/PID/fd truncate trick. Principal-level answers add alerting on disk growth rate (not just a threshold) and moving logs off the root volume.
Red Flags: Rebooting as the first step; running rm on more files hoping to free space; not understanding why du and df disagree; recommending kill -9 on the database.
Q42: Load average is 40 on an 8-core box, but CPU is mostly idle. How do you debug it?
Level: Senior
Scenario: Latency on a reporting service jumps from 50 ms to 5 s. uptime shows a load average of 40 on 8 cores, but top shows CPU around 10% user and 2% system.
Impact: Requests time out, health checks fail, and autoscaling adds hosts that hit the same bottleneck.
Root Cause: Linux load average counts tasks in uninterruptible sleep (D state) as well as runnable tasks. Many threads blocked on I/O raise the load without using CPU. Typical causes: a cloud block volume hitting its IOPS or throughput limit (or running out of burst credits), a saturated local disk from a batch job, or an unresponsive NFS server leaving processes stuck in D state.
Fix:
# Confirm the shape: high 'b' (blocked) and high 'wa' (I/O wait), low us/sy
vmstat 1 5
# Which device is saturated? Look for %util near 100 and high r_await/w_await
iostat -xz 1 5
# Which processes are stuck, and on what kernel function?
ps -eo state,pid,wchan:32,cmd | awk '$1 == "D"'
sudo cat /proc/2817/stack # kernel stack of one stuck task, e.g. PID 2817 (root only)
# Which processes generate the I/O?
sudo pidstat -d 1 5
sudo iotop -oPa # if installed
# Is a network filesystem involved?
mount | grep -E 'nfs|cifs'
Then act on what you find:
- Noisy neighbour on the same disk (backup,
find, log compression): throttle it withionice -c3, reschedule it, or move it to another volume. - Cloud volume limits: raise provisioned IOPS/throughput or change volume type; add caching.
- Hung NFS: tasks in D state cannot be killed until I/O completes; fix the server or network; review mount options for that share.
Workflow Diagram:
High load ─► vmstat
├─ us/sy high ............► CPU-bound: top / perf / profiler
└─ id high, wa high, b>0 ─► I/O-bound
├─ iostat %util ~100 ─► disk saturated ─► pidstat -d: who?
└─ disk idle, D tasks ─► wchan / stack ─► NFS or other remote I/O
Interviewer Notes: Looking for the knowledge that load is not CPU usage on Linux, use of vmstat b/wa columns, and iostat -x. Senior candidates mention cloud volume limits and that D-state processes ignore SIGKILL.
Red Flags: "Load is high so add more CPUs"; killing processes at random; never looking at disk metrics.
Q43: A nightly backup cron job "succeeded" for weeks, but the backups are empty. What went wrong?
Level: Senior
Scenario: 0 2 * * * /opt/scripts/backup.sh has been in the crontab for months. During an incident you discover that the last good backup is three weeks old. The script works perfectly when run by hand.
Impact: Restore point objective violated; potential permanent data loss.
Root Cause: A combination of cron's environment and missing error handling:
- Cron runs with a minimal environment (
PATHis typically/usr/bin:/bin), no login profile, and the user's home as the working directory. A tool installed in/usr/local/bin(for exampleawsorpg_dumpfrom a newer package) is "command not found" under cron. - Output and errors are mailed to the user, but no mail transport is configured, so they disappear.
- The script lacked
pipefail, sopg_dump ... | gzip > filereturned gzip's success. - A
%in the crontab line (for example indate +%F) is turned into a newline by cron unless escaped as\%. - Nobody alerted on the absence of a successful run.
Fix:
# crontab -e
SHELL=/bin/bash
PATH=/usr/local/bin:/usr/bin:/bin
MAILTO=""
0 2 * * * flock -n /var/lock/backup.lock /opt/scripts/backup.sh >> /var/log/backup.log 2>&1
#!/usr/bin/env bash
# /opt/scripts/backup.sh
set -Eeuo pipefail
trap 'echo "$(date -Is) FAILED at line $LINENO: $BASH_COMMAND" >&2' ERR
out="/backups/db-$(date +%F).sql.gz"
/usr/bin/pg_dump --no-password mydb | gzip > "$out"
# Verify the artifact, not just the exit code
size=$(stat -c %s "$out")
if (( size < 1048576 )); then
echo "$(date -Is) backup too small: $size bytes" >&2
exit 1
fi
gzip -t "$out"
# Dead man's switch: monitoring alerts if this ping does NOT arrive on schedule
curl -fsS -m 10 --retry 3 "$BACKUP_HEARTBEAT_URL" > /dev/null
echo "$(date -Is) backup ok: $out ($size bytes)"
Reproduce cron's environment when debugging:
env -i HOME="$HOME" LOGNAME="$USER" PATH=/usr/bin:/bin SHELL=/bin/sh /bin/sh -c '/opt/scripts/backup.sh'
Workflow Diagram:
cron 02:00 ─► backup.sh ─► pg_dump | gzip ─► verify size + gzip -t ─► heartbeat ping
│ │ │ │
└── any failure ──┴──────────────────┘ │
│ ▼
ERR trap logs line monitoring: "no ping by 03:00" ─► page on-call
Interviewer Notes: The key insight is "alert on missing success, not on presence of failure". Strong answers also include periodic restore tests, since a backup that has never been restored is unverified. Moving to a systemd timer (journal logs, OnFailure= hooks) is a good principal-level suggestion.
Red Flags: Only adding >> log 2>&1 with no alerting; trusting exit codes without verifying output; never testing restores.
Q44: A process is pinning the CPU at 100% in production. How do you investigate and contain it?
Level: Senior
Scenario: One of four API hosts shows 100% CPU on several cores. p99 latency on that host is 10x the others.
Impact: Degraded latency for a quarter of traffic; risk of cascading failure if the load balancer keeps sending requests.
Root Cause: Commonly a hot loop (retry without backoff, regex catastrophic backtracking, infinite loop on a malformed input), garbage collection thrashing near a heap limit, or a runaway background job. The goal is to identify which thread and which code path before you destroy the evidence by restarting.
Fix:
# 1. Which process, and which threads inside it?
ps -eo pid,ppid,user,%cpu,%mem,etime,cmd --sort=-%cpu | head
top -H -p 4123 # per-thread view; note hot thread IDs
# 2. Is it in user space or making system calls?
sudo strace -f -c -p 4123 # Ctrl-C after a few seconds for a syscall summary
# Almost no syscalls while burning CPU => a pure user-space loop
# 3. Where in the code? Use the runtime's own tools
sudo perf top -p 4123 # native code / kernel symbols
py-spy dump --pid 4123 # Python stack of every thread
jstack 4123 > /tmp/jstack.txt # JVM thread dump
printf '%x\n' 4188 # hot TID from top -H, in hex, to match "nid=0x..." in jstack
# 4. Contain while you investigate
sudo renice +19 -p 4123 # lowest scheduling priority
sudo systemctl set-property api.service CPUQuota=200% # cap at 2 CPUs via cgroup
# Take the host out of the load balancer, capture evidence, then restart
Workflow Diagram:
Alert: CPU 100%
│
▼
ps/top -H ─► hot PID + TIDs
│
├─ strace -c: busy syscalls? ─► I/O or polling loop (look at what it calls)
└─ strace -c: quiet ──────────► user-space loop ─► perf / py-spy / jstack
│
▼
Contain: drain from LB, renice / CPUQuota
│
▼
Capture stacks ─► restart ─► fix code + add a regression test / timeout / backoff
Interviewer Notes: Look for "capture evidence before restarting" and knowing how to map a hot thread to a stack (the top -H plus hex TID trick for the JVM is a strong senior signal). Principal-level: talk about protecting the fleet (circuit breakers, request timeouts, input validation) so one bad input cannot pin every host.
Red Flags: kill -9 immediately with no data; restarting every host at once; blaming "the server is too small" without profiling.
Q45: A service starts failing with "Too many open files" after running for a few days. What do you do?
Level: Senior
Scenario: An HTTP service runs fine after deploy, then after two to three days its error rate climbs and logs show EMFILE: Too many open files / accept: too many open files. A restart fixes it temporarily.
Impact: New connections are refused; outbound calls to databases and APIs fail.
Root Cause: Either the limit is too low for legitimate load, or (more often when it grows over days) a file descriptor leak: sockets or files opened and never closed. A frequent pattern is HTTP responses whose bodies are never closed, or a peer closed the connection but the app never called close(), leaving sockets in CLOSE_WAIT. Raising the limit in /etc/security/limits.conf does nothing for a systemd service, which only honours LimitNOFILE=.
Fix:
PID=$(systemctl show -p MainPID --value api)
# Current usage versus limit
ls /proc/$PID/fd | wc -l
grep 'open files' /proc/$PID/limits
# What kind of descriptors are they? (TYPE column: REG, IPv4, IPv6, sock, FIFO ...)
sudo lsof -nP -p "$PID" | awk 'NR>1 {print $5}' | sort | uniq -c | sort -rn
# Leaking sockets usually pile up in CLOSE_WAIT (peer closed, we did not)
ss -tanp state close-wait | head
ss -tan state close-wait | wc -l
# Watch growth over time to confirm a leak
watch -n 60 "ls /proc/$PID/fd | wc -l"
# sudo systemctl edit api (creates a drop-in override)
[Service]
LimitNOFILE=65536
sudo systemctl daemon-reload
sudo systemctl restart api
grep 'open files' /proc/$(systemctl show -p MainPID --value api)/limits
Raise the limit to buy time, but the real fix is in code: close response bodies, use one shared HTTP client with a bounded connection pool, use try-with-resources / with / defer, and add an fd-count metric so the leak is visible on a dashboard long before it causes an outage.
Workflow Diagram:
EMFILE errors
│
▼
fd count vs limit ──► at limit?
│
├─ count steady, limit low ──► raise LimitNOFILE (systemd drop-in)
└─ count grows over time ────► leak
│
▼
lsof by TYPE + ss state close-wait
│
▼
fix close()/pooling in code + fd metric + alert
Interviewer Notes: Seniors should know that limits.conf does not apply to systemd services, how to read /proc/PID/limits, and what CLOSE_WAIT means (the local application has not closed its side). Principal-level: export open-fd count as a metric (many runtimes' Prometheus clients expose process_open_fds) and alert on trend.
Red Flags: Setting the limit to a huge number and calling it done; confusing CLOSE_WAIT with TIME_WAIT; not checking the actual limit of the running process.
Q46: A critical service keeps getting killed by the OOM killer. How do you diagnose and protect it?
Level: Senior
Scenario: On a host running both an API and a nightly analytics job, the API process disappears a few times a week around 01:00. systemd restarts it, but each time in-flight requests fail.
Impact: Intermittent 502s, lost in-flight work, and on-call noise.
Root Cause: The analytics job's memory grows until the host runs out of memory. The kernel OOM killer picks the process with the highest oom_score, which is mostly driven by memory size, and the long-running API (with a large heap) is often the biggest target, not the job that caused the pressure. Page cache is not the culprit: it is reclaimed before the OOM killer runs.
Fix:
# 1. Prove it was the OOM killer, and see who was killed and why
journalctl -k --since "2 days ago" | grep -iE 'out of memory|oom-kill|killed process'
systemctl status api # look for "Failed with result 'oom-kill'"
# 2. Was it a global OOM or a cgroup (unit/container) limit?
# The kernel log line mentions the constraint and the memory cgroup involved.
journalctl -k | grep -i 'constraint='
# 3. Current scores
cat /proc/$(systemctl show -p MainPID --value api)/oom_score
# sudo systemctl edit analytics-job -> cap the job so it hits ITS limit, not the host's
[Service]
MemoryHigh=3G
MemoryMax=4G
# sudo systemctl edit api -> make the API a less attractive OOM victim
[Service]
OOMScoreAdjust=-500
Then address the cause: fix or stream the job's data processing, run the job on separate capacity, and set an explicit heap limit for runtimes that size their heap from host memory. MemoryHigh= throttles and forces reclaim before MemoryMax= triggers a kill, which gives a softer failure mode.
Workflow Diagram:
Process vanished ─► journalctl -k: "Killed process 4123 (api)" ─► global or cgroup OOM?
│
┌───────────────────────────────────────────────┴─────────────┐
▼ ▼
Global (host ran out) cgroup limit hit (unit/container)
│ │
cap the real consumer (MemoryMax), raise limit or fix the leak in
protect critical (OOMScoreAdjust), isolate workloads that service; set heap limits
Interviewer Notes: Look for: reading kernel logs for evidence, knowing the victim is not necessarily the cause, cgroup limits per workload, and oom_score_adj / OOMScoreAdjust. Principal-level: capacity isolation (separate node pools / hosts for batch versus latency-sensitive services) and memory-usage SLOs.
Red Flags: "Add swap" as the whole answer; disabling the OOM killer; setting -1000 on everything; dropping caches.
Q47: Two copies of a cron job ran at the same time and processed the same records twice. How do you prevent it?
Level: Senior
Scenario: A */5 * * * * job syncs invoices to a payment provider. It normally takes 2 minutes, but during month-end it took 8 minutes; the next run started before the first finished, and some customers were charged twice.
Impact: Duplicate charges, refunds, support load, and loss of trust.
Root Cause: Cron starts jobs on schedule regardless of whether the previous run finished. The job itself was not idempotent, and there was no mutual exclusion. A PID-file check existed but was racy and left stale files after crashes.
Fix:
Single host: use flock, which the kernel releases automatically when the process exits.
# crontab
*/5 * * * * flock -n /var/lock/invoice-sync.lock /opt/jobs/invoice-sync.sh >> /var/log/invoice-sync.log 2>&1
Multiple hosts: a local lock is not enough. Use a lock all hosts can see, such as a PostgreSQL advisory lock held for the session that does the work:
#!/usr/bin/env bash
set -euo pipefail
# The lock is held for the lifetime of this psql session and released when it ends.
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 <<'SQL'
SELECT pg_try_advisory_lock(42) AS got_lock \gset
\if :got_lock
\echo 'lock acquired, syncing'
CALL sync_pending_invoices();
\else
\echo 'another run holds the lock, skipping'
\endif
SQL
Most importantly, make the work idempotent: mark records as claimed in the same transaction that selects them (SELECT ... FOR UPDATE SKIP LOCKED), and send an idempotency key to the payment provider so a retry or duplicate cannot double-charge.
Workflow Diagram:
02:00 run A ──── takes 8 min ──────────────────────────────►
02:05 run B ──► flock -n fails ──► exit 0 (skipped, logged)
02:10 run C ──► flock -n fails ──► exit 0
02:10:30 A finishes ──► lock released by kernel
02:15 run D ──► acquires lock ──► processes only unclaimed rows (idempotent)
Interviewer Notes: Good answers layer three defences: mutual exclusion, idempotent processing, and idempotency keys at external boundaries. Principal-level: question whether this should be a queue consumer instead of a polling cron, and add a metric for job duration versus schedule interval.
Red Flags: Only increasing the interval; PID files without understanding stale-lock problems; relying on flock across multiple servers.
Q48: A cleanup script ran rm -rf on the wrong directory. How do you write destructive scripts safely?
Level: Senior / Principal
Scenario: A CI cleanup step runs rm -rf "$BUILD_ROOT/"*. A refactor renamed the variable that set BUILD_ROOT, so it expanded to an empty string and the command became rm -rf /* on a self-hosted runner. A widely reported Steam for Linux bug had exactly this shape.
Impact: A wiped runner, and on a shared host potentially every mounted volume, including network mounts.
Root Cause: An unset or empty variable in a destructive path, with no set -u, no validation, and no guard. GNU rm's default --preserve-root protects only the literal / argument; /* is expanded by the shell into /bin, /etc, ... before rm sees it.
Fix:
#!/usr/bin/env bash
set -euo pipefail
: "${BUILD_ROOT:?BUILD_ROOT must be set}" # abort if unset or empty
# Resolve and validate before deleting
target=$(realpath -- "$BUILD_ROOT")
case "$target" in
/srv/ci/builds/*) ;; # allow-list of safe prefixes
*) echo "refusing to delete outside /srv/ci/builds: $target" >&2; exit 1 ;;
esac
# ${var:?} inside the command as a second guard; stay on one filesystem
rm -rf --one-file-system -- "${target:?}"/*
# Safer pattern for scheduled cleanups: dry-run first, then act
find "$target" -mindepth 1 -maxdepth 1 -mtime +3 -print
find "$target" -mindepth 1 -maxdepth 1 -mtime +3 -exec rm -rf -- {} +
Process safeguards: run shellcheck in CI (rule SC2115 flags exactly this "$var/"* pattern), run CI jobs as an unprivileged user in a disposable container, and require review for scripts that delete data.
Workflow Diagram:
variable ─► set? (:?) ─► non-empty? ─► realpath ─► inside allow-listed prefix? ─► dry-run list ─► delete
│ │ │ │
└── no ──────┴──── no ─────┴──────────── no ──────────────┴──► abort with clear error
Interviewer Notes: Look for set -u, ${var:?}, allow-list validation, -- to end option parsing, and least privilege. A principal answer focuses on blast radius: unprivileged runners, ephemeral machines, and backups.
Red Flags: "Just be careful"; running cleanup scripts as root on long-lived hosts; not knowing why --preserve-root did not help.
Q49: Outbound calls fail with "Cannot assign requested address" under load. What is happening?
Level: Principal
Scenario: A service calls an internal pricing API at about 3,000 requests per second. During a traffic spike, calls start failing with EADDRNOTAVAIL ("Cannot assign requested address") even though the pricing API is healthy. ss -s shows tens of thousands of sockets in TIME-WAIT.
Impact: Pricing lookups fail and checkout errors spike.
Root Cause: Ephemeral port exhaustion. Each outbound TCP connection to the same destination IP and port needs a unique local port. The client opens a new connection per request and closes it; the side that closes first keeps the socket in TIME_WAIT for about 60 seconds on Linux. With the default ephemeral range 32768-60999 (about 28,000 ports), sustained new-connection rates of several hundred per second to a single destination exhaust the range.
Fix:
# Confirm
ss -s
ss -tan state time-wait | wc -l
ss -tan state time-wait '( dport = :8443 )' | wc -l # to the pricing API
sysctl net.ipv4.ip_local_port_range
# Short-term mitigations (persist them in /etc/sysctl.d/ if kept)
sudo sysctl -w net.ipv4.ip_local_port_range="15000 65000"
sudo sysctl -w net.ipv4.tcp_tw_reuse=1 # allow reuse of TIME_WAIT sockets for new outgoing connections
The real fix is in the application: reuse connections with HTTP keep-alive and a bounded connection pool (one shared client per process, not a new client per request). A pool of a few hundred persistent connections replaces hundreds of thousands of short-lived ones. Do not use tcp_tw_recycle: it broke clients behind NAT and was removed from the kernel in 4.12.
Workflow Diagram:
Per-request client Pooled keep-alive client
────────────────── ────────────────────────
req ─► connect (new port) ─► close req ─► borrow conn ─► return to pool
│ │
▼ ▼
TIME_WAIT ~60s holds the port same few hundred ports reused
│
▼
~28k ports / 60s ≈ 470 new conns/s max per destination ─► EADDRNOTAVAIL beyond that
Interviewer Notes: Strong answers do the arithmetic, explain that TIME_WAIT exists to protect against stray delayed packets, and know it is the side that closes first that holds it. Principal-level: connection pooling standards in shared client libraries, and metrics for connection creation rate.
Red Flags: Lowering tcp_fin_timeout and believing it shortens TIME_WAIT (it controls FIN_WAIT_2); recommending tcp_tw_recycle; blaming the downstream service.
Q50: A deploy script printed "Deployment successful" but production is broken. How did the failure get masked?
Level: Principal
Scenario: A Bash deploy script runs migrations, builds assets and restarts the service. It has set -e at the top. The migration step failed, but the script continued, restarted the app against an old schema, and reported success to the pipeline.
Impact: Partial outage after a "green" deploy, and on-call engineers lose time because the pipeline says nothing is wrong.
Root Cause: Several Bash behaviours each disable or bypass set -e:
#!/usr/bin/env bash
set -e
migrate() {
local output=$(./manage.py migrate 2>&1) # 'local' returns 0, masking the failure
echo "$output"
}
run_migrations() {
migrate
echo "migrations done"
}
# errexit is ignored for everything inside a function called as an if condition
if run_migrations; then
echo "ok"
fi
# Without pipefail, the pipeline status is tee's status (0)
npm run build | tee build.log
# In Bash, errexit is not inherited by command substitutions unless inherit_errexit is set:
# the failing check is ignored inside $( ) and echo's status 0 becomes the assignment's status
summary=$(./manage.py check --deploy; echo "checks finished")
systemctl restart app
echo "Deployment successful"
Fix:
#!/usr/bin/env bash
set -Eeuo pipefail
shopt -s inherit_errexit # Bash 4.4+: command substitutions inherit errexit
trap 'echo "deploy FAILED at line $LINENO: $BASH_COMMAND (exit $?)" >&2' ERR
migrate() {
local output
output=$(./manage.py migrate 2>&1) # declare first, assign second
printf '%s\n' "$output"
}
migrate
npm run build | tee build.log
systemctl restart app
# Verify the outcome, not the steps
for i in {1..30}; do
if curl -fsS -m 2 http://127.0.0.1:8080/healthz > /dev/null; then
echo "Deployment successful"
exit 0
fi
sleep 2
done
echo "service unhealthy after restart" >&2
exit 1
Workflow Diagram:
step ─► exit code checked? ─► yes ─► next step ─► ... ─► restart ─► health check passes? ─► report success
│ │
no (masked by local / if / pipe / $( )) no
│ │
▼ ▼
silent continue ─► "green" deploy, broken prod trap ERR + exit 1 ─► pipeline red ─► rollback
Interviewer Notes: Excellent candidates can list the specific contexts where set -e is ignored and explain that success should be defined by a post-deploy health check, not by the absence of errors. Principal-level: move complex deploy logic out of Bash into a tool with real error handling, and gate traffic with readiness checks and automated rollback.
Red Flags: Believing set -e guarantees the script stops on any error; no post-deploy verification; parsing output with grep to decide success.