Why Every DevOps Engineer Should Learn Unix: The Backbone of Modern DevOps

# Find all error logs from the last hour
find /var/log -name "*.log" -mmin -60 -exec grep -l "ERROR" {} \;

# Count critical errors by service
grep "CRITICAL" /var/log/app.log | awk '{print $5}' | sort | uniq -c

# Monitor real-time logs with filtering  
tail -f /var/log/app.log | grep -E "(ERROR|CRITICAL|FATAL)"

# Find top 10 IP addresses in access logs
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -nr | head -10

# Check system resources in one command
echo "=== DISK USAGE ===" && df -h && echo "=== MEMORY ===" && free -h && echo "=== CPU ===" && top -bn1 | head -5

# Find processes consuming most memory
ps aux --sort=-%mem | head -10

# Check for zombie processes
ps aux | awk '$8 ~ /^Z/ { print $2 }'

# Monitor network connections
netstat -tuln | grep LISTEN | sort -k4

#!/bin/bash
# Deployment script example

# Backup current application
tar -czf "backup-$(date +%Y%m%d-%H%M%S).tar.gz" /opt/myapp/

# Stop application service
systemctl stop myapp

# Deploy new version
rsync -avz --delete /tmp/deploy/ /opt/myapp/

# Set proper permissions
chown -R appuser:appuser /opt/myapp/
chmod +x /opt/myapp/bin/start.sh

# Start service and check status
systemctl start myapp
systemctl status myapp

# Verify deployment
curl -f http://localhost:8080/health || { echo "Deployment failed"; exit 1; }

# Find files with world-writable permissions
find /etc /opt /var -type f -perm -o+w 2>/dev/null

# Check for users with empty passwords
awk -F: '$2 == "" {print $1}' /etc/shadow

# Find SUID/SGID files
find /usr /bin /sbin -type f \( -perm -4000 -o -perm -2000 \) 2>/dev/null

# Check failed login attempts
grep "Failed password" /var/log/auth.log | awk '{print $11}' | sort | uniq -c | sort -nr

# Docker container management
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
docker logs -f --tail 100 container_name

# Kubernetes troubleshooting
kubectl get pods --all-namespaces -o wide
kubectl describe pod podname
kubectl logs -f deployment/myapp

# Resource monitoring
kubectl top nodes
kubectl top pods --all-namespaces --sort-by=memory

# One-liner deployment status check
for service in web api database; do echo "=== $service ===" && systemctl is-active $service && echo; done

# Batch file processing
find /data/incoming -name "*.csv" -type f -exec mv {} /data/processed/ \; -exec echo "Processed {}" \;

# Log rotation and cleanup
find /var/log -name "*.log" -mtime +30 -delete && echo "Old logs cleaned"

# System maintenance script
{ df -h; echo; free -h; echo; uptime; } | mail -s "Daily System Report" admin@company.com

# CPU usage alert
cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
if (( $(echo "$cpu_usage > 80" | bc -l) )); then
    echo "High CPU usage: $cpu_usage%" | mail -s "CPU Alert" admin@company.com
fi

# Disk space monitoring
df -h | awk '$5 > "85%" {print "Warning: " $0}' | mail -s "Disk Space Alert" admin@company.com

# Service health check
for service in nginx mysql redis; do
    if ! systemctl is-active --quiet $service; then
        echo "Service $service is down!" | mail -s "Service Alert" admin@company.com
    fi
done

# Basic system info
hostname                    # Shows system host name
hostid                     # Shows host ID assigned by OS
date                       # Shows current date and time in UTC
uptime                     # Shows system uptime and load
uname                      # Shows Unix system name
uname -a                   # Shows all system information

# System status
printenv                   # Displays all environment variables
last                       # Shows previous logins in the system
echo $?                    # Shows exit status of last command (0=success)
history                    # Lists all executed commands
clear                      # Clears the screen

# System control
systemctl status sshd      # Check service status
systemctl start sshd       # Start a service
systemctl stop sshd        # Stop a service
systemctl enable sshd      # Enable service at boot
sudo shutdown -r now       # Restart system immediately

# File creation and viewing
touch filename.txt             # Create empty file or update timestamp
touch file1.txt file2.txt     # Create multiple files
cat filename.txt              # Display file contents
cat > filename.txt            # Create file with interactive input
head filename.txt             # Show first 10 lines
head -n 5 filename.txt        # Show first 5 lines
tail filename.txt             # Show last 10 lines
tail -f filename.txt          # Follow file changes in real-time
tail -F filename.txt          # Follow with file rotation handling
less filename.txt             # View large files with pagination

# File manipulation
cp source destination         # Copy files
cp -r dir1 dir2              # Copy directories recursively
mv old_name new_name         # Move/rename files
rm filename.txt              # Remove file
rm -r directory/             # Remove directory recursively
rm -rf directory/            # Force remove directory
chmod 755 script.sh          # Change file permissions
find /path -name "*.log"     # Find files by pattern
locate filename.txt          # Find files using database

# View permissions
ls -l filename.txt           # Show file permissions
ls -ld directory/            # Show directory permissions

# Change permissions (octal notation)
chmod 755 filename.txt       # rwxr-xr-x (read=4, write=2, execute=1)
chmod 644 filename.txt       # rw-r--r--
chmod 777 filename.txt       # rwxrwxrwx (full permissions)
chmod -R 755 directory/      # Change permissions recursively

# Change ownership
chown user filename.txt      # Change file owner
chown user:group filename.txt # Change owner and group
chgrp groupname filename.txt # Change group ownership

# Access Control Lists (ACL)
getfacl filename.txt         # Show file ACL
setfacl -m u:user:rwx file   # Modify user ACL
setfacl -x u:user: file      # Remove user ACL
setfacl -m g:group:rwx file  # Modify group ACL

# Search and filter
grep "error" logfile.txt        # Search text in files
grep -i "error" logfile.txt     # Case-insensitive search
grep -v "success" logfile.txt   # Invert match (show non-matching)
grep -l "pattern" *.txt         # Show filenames with matches
awk '{print $1}' file.txt       # Extract first column
sed 's/old/new/g' file.txt      # Replace text globally
sort file.txt                   # Sort lines alphabetically
sort file.txt | uniq            # Sort and remove duplicates
wc file.txt                     # Word, line, character count
wc -l file.txt                  # Count lines only

# Advanced grep variants
egrep "pattern1|pattern2" file  # Extended regex
fgrep "literal.string" file     # Fixed string search
zgrep "pattern" file.gz         # Search in compressed files

# Process management
ps                          # Show running processes
ps aux                      # Show all processes with details
ps -u username              # Show user processes
top                         # Real-time process viewer
htop                        # Enhanced process viewer (if available)
kill PID                    # Gracefully terminate process
kill -9 PID                 # Force kill process
killall processname         # Kill processes by name
pgrep processname           # Find process IDs by name

# Background/foreground control
command &                   # Run command in background
nohup command &            # Run command immune to hangups
bg                         # Send job to background
fg                         # Bring job to foreground
jobs                       # List active jobs

# System resources
free -h                    # Show memory usage (human readable)
df -h                      # Show disk space usage
du -h directory/           # Show directory disk usage
du -sh directory/          # Show total directory size
iostat                     # I/O statistics
vmstat                     # Virtual memory statistics

# User management
useradd username           # Create user account
useradd -m username        # Create user with home directory
passwd username            # Set user password
userdel username           # Delete user account
userdel -r username        # Delete user with home directory
usermod -aG group user     # Add user to group
su username                # Switch to user
su -                       # Switch to root with environment
exit                       # Logout from current user

# User information
whoami                     # Show current username
id                         # Show user and group IDs
who                        # Show logged in users
w                          # Show who is logged in and what they're doing
cat /etc/passwd            # Show all users
cat /etc/shadow            # Show password hashes (root only)

# Group management
groupadd groupname         # Create group
groupdel groupname         # Delete group
gpasswd groupname          # Set group password
gpasswd -a user group      # Add user to group
gpasswd -d user group      # Remove user from group
cat /etc/group             # Show all groups

# Network connectivity
ping hostname              # Test network connectivity
ping -c 4 hostname         # Ping 4 times only
traceroute hostname        # Trace network route
nc -vz hostname port       # Test port connectivity
curl ifconfig.me           # Show public IP address

# Network information
ifconfig                   # Show network interfaces (deprecated)
ip addr                    # Show IP addresses (modern)
ip route                   # Show routing table
netstat -antp              # Show all TCP connections
netstat -tulpn             # Show listening ports
ss -tulpn                  # Modern replacement for netstat

# File transfers
wget URL                   # Download files from web
curl URL                   # Transfer data from/to server
curl -I URL                # Get HTTP headers only
scp file user@host:/path   # Secure copy over SSH
rsync -avz local/ user@host:remote/  # Synchronize directories

# Create archives
tar -czf backup.tar.gz /data/     # Create compressed tar archive
tar -cjf backup.tar.bz2 /data/    # Create bzip2 compressed archive
zip -r archive.zip directory/     # Create ZIP archive

# Extract archives
tar -xzf backup.tar.gz            # Extract gzip compressed tar
tar -xjf backup.tar.bz2           # Extract bzip2 compressed tar
tar -xf archive.tar               # Extract uncompressed tar
unzip archive.zip                 # Extract ZIP archive

# Compression
gzip file.txt                     # Compress file
gunzip file.txt.gz               # Decompress gzip file
bzip2 file.txt                   # Compress with bzip2
bunzip2 file.txt.bz2             # Decompress bzip2 file

# Package operations
apt update                        # Update package list
apt upgrade                       # Upgrade installed packages
apt install package_name          # Install package
apt remove package_name           # Remove package
apt purge package_name            # Remove package with configs
apt autoremove                    # Remove unused dependencies

# Package information
apt list --installed              # List installed packages
apt list --installed | grep package  # Check if package installed
apt show package_name             # Show package information
apt search package_name           # Search for packages

# Navigation
pwd                        # Print working directory
cd /path/to/directory      # Change to directory
cd ..                      # Go up one directory level
cd ../..                   # Go up two directory levels  
cd ~                       # Go to home directory
cd -                       # Go to previous directory

# Directory operations
mkdir directory_name       # Create directory
mkdir -p path/to/dir      # Create directory with parents
rmdir directory_name       # Remove empty directory
ls                        # List directory contents
ls -la                    # List with details and hidden files
ls -lh                    # List with human-readable sizes
tree                      # Show directory tree structure

# System administration
crontab -e                 # Edit cron jobs
crontab -l                 # List cron jobs  
systemctl status service   # Check systemd service status
journalctl -u service      # View service logs
mount /dev/device /mnt     # Mount filesystem
umount /mnt                # Unmount filesystem
lsof                       # List open files
lsof -i :80               # Show what's using port 80

# Text manipulation pipelines
cat logfile | grep ERROR | wc -l        # Count error lines
ps aux | grep nginx | grep -v grep      # Find nginx processes
df -h | grep -v tmpfs | sort -k5 -nr    # Sort by disk usage

# File and directory monitoring
stat filename.txt          # Show detailed file information
file filename.txt          # Determine file type
which command              # Show command location
whereis command            # Show command and manual locations

# Input/Output redirection
command > output.txt       # Redirect stdout to file
command >> output.txt      # Append stdout to file
command 2> error.txt       # Redirect stderr to file
command &> all.txt         # Redirect both stdout and stderr
command | tee output.txt   # Output to both file and terminal

# Most frequently used commands in DevOps
ls -la                     # 1. List files with details
cd /path                   # 2. Change directory  
pwd                        # 3. Show current directory
cat file.txt               # 4. Display file contents
tail -f /var/log/app.log   # 5. Monitor logs in real-time
grep "error" *.log         # 6. Search for patterns
ps aux                     # 7. List running processes
top                        # 8. Monitor system resources
df -h                      # 9. Check disk space
free -h                    # 10. Check memory usage
systemctl status service   # 11. Check service status
chmod 755 script.sh        # 12. Change file permissions
scp file user@host:/path   # 13. Secure file copy
ssh user@hostname          # 14. Remote connection
tar -czf backup.tar.gz /data # 15. Create archive
find /path -name "*.log"   # 16. Find files
kill -9 PID               # 17. Kill process
netstat -tulpn            # 18. Show network connections
crontab -l                # 19. List scheduled tasks
history | grep command    # 20. Search command history

# When things go wrong, start here:
df -h                      # Check disk space
free -h                    # Check memory
top                        # Check CPU and processes  
systemctl --failed         # Check failed services
journalctl -f              # Check recent system logs
netstat -tulpn             # Check network ports
ps aux | grep -v root      # Check user processes

# System performance deep dive
iostat -x 1                # I/O statistics every second
vmstat 1                   # Memory and CPU stats
sar -u 1 10               # CPU usage for 10 seconds
lsof +D /var/log          # Show processes using directory
ss -tuln                  # Modern netstat alternative
dmesg | tail -20          # Recent kernel messages

# Complex pipeline examples
# Find and analyze large files
find /var -type f -size +100M -exec ls -lh {} \; | sort -k5 -h | tail -10

# Process monitoring with alerting
while true; do
    cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    if (( $(echo "$cpu > 90" | bc -l) )); then
        echo "$(date): High CPU usage $cpu%" >> /var/log/alerts.log
        # Send alert notification
    fi
    sleep 60
done

# Automated log analysis
grep -E "(ERROR|FATAL)" /var/log/app.log | \
    awk '{print $1, $2, $NF}' | \
    sort | uniq -c | \
    sort -nr | \
    head -20

# Advanced file operations
# Find duplicate files
find /data -type f -exec md5sum {} \; | sort | uniq -d -w32

# Clean up old files by date
find /tmp -type f -mtime +7 -delete
find /var/log -name "*.log.gz" -mtime +30 -delete

# Secure file deletion
shred -vfz -n 3 sensitive_file.txt

# Create directory structure
mkdir -p /opt/app/{bin,conf,logs,data,backup}