Lesson 3 of 5 · 20 min read

Linux Artifact Collection — Triage Methodology and osquery

Linux triage follows the same order-of-volatility principle as Windows — but the tool names, log paths, and collection methods are entirely different. Responders who learned Windows IR must build separate muscle memory for Linux.

Linux Volatile Collection — First 30 Minutes

Step 1: Network State (3 minutes)

# Save to network share or USB — NOT local disk
# Current network connections with PID and process name
netstat -anop 2>/dev/null > /forensic-share/case/server01_netstat.txt
# OR (modern systems — ss is preferred):
ss -tulnp > /forensic-share/case/server01_netstat.txt

# ARP cache (recently contacted hosts)
arp -n >> /forensic-share/case/server01_net.txt

# Routing table
route -n >> /forensic-share/case/server01_net.txt
ip route show >> /forensic-share/case/server01_net.txt

# Current DNS resolver config
cat /etc/resolv.conf >> /forensic-share/case/server01_net.txt

Step 2: Running Processes (2 minutes)

# Process forest (parent-child relationships visible)
ps auxf > /forensic-share/case/server01_processes.txt

# Processes with full command lines from /proc/
for pid in /proc/[0-9]*/; do
  pid_num=$(basename $pid)
  exe=$(readlink /proc/$pid_num/exe 2>/dev/null)
  cmd=$(cat /proc/$pid_num/cmdline 2>/dev/null | tr '\0' ' ')
  echo "$pid_num | $exe | $cmd"
done > /forensic-share/case/server01_proc_cmdlines.txt

# Find processes with deleted executables (fileless malware indicator)
ls -la /proc/*/exe 2>/dev/null | grep "(deleted)" > \
  /forensic-share/case/server01_deleted_exe.txt

Step 3: Active User Sessions (2 minutes)

# Currently logged in users
w > /forensic-share/case/server01_sessions.txt
who >> /forensic-share/case/server01_sessions.txt

# Recent logins (wtmp)
last -n 50 >> /forensic-share/case/server01_sessions.txt

# Failed logins (btmp)
lastb -n 50 > /forensic-share/case/server01_failed_logins.txt

# Active pseudo-terminals (identify attacker's shell)
ls -la /dev/pts/

Step 4: Open Files and Sockets (3 minutes)

# Network sockets with owning process
lsof -i -nP > /forensic-share/case/server01_lsof_net.txt

# Open files — useful for identifying config files, C2 scripts
lsof -nP > /forensic-share/case/server01_lsof_all.txt

# Look for suspicious open files:
grep -E "/tmp|/dev/shm|/var/tmp|deleted" /forensic-share/case/server01_lsof_all.txt

Step 5: Recover Deleted Running Binaries

# For each process with a deleted executable, recover the binary
grep "(deleted)" /forensic-share/case/server01_deleted_exe.txt | \
  awk '{print $NF}' | sed 's/\(.*\) ->.*/\1/' | while read proc_exe; do
    pid=$(echo "$proc_exe" | grep -oP '(?<=/proc/)\d+')
    cp /proc/$pid/exe /forensic-share/case/recovered_pid${pid}.bin 2>/dev/null
    sha256sum /forensic-share/case/recovered_pid${pid}.bin
done

Log Collection

# Compress and copy all logs
tar czf /forensic-share/case/server01_logs.tar.gz /var/log/ 2>/dev/null

# Or selective copy:
cp /var/log/auth.log* /forensic-share/case/logs/
cp /var/log/syslog* /forensic-share/case/logs/
cp /var/log/cron* /forensic-share/case/logs/

# Export journald (if persistent logging configured)
journalctl --output=json > /forensic-share/case/server01_journal.json

# Binary login databases
cp /var/log/wtmp /forensic-share/case/wtmp
cp /var/log/btmp /forensic-share/case/btmp
cp /var/log/lastlog /forensic-share/case/lastlog

# bash_history for all users
for home in /home/*/ /root/; do
  user=$(basename "$home")
  [ -f "${home}.bash_history" ] && \
    cp "${home}.bash_history" "/forensic-share/case/${user}_bash_history"
done

# /etc/ (configuration, cron, sudoers, passwd/shadow)
tar czf /forensic-share/case/server01_etc.tar.gz /etc/ 2>/dev/null

osquery — Standardized Forensic Queries

-- Run osqueryi (interactive) for ad-hoc queries

-- Active network connections with process context
SELECT DISTINCT p.name, p.pid, p.parent, p.path, p.cmdline,
       s.remote_address, s.remote_port, s.state
FROM process_open_sockets s
JOIN processes p USING (pid)
WHERE s.remote_address NOT IN ('', '0.0.0.0', '::')
  AND s.state = 'ESTABLISHED';

-- All cron jobs on the system
SELECT event, minute, hour, day_of_month, month, day_of_week, command, path
FROM crontab;

-- SSH authorized_keys (all users)
SELECT username, key_file, key, algorithm, comment
FROM authorized_keys;

-- Processes with deleted executables
SELECT pid, name, path, cmdline
FROM processes
WHERE on_disk = 0;

-- Recently modified files in /tmp/ and /dev/shm/ (IOC sweep)
SELECT path, filename, size, mtime, sha256
FROM file
WHERE directory IN ('/tmp/', '/dev/shm/', '/var/tmp/')
  AND mtime > (strftime('%s','now') - 86400);
-- mtime > (now - 86400 seconds) = modified in last 24 hours

-- Shell history for all users
SELECT uid, time, command
FROM shell_history
ORDER BY time DESC
LIMIT 100;

Memory Acquisition on Linux

# Method 1: LiME (Linux Memory Extractor) — kernel module
# Must be compiled for the target kernel version
# Get kernel version:
uname -r   # e.g., 5.15.0-91-generic

# Load LiME module and dump to network path
insmod /usb/lime-5.15.0-91-generic.ko \
  "path=tcp:4444 format=lime"
# On forensic workstation:
nc [server_IP] 4444 > server01_memory.lime

# OR: dump directly to file path
insmod /usb/lime-5.15.0-91-generic.ko \
  "path=/forensic-share/case/server01_memory.lime format=lime"

# Method 2: avml (static binary, no kernel module needed)
./avml /forensic-share/case/server01_memory.lime

# Hash immediately after acquisition
sha256sum /forensic-share/case/server01_memory.lime
md5sum /forensic-share/case/server01_memory.lime

# Analyze with Volatility 3
vol.py -f server01_memory.lime linux.pslist
vol.py -f server01_memory.lime linux.pstree
vol.py -f server01_memory.lime linux.netstat
vol.py -f server01_memory.lime linux.bash  # bash history from memory