Lesson 2 of 3 · 25 min read

AWS CloudTrail Investigation — IAM Abuse, Credential Compromise, and Data Exfiltration

AWS IR is log-centric: CloudTrail provides a complete record of every API call. The challenge is not evidence availability but signal-to-noise — AWS environments generate thousands of legitimate API calls daily.

CloudTrail Investigation Queries

# Query CloudTrail with AWS CLI (event history — last 90 days)

# Find all API calls from a specific access key
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=AccessKeyId,AttributeValue=AKIAIOSFODNN7EXAMPLE \
  --start-time 2024-09-01 \
  --end-time 2024-09-03 \
  --output json

# Find all CreateUser events in last 24 hours
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=CreateUser \
  --start-time $(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) \
  --output json

# Find all console logins
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=ConsoleLogin \
  --start-time 2024-09-01 \
  --output json | jq '.Events[] | {time: .EventTime, user: .Username, ip: (.CloudTrailEvent | fromjson | .sourceIPAddress), mfa: (.CloudTrailEvent | fromjson | .additionalEventData.MFAUsed)}'

Athena Queries for Large-Scale CloudTrail Analysis

-- CloudTrail logs in S3 analyzed via Athena

-- 1. Find IAM backdoor creation sequence
SELECT eventTime, userIdentity.arn, eventName, sourceIPAddress,
       requestParameters
FROM cloudtrail_logs
WHERE eventName IN ('CreateUser', 'CreateAccessKey', 'AttachUserPolicy', 
                    'CreateLoginProfile', 'AddUserToGroup')
  AND eventTime > '2024-09-01'
ORDER BY eventTime;

-- 2. Find S3 data access volume by principal
SELECT userIdentity.arn, sourceIPAddress,
       COUNT(*) as request_count,
       SUM(CAST(responseElements AS VARCHAR) LIKE '%ContentLength%') as downloads
FROM cloudtrail_logs
WHERE eventName = 'GetObject'
  AND eventTime > '2024-09-01'
GROUP BY userIdentity.arn, sourceIPAddress
ORDER BY request_count DESC
LIMIT 20;

-- 3. Find AssumeRole chains (lateral movement)
SELECT eventTime, userIdentity.arn, 
       requestParameters,
       sourceIPAddress
FROM cloudtrail_logs
WHERE eventName = 'AssumeRole'
  AND eventTime > '2024-09-01'
ORDER BY eventTime;

-- 4. Find GetCallerIdentity calls (attacker verifying credentials)
SELECT eventTime, userIdentity.arn, sourceIPAddress, userAgent
FROM cloudtrail_logs  
WHERE eventName = 'GetCallerIdentity'
  AND sourceIPAddress NOT IN ('amazonaws.com', 'cloudformation.amazonaws.com')
ORDER BY eventTime DESC
LIMIT 100;

AWS IR Containment

# Step 1: Disable the compromised access key
aws iam update-access-key \
  --access-key-id AKIAIOSFODNN7EXAMPLE \
  --status Inactive \
  --user-name jsmith

# Step 2: Revoke all active console sessions
# (Force logout by ending all existing sessions)
aws iam delete-login-profile --user-name jsmith  # Removes console access

# Step 3: Detach all policies from compromised principal
aws iam list-attached-user-policies --user-name jsmith
aws iam detach-user-policy \
  --user-name jsmith \
  --policy-arn arn:aws:iam::aws:policy/AdministratorAccess

# Step 4: Check for attacker-created backdoor users
aws iam list-users --query 'Users[?CreateDate>`2024-09-01`]' --output table
# List and investigate any users created after the suspected compromise time

# Step 5: Check for new access keys on all users
aws iam generate-credential-report && sleep 10
aws iam get-credential-report --query 'Content' --output text | base64 -d | \
  grep -E "access_key_1_active|access_key_2_active" | \
  awk -F, '{if($9=="true" || $14=="true") print $1}'

# Step 6: Isolate attacker-launched EC2 instances
# Find instances launched during attack window
aws ec2 describe-instances \
  --filters "Name=launch-time,Values=2024-09-01*" \
  --query 'Reservations[].Instances[].[InstanceId,LaunchTime,State.Name,PublicIpAddress]' \
  --output table

# Isolate by modifying security group to block all traffic
aws ec2 modify-instance-attribute \
  --instance-id i-1234567890abcdef0 \
  --groups sg-isolate-ir  # A security group with no inbound/outbound rules