# Basic process termination commands
kill PID                        # Send TERM signal (graceful termination)
kill -9 PID                     # Send KILL signal (force termination)
kill -TERM PID                  # Explicit graceful termination
kill -HUP PID                   # Hang up signal (often reloads configuration)
kill -INT PID                   # Interrupt signal (equivalent to Ctrl+C)

# Killing processes by name or pattern
killall process_name            # Kill all processes with specific name
killall mosquitto               # Kill all MQTT broker processes
pkill pattern                   # Kill processes matching pattern
pkill -f "python.*sensor"      # Kill Python processes containing "sensor"
pkill -u iot                    # Kill all processes owned by 'iot' user

# Signal types and their purposes
kill -l                         # List all available signals
# Common signals for IoT process management:
# TERM (15) - Graceful termination (default, allows cleanup)
# KILL (9)  - Force kill (cannot be caught or ignored)
# HUP (1)   - Hang up (often triggers config reload)
# INT (2)   - Interrupt (Ctrl+C equivalent)
# STOP (19) - Pause process (cannot be caught)
# CONT (18) - Continue paused process
# USR1 (10) - User-defined signal 1 (application-specific)
# USR2 (12) - User-defined signal 2 (application-specific)

# Professional IoT process management examples
# Gracefully restart MQTT broker
sudo kill -HUP $(pgrep mosquitto)              # Reload configuration
sudo systemctl reload mosquitto                # Alternative using systemd

# Stop sensor data collection safely
kill -TERM $(pgrep -f "sensor_collector")      # Allow data to be saved
sleep 5                                         # Wait for graceful shutdown
kill -9 $(pgrep -f "sensor_collector") 2>/dev/null  # Force kill if still running

# Emergency process cleanup
sudo pkill -9 -f "runaway_process"             # Force kill problematic process
sudo pkill -TERM -u iot                        # Gracefully stop all IoT user processes

# Process management with PID files (common in IoT daemons)
# Many IoT services store their PID in files for management
kill -HUP $(cat /var/run/iot_daemon.pid)       # Reload daemon using PID file
kill -TERM $(cat /run/sensor_service.pid)      # Stop service using PID file

# Monitoring process termination
kill -TERM $PID && echo "Termination signal sent"
while kill -0 $PID 2>/dev/null; do             # Check if process still exists
    echo "Waiting for process to terminate..."
    sleep 1
done
echo "Process terminated successfully"

# Running processes in background
command &                       # Start command in background
python3 sensor_monitor.py &     # Run sensor monitoring in background
nohup command &                 # Run command immune to hangups (terminal closure)
nohup python3 data_logger.py > /dev/null 2>&1 &  # Silent background execution

# Professional background execution for IoT
nohup python3 /opt/iot/sensor_reader.py > /var/log/iot/sensor.log 2>&1 &
nohup mosquitto -c /etc/mosquitto/mosquitto.conf > /var/log/iot/mqtt.log 2>&1 &
nohup node /opt/iot/dashboard/server.js > /var/log/iot/dashboard.log 2>&1 &

# Job control commands
jobs                            # List active background jobs
jobs -l                         # List jobs with process IDs
fg                              # Bring last background job to foreground
fg %1                           # Bring job number 1 to foreground
bg                              # Send last suspended job to background
bg %2                           # Send job number 2 to background

# Process suspension and control
Ctrl+Z                          # Suspend current foreground process
Ctrl+C                          # Interrupt (terminate) current foreground process
kill -STOP PID                  # Suspend process by PID
kill -CONT PID                  # Resume suspended process by PID

# Advanced background process management
disown %1                       # Remove job from shell's job table
disown -a                       # Remove all jobs from job table
disown -h %1                    # Mark job to not receive SIGHUP

# Screen and tmux for persistent sessions (essential for IoT)
screen -S iot_monitor python3 sensor_monitor.py    # Run in named screen session
screen -ls                      # List screen sessions
screen -r iot_monitor           # Reattach to screen session

tmux new-session -d -s iot_dashboard "python3 /opt/iot/dashboard.py"  # Tmux session
tmux list-sessions              # List tmux sessions
tmux attach-session -t iot_dashboard  # Attach to tmux session

# IoT daemon management examples
# Start IoT services that survive terminal disconnection
nohup /opt/iot/bin/temperature_service &
echo $! > /var/run/temperature_service.pid  # Save PID for later management

# Monitor background IoT processes
jobs -l | grep -E "(sensor|iot|mqtt)"      # List IoT-related background jobs
ps aux | grep -E "(sensor|iot|mqtt)" | grep -v grep  # Check running IoT processes

# Automated background process management
#!/bin/bash
# IoT service startup script
start_iot_services() {
    echo "Starting IoT services..."
    nohup python3 /opt/iot/sensor_collector.py > /var/log/iot/collector.log 2>&1 &
    echo $! > /var/run/iot_collector.pid
    
    nohup python3 /opt/iot/data_processor.py > /var/log/iot/processor.log 2>&1 &
    echo $! > /var/run/iot_processor.pid
    
    echo "IoT services started successfully"
}

# Process monitoring and restart
monitor_process() {
    local pid_file=$1
    local command=$2
    
    if [ -f "$pid_file" ]; then
        local pid=$(cat "$pid_file")
        if ! kill -0 "$pid" 2>/dev/null; then
            echo "Process died, restarting..."
            nohup $command > /var/log/iot/restart.log 2>&1 &
            echo $! > "$pid_file"
        fi
    fi
}

# Understanding nice values and process priorities
# Nice values range: -20 (highest priority) to +19 (lowest priority)
# Default nice value: 0
# Only root can set negative values (higher priority)
# Lower nice value = higher priority = more CPU time

# Starting processes with specific priority
nice -n 10 backup_script.sh     # Start with lower priority (nice value 10)
nice -n -5 critical_iot_daemon   # Start with higher priority (root only)
nice --adjustment=5 data_processor.py  # Alternative syntax

# Changing priority of running processes
renice 10 PID                    # Change process priority to nice value 10
renice -n -5 PID                 # Set higher priority (root only)
renice +5 -u iot                 # Change priority for all 'iot' user processes
renice 0 -g iot_group            # Reset priority for process group

# Viewing process priorities
ps -eo pid,ni,cmd                # Show PID, nice value, and command
ps aux | grep PID                # Check specific process priority in NI column
top                              # Shows NI column for nice values
htop                             # Color-coded priority display

# IoT priority management strategies and examples
# High priority for critical IoT processes
sudo renice -10 $(pgrep mosquitto)          # High priority MQTT broker
sudo renice -5 $(pgrep -f "critical_sensor") # Critical sensor monitoring

# Medium priority for normal IoT operations
nice -n 0 python3 /opt/iot/sensor_reader.py &    # Default priority
renice 5 $(pgrep -f "data_logger")               # Slightly lower priority logging

# Low priority for background maintenance
nice -n 19 /opt/iot/scripts/cleanup.sh &        # Lowest priority cleanup
nice -n 15 rsync -av /data/ /backup/ &          # Low priority backup
nice -n 10 /opt/iot/scripts/log_rotation.sh &   # Low priority maintenance

# Real-world IoT priority scenarios
# Emergency response system
sudo nice -n -10 /opt/iot/emergency_monitor &   # Highest priority emergency monitoring
nice -n 5 /opt/iot/routine_sensors.py &         # Lower priority routine sensors

# Battery-powered IoT device optimization
nice -n 10 /opt/iot/power_saving_mode.py &      # Lower priority to save power
sudo renice -5 $(pgrep -f "battery_monitor")    # High priority battery monitoring

# Industrial IoT system
sudo renice -15 $(pgrep -f "safety_system")     # Critical safety monitoring
renice 0 $(pgrep -f "production_monitor")       # Normal priority production
nice -n 15 /opt/iot/analytics/report_generator.py &  # Low priority analytics

# Monitoring and adjusting priorities dynamically
#!/bin/bash
# Dynamic priority adjustment based on system load
adjust_priorities() {
    local load=$(cat /proc/loadavg | cut -d' ' -f1)
    local load_int=$(echo "$load * 100" | bc | cut -d'.' -f1)
    
    if [ $load_int -gt 150 ]; then  # Load > 1.5
        echo "High load detected, adjusting priorities..."
        renice 10 $(pgrep -f "backup")      # Lower backup priority
        renice 15 $(pgrep -f "analytics")   # Lower analytics priority
        sudo renice -5 $(pgrep -f "critical")  # Raise critical process priority
    fi
}

# Process resource limiting (advanced topic)
# Using systemd for IoT service resource management
# /etc/systemd/system/iot-sensor.service
[Unit]
Description=IoT Sensor Service
After=network.target

[Service]
Type=simple
User=iot
ExecStart=/opt/iot/bin/sensor_service
Nice=5                    # Set nice value
CPUQuota=50%             # Limit to 50% CPU
MemoryLimit=128M         # Limit memory usage
Restart=always

[Install]
WantedBy=multi-user.target

# Monitoring resource usage and priorities
watch "ps aux --sort=-%cpu | head -10"          # Monitor CPU usage
watch "ps -eo pid,ni,pcpu,pmem,cmd --sort=-pcpu | head -15"  # Monitor with priorities

# Exercise 1: IoT Process Monitoring and Optimization
# Set up comprehensive monitoring for IoT systems

# Step 1: Create sample IoT processes for monitoring
# Temperature sensor simulator
cat > /tmp/temp_sensor.py << 'EOF'
#!/usr/bin/env python3
import time
import random
import json
import os

while True:
    temp = 20 + random.uniform(-5, 15)
    data = {"temperature": temp, "timestamp": time.time()}
    print(f"Temperature: {temp:.2f}°C")
    
    # Simulate some CPU work
    for i in range(10000):
        pass
    
    time.sleep(2)
EOF

# Data processor simulator
cat > /tmp/data_processor.py << 'EOF'
#!/usr/bin/env python3
import time
import random

while True:
    # Simulate data processing
    print("Processing sensor data...")
    
    # Simulate variable CPU load
    work_amount = random.randint(50000, 200000)
    for i in range(work_amount):
        pass
    
    time.sleep(5)
EOF

chmod +x /tmp/temp_sensor.py /tmp/data_processor.py

# Step 2: Start processes with different priorities
nice -n 5 python3 /tmp/temp_sensor.py > /tmp/sensor.log 2>&1 &
SENSOR_PID=$!
echo $SENSOR_PID > /tmp/sensor.pid

nice -n 10 python3 /tmp/data_processor.py > /tmp/processor.log 2>&1 &
PROCESSOR_PID=$!
echo $PROCESSOR_PID > /tmp/processor.pid

# Step 3: Monitor process performance
echo "=== IoT Process Monitoring ==="
ps -eo pid,ni,pcpu,pmem,cmd | grep -E "(temp_sensor|data_processor)"
echo ""
echo "=== System Resource Usage ==="
free -h
echo ""
echo "=== Process Details ==="
ps aux | grep -E "(temp_sensor|data_processor)" | grep -v grep

# Step 4: Adjust priorities based on monitoring
echo "Adjusting process priorities..."
renice 0 $SENSOR_PID                        # Normal priority for sensor
renice 15 $PROCESSOR_PID                    # Lower priority for processor

# Step 5: Create monitoring script
cat > /tmp/iot_monitor.sh << 'EOF'
#!/bin/bash
echo "=== IoT System Health Monitor ==="
echo "Time: $(date)"
echo "Uptime: $(uptime)"
echo "Memory: $(free -h | grep Mem | awk '{print $3 "/" $2}')"
echo "Load: $(cat /proc/loadavg | cut -d' ' -f1-3)"
echo ""
echo "=== IoT Processes ==="
ps aux | grep -E "(temp_sensor|data_processor)" | grep -v grep
echo ""
echo "=== Resource Top Consumers ==="
ps aux --sort=-%cpu | head -5
EOF

chmod +x /tmp/iot_monitor.sh
/tmp/iot_monitor.sh

# Exercise 2: Custom systemd Service Creation
# Create production-ready IoT services

# Step 1: Create IoT application directory structure
sudo mkdir -p /opt/iot/{bin,config,logs}
sudo mkdir -p /var/lib/iot
sudo mkdir -p /var/log/iot

# Step 2: Create sample IoT application
sudo tee /opt/iot/bin/iot_temperature_service.py > /dev/null << 'EOF'
#!/usr/bin/env python3
import time
import json
import random
import signal
import sys
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

class TemperatureService:
    def __init__(self):
        self.running = True
        signal.signal(signal.SIGTERM, self.signal_handler)
        signal.signal(signal.SIGINT, self.signal_handler)
        
    def signal_handler(self, signum, frame):
        logger.info(f"Received signal {signum}, shutting down gracefully...")
        self.running = False
        
    def read_temperature(self):
        # Simulate temperature reading
        return 20 + random.uniform(-5, 15)
        
    def run(self):
        logger.info("IoT Temperature Service starting...")
        
        while self.running:
            try:
                temp = self.read_temperature()
                data = {
                    "temperature": temp,
                    "timestamp": time.time(),
                    "unit": "celsius"
                }
                
                logger.info(f"Temperature reading: {temp:.2f}°C")
                
                # Write to data file
                with open("/var/lib/iot/temperature.json", "w") as f:
                    json.dump(data, f)
                    
                time.sleep(10)
                
            except Exception as e:
                logger.error(f"Error in temperature service: {e}")
                time.sleep(5)
                
        logger.info("IoT Temperature Service stopped")

if __name__ == "__main__":
    service = TemperatureService()
    service.run()
EOF

sudo chmod +x /opt/iot/bin/iot_temperature_service.py

# Step 3: Create systemd service file
sudo tee /etc/systemd/system/iot-temperature.service > /dev/null << 'EOF'
[Unit]
Description=IoT Temperature Monitoring Service
Documentation=https://example.com/iot-docs
After=network.target
Wants=network.target

[Service]
Type=simple
User=iot
Group=iot
WorkingDirectory=/opt/iot
ExecStart=/usr/bin/python3 /opt/iot/bin/iot_temperature_service.py
ExecReload=/bin/kill -HUP $MAINPID
Restart=always
RestartSec=10
TimeoutStopSec=30

# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=iot-temperature

# Resource limits for IoT devices
MemoryLimit=64M
CPUQuota=25%

# Security settings
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ReadWritePaths=/var/lib/iot /var/log/iot
ProtectHome=true
ProtectKernelTunables=true
ProtectControlGroups=true

[Install]
WantedBy=multi-user.target
EOF

# Step 4: Set up service user and permissions
sudo useradd -r -s /bin/false iot 2>/dev/null || true
sudo chown -R iot:iot /opt/iot /var/lib/iot /var/log/iot

# Step 5: Enable and start the service
sudo systemctl daemon-reload
sudo systemctl enable iot-temperature
sudo systemctl start iot-temperature

# Step 6: Monitor the service
echo "=== Service Status ==="
systemctl status iot-temperature --no-pager
echo ""
echo "=== Service Logs ==="
journalctl -u iot-temperature -n 10 --no-pager

# Exercise 3: Automated Process Monitoring and Recovery
# Create comprehensive monitoring system

# Step 1: Create process monitoring script
sudo tee /opt/iot/bin/iot_process_monitor.sh > /dev/null << 'EOF'
#!/bin/bash

# Configuration
LOG_FILE="/var/log/iot/process_monitor.log"
ALERT_THRESHOLD_CPU=80
ALERT_THRESHOLD_MEM=85
CHECK_INTERVAL=60

# Services to monitor
IOT_SERVICES=("iot-temperature" "mosquitto")

# Logging function
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

# Check system resources
check_system_resources() {
    # CPU usage
    local cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
    if (( $(echo "$cpu_usage > $ALERT_THRESHOLD_CPU" | bc -l) )); then
        log_message "ALERT: High CPU usage: ${cpu_usage}%"
    fi
    
    # Memory usage
    local mem_usage=$(free | grep Mem | awk '{printf "%.0f", $3/$2 * 100.0}')
    if [ "$mem_usage" -gt "$ALERT_THRESHOLD_MEM" ]; then
        log_message "ALERT: High memory usage: ${mem_usage}%"
    fi
    
    # Disk usage
    local disk_usage=$(df / | tail -1 | awk '{print $5}' | cut -d'%' -f1)
    if [ "$disk_usage" -gt 90 ]; then
        log_message "ALERT: High disk usage: ${disk_usage}%"
    fi
}

# Check IoT services
check_iot_services() {
    for service in "${IOT_SERVICES[@]}"; do
        if systemctl is-active --quiet "$service"; then
            log_message "INFO: Service $service is running"
        else
            log_message "ALERT: Service $service is not running"
            restart_service "$service"
        fi
    done
}

# Restart failed service
restart_service() {
    local service=$1
    log_message "INFO: Attempting to restart $service"
    
    if systemctl restart "$service"; then
        log_message "INFO: Successfully restarted $service"
    else
        log_message "ERROR: Failed to restart $service"
    fi
}

# Main monitoring loop
main() {
    log_message "INFO: Starting IoT process monitoring"
    
    while true; do
        check_system_resources
        check_iot_services
        sleep "$CHECK_INTERVAL"
    done
}

# Handle signals for graceful shutdown
trap 'log_message "INFO: Process monitor shutting down"; exit 0' SIGTERM SIGINT

main "$@"
EOF

sudo chmod +x /opt/iot/bin/iot_process_monitor.sh

# Step 2: Create systemd service for the monitor
sudo tee /etc/systemd/system/iot-process-monitor.service > /dev/null << 'EOF'
[Unit]
Description=IoT Process Monitor
After=multi-user.target

[Service]
Type=simple
User=root
ExecStart=/opt/iot/bin/iot_process_monitor.sh
Restart=always
RestartSec=30

[Install]
WantedBy=multi-user.target
EOF

# Step 3: Enable and start the monitor
sudo systemctl daemon-reload
sudo systemctl enable iot-process-monitor
sudo systemctl start iot-process-monitor

# Exercise 4: Troubleshooting Complex Process Issues
# Simulate and resolve common IoT process problems

# Step 1: Create problematic process for troubleshooting
cat > /tmp/problematic_iot_app.py << 'EOF'
#!/usr/bin/env python3
import time
import random
import os

# Simulate memory leak
data_accumulator = []

while True:
    # Memory leak simulation
    for i in range(1000):
        data_accumulator.append(f"sensor_reading_{i}_{time.time()}")
    
    # CPU spike simulation
    if random.random() < 0.3:  # 30% chance
        print("CPU spike simulation")
        for i in range(1000000):
            pass
    
    # Occasional crash simulation
    if random.random() < 0.1:  # 10% chance
        print("Simulating application crash")
        os._exit(1)
    
    print(f"Memory usage growing... {len(data_accumulator)} items")
    time.sleep(5)
EOF

chmod +x /tmp/problematic_iot_app.py

# Step 2: Start problematic process and monitor
python3 /tmp/problematic_iot_app.py &
PROBLEM_PID=$!

# Step 3: Diagnostic commands for troubleshooting
echo "=== Troubleshooting IoT Process Issues ==="
echo "Problem PID: $PROBLEM_PID"
echo ""

# Monitor process behavior
echo "=== Process Information ==="
ps -p $PROBLEM_PID -o pid,ppid,pcpu,pmem,vsz,rss,stat,start,time,cmd

echo ""
echo "=== Memory Usage Tracking ==="
while kill -0 $PROBLEM_PID 2>/dev/null; do
    ps -p $PROBLEM_PID -o pid,pmem,vsz,rss --no-headers
    sleep 2
done &
MONITOR_PID=$!

# Let it run for a bit then clean up
sleep 20
kill $PROBLEM_PID 2>/dev/null
kill $MONITOR_PID 2>/dev/null

# Cleanup all test processes
echo ""
echo "=== Cleaning up test processes ==="
[ -f /tmp/sensor.pid ] && kill $(cat /tmp/sensor.pid) 2>/dev/null
[ -f /tmp/processor.pid ] && kill $(cat /tmp/processor.pid) 2>/dev/null
sudo systemctl stop iot-temperature 2>/dev/null
sudo systemctl stop iot-process-monitor 2>/dev/null