One Hat Cyber Team

Edit File: irqtop

#!/usr/bin/perl
# irqtop
#
# by Robert Elliott, HP
# contributed to the sysstat project
#
#########################################################################
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published  by  the
# Free Software Foundation; either version 2 of the License, or (at  your
# option) any later version.
#
# This program is distributed in the hope that it  will  be  useful,  but
# WITHOUT ANY WARRANTY; without the implied warranty  of  MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#########################################################################
#
# Monitor differences in /proc/interrupts and /proc/softirqs
# per CPU, along with CPU statistics
#
# Usage: irqtop [interval]
#
# Displays interrupts that have occurred since this program
# began, filtering those that have always been zeros.
#
# TODO features:
# * increase column widths automatically
# * add an all-CPU column
# * add option to choose between
#     - online CPUs
#     - just the all-CPU column
#     - select CPUs (e.g., ranges like in smp_affinity_list)
# * automatically determine interrupts to total (e.g.
#   all mpt3sas or hpsaN interrupts) based on them having
#   common prefixes in their names or common names
#
use strict;
use Getopt::Long;

my $myname = "irqtop";
my $myversion = "0.2";

sub print_usage {
		print "Usage: $myname [interval] [-V | --version] [-h | --help]\n\n";
		print "  [internal]        time in seconds between updates (default: 1 second)\n";
		print "  [-V | --version]  display version number\n";
		print "  [-h | --help]     display usage\n\n";
		print "Use Control-C to exit\n";
}

#my $mpstat = "../sysstat/mpstat";
my $mpstat = "mpstat";

# calculate total interrupt stats for this list
#my @track_list = ();
my @track_list = ("eth", "hpsa1", "hpsa2", "mpt3sas");

# exclude these from per-vector display (e.g., eth while studying storage)
# still included in totals if also in track_list
#my @exclude_list = ();
my @exclude_list = ("eth");

my $interval = 1;	# default interval in seconds

# hashes of arrays
# key is the interrupt number or name (e.g., 95 or TIMER)
# array is the interrupt counts and the description on the right
my %current;
my %delta;	# hold the deltas between collection and printing

# hash of values
my %ever_changed; 	# indicates if this row ever changed
my %track_interrupts;	# number of each type

my $online_cpus;	# as seen by process_hardirq, used to filter columns in softirq

# return 1 to exclude, 0 to not
sub is_excluded {
	my ($line) = @_;

foreach (@exclude_list) {
		return 1 if ($line =~ /$_/);
	}
	return 0;
}

# convert affinity_list bitmask (64-bit binary) into numbered list
# Example: 5800a000f -> "0-3,17,19,31-32,34"
# used for parsing /proc/irq/NN/affinity_hint
sub bitmask_to_list {
	my ($bitmask) = @_;
	my $inrange;
	my $rangelist;
	my $rangenew;
	my $lowcpu;
	my $highcpu;

for (my $i = 0; $i < 63; $i++) {
		if (!$inrange && $bitmask & 1) {
			$inrange = 1;
			$lowcpu = $i;
			$highcpu = $i;
		} elsif ($inrange && $bitmask & 1) {
			$highcpu = $i;
		} elsif ($inrange && !($bitmask & 1)) {
			$inrange = 0;
			if ($lowcpu == $highcpu) {
				$rangenew = "$lowcpu";
			} else {
				$rangenew = "$lowcpu-$highcpu";
			}

if ($rangelist) {
				$rangelist = "$rangelist,$rangenew";
			} else {
				$rangelist = "$rangenew";
			}
		}

$bitmask = $bitmask >> 1;
	}
	if ($inrange) {
		$rangelist = "$rangelist,$lowcpu-$highcpu";
	}
	if (!$rangelist) {
		$rangelist = "none";
	}
	return $rangelist;
}

# process /proc/interrupts
# argument:
# 0 do not display - use the first time to not display values since reboot
# 1 display - use on all subsequent calls
sub collect_hardirqs {
	my ($firstpass) = @_;

open HARDIRQFILE, "/proc/interrupts" or die "Cannot open $_";

foreach (@track_list) {
		$track_interrupts{$_} = 0;
	}

# /proc/interrupts lists only online cpus
	# first line contains CPUnn headers for each column
	#            CPU0       CPU1       CPU2       CPU3       CPU4       CPU5       CPU6       CPU7       CPU8       CPU9       CPU10      CPU11
	$_ = <HARDIRQFILE>;
	my @cpulist = split;
	$online_cpus = $#cpulist + 1;	# hardirqs are a source of online_cpus

# remaining lines contain vector number: per-cpu counts, interrupt type, device
	# or vector name, per-CPU counts, interrupt type, description
	while (<HARDIRQFILE>) {
		my @values = split;
		my $irqname = $values[0];
		my $cpu;
		my $line = $_;

for ($cpu = 0; $cpu < $online_cpus; $cpu++) {
			$delta{$irqname}[$cpu] = $values[$cpu + 1] - $current{$irqname}[$cpu];
			$current{$irqname}[$cpu] = $values[$cpu + 1];

# if this is not the first pass,
			# keep track of whether the values have changed
			if ($delta{$irqname}[$cpu] && !$firstpass && !is_excluded($line)) {
				$ever_changed{$irqname} = 1;
			}

foreach (@track_list) {
				if ($line =~ /$_/) {
					$track_interrupts{$_} += $delta{$irqname}[$cpu];
				}
			}
		}
		# capture the rest of the line (interrupt type, handlers)
		# these are not really per-cpu values, but continue storing
		# each word in %current since it's convenient
		for (; $cpu < $#values + 1; $cpu++ ) {
			$current{$irqname}[$cpu] = $values[$cpu + 1];
		}
	}
	close HARDIRQFILE;

}

# process /proc/softirqs
# argument:
# 0 do not display - use the first time to not display values since reboot
# 1 display - use on all subsequent calls
sub collect_softirqs {
	my ($firstpass) = @_;

open SOFTIRQFILE, "/proc/softirqs" or die "Cannot open $_";

# /proc/softirqs includes all possible cpus, not all online cpus
	# this function ignores all those extra cpus
	# first line contains CPUnn headers for each column
	#            CPU0       CPU1       CPU2       CPU3       CPU4       CPU5       CPU6       CPU7       CPU8       CPU9       CPU10      CPU11
	$_ = <SOFTIRQFILE>;
	my @cpulist = split;	# discard the first line

# remaining lines contain
	#    vector number: per-cpu counts, interrupt type, device
	# or   vector name: per-cpu counts, interrupt type, description
	while (<SOFTIRQFILE>) {
		my @values = split;
		my $irqname = $values[0];
		my $line = $_;

# just remember the values for online cpus, not offline cpus
		for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
			$delta{$irqname}[$cpu] = $values[$cpu + 1] - $current{$irqname}[$cpu];
			$current{$irqname}[$cpu] = $values[$cpu + 1];
			if ($delta{$irqname}[$cpu] && !$firstpass && !is_excluded($line)) {
				$ever_changed{$irqname} = 1;
			}
		}
	}
	close SOFTIRQFILE;
}

# print the CPU0 CPU1 CPU2... header line for the online cpus
sub print_cpulist {
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		my $cpustring;
		$cpustring = sprintf("CPU%d", $cpu);
		printf("%6s ", $cpustring);
	}
	print "\n";
}

# print hardirqs and softirqs from %delta that have ever %changed
sub print_irqs {

# header line
	printf("%13s ", "--- IRQs ---");
	# print_cpulist(); # uncomment if print_mpstat is not used, since that prints the header line
	print "\n";

foreach (sort keys %delta) {
		my $irqname = $_;
		# if any values have ever changed for an interrupt, print the delta values
		if ($ever_changed{$irqname}) {
			# match the width of softirq names 12 characters plus :
			printf("%13s ", $irqname);
			my $cpu;
			for ($cpu = 0; $cpu < $online_cpus; $cpu++) {
				printf("%6d ", $delta{$irqname}[$cpu]);
			}
			# print the rest of the line (interrupt type, handlers)
			for (; $cpu < @{$current{$irqname}} + 1; $cpu++ ) {
				print "$current{$irqname}[$cpu] ";
			}

# print the irq smp affinity list, if any
			my $irqname_nocolon = $irqname;
			$irqname_nocolon =~ s/://;

if (!($irqname =~ /[A-Z]/)) {
				my $affinity_hint_file = "/proc/irq/$irqname_nocolon/affinity_hint";
				if (-e $affinity_hint_file) {
					open IRQAFF, "<$affinity_hint_file";
					my $affhint = <IRQAFF>;
					close IRQAFF;
					chomp $affhint;
					my ($affhigh,$afflow) = $affhint =~ /(.*),(.*)/;
					my $aff = hex $affhigh << 32 | hex $afflow;
					printf("hint=%s,", bitmask_to_list($aff));
				} else {
					print "hint=none,";
				}
				my $smp_affinity_list_file = "/proc/irq/$irqname_nocolon/smp_affinity_list";
				if (-e $smp_affinity_list_file) {
					open IRQAFF, "<$smp_affinity_list_file";
					my $afflist = <IRQAFF>;
					close IRQAFF;
					chomp $afflist;
					print "aff=$afflist";
				} else {
					print "aff=none";
				}
			}

print "\n";
		}
	}

# print summary of selected sets of interrupts
	foreach (@track_list) {
		printf("%12s: total=%-7d, average=%-7d; total/s=%-7d, average/s=%-7d\n",
			$_,
			$track_interrupts{$_},
			$track_interrupts{$_} / $online_cpus,
			$track_interrupts{$_} / $interval,
			$track_interrupts{$_} / $interval / $online_cpus);
	}
}

# collect interesting CPU statistics from mpstat
#	%usr, %sys, %iowait, %idle, %irq, %soft
# mpstat displays:
# 06:14:48 PM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle
# plain "mpstat" shows averages since boot, which is not useful
# so, use the interval option based on the first argument to this function,
# which also causes this function to incur the delay.
# per-cpu arrays of value strings
my @usr;
my @sys;
my @irq;
my @soft;
my @iowait;
my @idle;

sub collect_mpstat {
	my ($delay) = @_;

open MPSTATFILE, "$mpstat -P ON -u $delay 1 |" or sleep($delay);
	while (<MPSTATFILE>) {
		if (/CPU/ || /all/ || /^$/ || /Average/) {
			next;
		}
		my ($time, $ampm, $cpu, $usr, $nice, $sys, $iowait, $irq, $soft, $steal, $guest, $idle) = split;
		$usr[$cpu] = $usr;
		$sys[$cpu] = $sys;
		$irq[$cpu] = $irq;
		$soft[$cpu] = $soft;
		$iowait[$cpu] = $iowait;
		$idle[$cpu] = $idle;
		$online_cpus = $cpu + 1;	# mpstat is a source for # online CPUs
	}
	close MPSTATFILE;
}

# print the interesting CPU statistics from mpstat
# collected in collect_mpstat()
sub print_mpstat {

# header line
	printf("%13s ", "CPU usage:");
	print_cpulist();

printf("%13s ", "\%usr:");
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $usr[$cpu]);
	}
	print "\n";
	printf("%13s ", "\%sys:");
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $sys[$cpu]);
	}
	print "\n";
	printf("%13s ", "\%irq:");
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $irq[$cpu]);
	}
	print "\n";
	printf("%13s ", "\%soft:");
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $soft[$cpu]);
	}
	print "\n";
	printf("%13s ", "\%iowait idle:"); # clarify that iowait is really idle time
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $iowait[$cpu]);
	}
	print "\n";
	printf("%13s ", "\%idle:");
	for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
		printf("%6s ", $idle[$cpu]);
	}
	print "\n";
}

#
# start of program
#
foreach (@ARGV) {
	if (/--version/ || /-V/) {
		print "$myname version $myversion\n";
		exit;
	} elsif (/--help/ || /-h/) {
		print_usage();
		exit;
	} elsif (!/[a-z]/) {
		# assume an all-numeric argument is the interval
		$interval = $_;
	}
	print "Collecting CPU and interrupt activity for $interval seconds between updates\n";
}

# remember the clear screen characters to avoid system() during run time
my $clearscreen = `clear`;
collect_hardirqs(1);
collect_softirqs(1);
while (1) {
	collect_hardirqs(0);
	collect_softirqs(0);
	collect_mpstat($interval);
	my $datestring = localtime();
	print "${clearscreen}$myname $datestring interval: $interval s\n";
	print_mpstat();
	print_irqs();

# if collect_mpstat(), which delays, is commented out, sleep here
	# sleep($interval);
}