897 lines
31 KiB
Perl
Executable File
897 lines
31 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
|
|
#Copyright (c) 2016 Wind River Systems, Inc.
|
|
#
|
|
#SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
# Usage:
|
|
# parse_filestats [--list | --all | --name <pattern>] file1 file2 file3.gz ...
|
|
# [--pid <pid1>] ...
|
|
# [--cmd <string>] ...
|
|
# [--exact <string> ...]
|
|
# [--excl <string>] ...
|
|
# [--detail]
|
|
# [--thresh <MiB/d>]
|
|
# [--transient]
|
|
# [--dir <path>]
|
|
# [--dur <days>]
|
|
# [--report]
|
|
# [--help]
|
|
#
|
|
# Purpose: Parse and summarize file descriptor usage per process by name/pid.
|
|
# Display hirunners based on daily growth rate.
|
|
#
|
|
# Modification history:
|
|
# - 2016-Oct-06 - Lachlan Plant, created based on parse_memstats.
|
|
|
|
##############################################################################
|
|
use 5.10.0;
|
|
use warnings;
|
|
use strict;
|
|
use Time::Local 'timelocal_nocheck'; # inverse time functions
|
|
use File::Basename;
|
|
use File::Spec ();
|
|
use Data::Dumper;
|
|
my $SCRIPT = basename($0);
|
|
my $DEFAULT_PATH = ".";
|
|
my $iter = 0;
|
|
|
|
# Hash storage data-structures
|
|
my (%data, %overall, %timestamps, %days, %matched, %stats) = ();
|
|
my (%files_current, %file_stats, %file_slope) = ();
|
|
|
|
# Timestamp variables
|
|
my ($wday, $month, $day, $hh, $mm, $ss, $yy, $ns) = ();
|
|
|
|
# Uptime data
|
|
my ($uptime, $idle);
|
|
|
|
my ($ftotal, $files, $sockets, $pipes);
|
|
# Process data
|
|
my ($PID, $TOTAL, $FD ,$U, $W, $R, $CWD, $RTD, $TXT, $MEM, $DEL, $TCP, $COMMAND, ) = ();
|
|
|
|
|
|
# Free memory
|
|
my ($is_strict) = (0);
|
|
my ($avail_free_mem_MB, $unlinked_files_MB) = ();
|
|
my ($fs_root_MB, $fs_root_p_use, $fs_tmp_MB, $fs_tmp_p_use) = ();
|
|
|
|
# Argument list parameters
|
|
our ($arg_all, $arg_match, $arg_name,
|
|
@arg_pids, @arg_commands, @arg_exact, @arg_commands_excl,
|
|
$arg_list, $arg_detail, $arg_thresh, $arg_transient, $arg_path, $arg_dur,
|
|
$arg_report, @arg_files) = ();
|
|
|
|
# Determine location of gunzip binary
|
|
our $GUNZIP = which('gunzip');
|
|
if (!(defined $GUNZIP)) {
|
|
die "*error* cannot find 'gunzip' binary. Cannot continue.\n";
|
|
}
|
|
our $BUNZIP2 = which('bunzip2');
|
|
if (!(defined $BUNZIP2)) {
|
|
die "*error* cannot find 'bunzip2' binary. Cannot continue.\n";
|
|
}
|
|
|
|
# Parse input arguments and print tool usage if necessary
|
|
# -- note: @arg_pids, and @arg_commands are only defined if they are set
|
|
&get_parse_filestats_args(\$arg_all, \$arg_match, \$arg_name,
|
|
\@arg_pids, \@arg_commands, \@arg_exact, \@arg_commands_excl,
|
|
\$arg_list, \$arg_detail, \$arg_thresh, \$arg_transient,
|
|
\$arg_path, \$arg_dur, \$arg_report, \@arg_files);
|
|
|
|
# Print list of file information
|
|
if (defined $arg_list) {
|
|
my @list = (); my %chrono = (); my ($host, $time) = ();
|
|
opendir(DIR, $arg_path) || die "can't opendir $arg_path: ($!)";
|
|
@list = sort {$a cmp $b}
|
|
grep { /_(\d{4}-\d{2}-\d{2}_\d{4})_filestats?.?g?z\b/ && -f "$arg_path/$_" }
|
|
readdir(DIR);
|
|
closedir DIR;
|
|
foreach my $file (@list) {
|
|
$_ = $file;
|
|
($host, $time) = /(.*)_(\d{4}-\d{2}-\d{2}_\d{4})_filestats/;
|
|
$chrono{$host}{$time} = 1;
|
|
}
|
|
# Print out summary of hosts with oldests and newest files
|
|
printf "%s: List of available 'filestat' data:\n\n", $SCRIPT;
|
|
printf "%-20s %15s %15s\n", "host", "oldest", "newest";
|
|
printf "%-20s %15s %15s\n", "-"x20, "-"x15, "-"x15;
|
|
foreach $host (sort keys %chrono) {
|
|
my @times = sort {$a cmp $b} keys %{$chrono{$host}};
|
|
printf "%-20s %15s %15s\n", $host, $times[0], $times[-1];
|
|
}
|
|
exit 1;
|
|
}
|
|
|
|
# Print selected options (except for file list)
|
|
if ((@arg_pids) || (@arg_commands)) {
|
|
printf "selected pids/patterns: @arg_pids @arg_commands\n";
|
|
}
|
|
printf "this may take a while...\n";
|
|
|
|
# Determine file list based on smart file GLOB
|
|
if (!@arg_files) {
|
|
if (defined $arg_name) {
|
|
@arg_files = <$arg_path/*$arg_name*>;
|
|
} else {
|
|
@arg_files = <$arg_path/*file*>
|
|
}
|
|
if (!@arg_files) {
|
|
printf "no files selected.\n";
|
|
}
|
|
}
|
|
|
|
# Compile regular expressions command string patterns
|
|
# -- store list of expressions to INCLUDE
|
|
my @re_commands = ();
|
|
foreach my $arg (@arg_commands) {
|
|
push @re_commands, qr/\Q$arg\E/;
|
|
}
|
|
my @re_exact_commands = ();
|
|
foreach my $arg (@arg_exact) {
|
|
push @re_exact_commands, qr/\Q$arg\E/;
|
|
}
|
|
# -- store list of expressions to EXCLUDE
|
|
my @nre_commands = ();
|
|
push @arg_commands_excl, $SCRIPT;
|
|
foreach my $arg (@arg_commands_excl) {
|
|
push @nre_commands, qr/\Q$arg\E/;
|
|
}
|
|
|
|
# Determine list of files per matching hostname
|
|
my %filenames = ();
|
|
foreach my $file (sort @arg_files) {
|
|
|
|
if ($file !~ /_\d{4}-\d{2}-\d{2}_\d{4}_file/) {
|
|
printf "ignoring: '$file', does not match '_<yyyy>-<mm>-<dd>-<hhmm>_file' format.\n";
|
|
next;
|
|
}
|
|
|
|
my $host = $file; $host =~ s/_\d{4}-\d{2}-\d{2}_\d{4}_.*$//; $host =~ s/^.*\///;
|
|
printf "host = $host, file = $file\n";
|
|
push @{$filenames{$host}}, $file;
|
|
}
|
|
|
|
# Prune file list retain most recent number of --dur <days> per host
|
|
my $max_files = int($arg_dur*24);
|
|
foreach my $host (keys %filenames) {
|
|
if (scalar(@{$filenames{$host}}) > $max_files) { # prune to size of max_files, keeping end of list (most recent)
|
|
@{$filenames{$host}} = splice(@{$filenames{$host}},-$max_files);
|
|
}
|
|
}
|
|
|
|
my $seconds_in_day = 24*60*60;
|
|
my $first_time = 0.0;
|
|
my $last_time = 0.0;
|
|
|
|
# PROCESS ALL MATCHING HOSTS
|
|
# -- then process all files per host in chronological order
|
|
foreach my $host (sort keys %filenames) {
|
|
my $pass = 1;
|
|
REPEAT_CALCULATION:
|
|
$iter = 0; $first_time = 0.0;
|
|
%matched = ();
|
|
%data = ();
|
|
%timestamps = ();
|
|
%overall = ();
|
|
%days = ();
|
|
%stats = ();
|
|
%file_stats = ();
|
|
|
|
|
|
# Evalutate first and last filename's time and convert time to days relative to first_time
|
|
my $first_file = ${$filenames{$host}}[0];
|
|
my $last_file = ${$filenames{$host}}[-1];
|
|
$_ = $first_file; ($yy, $month, $day, $hh) = /_(\d{4})-(\d{2})-(\d{2})_(\d{2})\d{2}_file/;
|
|
$first_time = timelocal_nocheck(00, 00, $hh, $day, $month-1, $yy-1900)/$seconds_in_day;
|
|
my $first_date = sprintf("%4d-%02d-%02d %02d:00", $yy, $month, $day, $hh);
|
|
$_ = $last_file; ($yy, $month, $day, $hh) = /_(\d{4})-(\d{2})-(\d{2})_(\d{2})\d{2}_file/;
|
|
$last_time = timelocal_nocheck(59, 59, $hh, $day, $month-1, $yy-1900)/$seconds_in_day - $first_time;
|
|
my $last_date = sprintf("%4d-%02d-%02d %02d:00", $yy, $month, $day, $hh);
|
|
|
|
FILE_LIST: foreach my $file ( @{$filenames{$host}} ) {
|
|
my $FOUND = 0; # handle files being decompressed while parser is running
|
|
if ( -e $file ) {
|
|
if ($file =~ /\.gz$/) {
|
|
open(FILE, "$::GUNZIP -c $file |") || die "Cannot open file: $file ($!)\n";
|
|
} elsif ($file =~ /\.bz2$/) {
|
|
open(FILE, "$::BUNZIP2 -c $file |") || die "Cannot open file: $file ($!)\n";
|
|
} else {
|
|
open(FILE, $file) || die "Cannot open file: $file ($!)\n";
|
|
}
|
|
$FOUND = 1;
|
|
} else {
|
|
if ($file =~ /\.gz$/) {$file =~ s/\.gz//;} else {$file .= '.gz';}
|
|
if ($file =~ /\.bz2$/) {$file =~ s/\.bz2//;} else {$file .= '.bz2';}
|
|
if ( -e $file ) {
|
|
if ($file =~ /\.gz$/) {
|
|
open(FILE, "$::GUNZIP -c $file |") || die "Cannot open file: $file ($!)\n";
|
|
} elsif ($file =~ /\.bz2$/) {
|
|
open(FILE, "$::BUNZIP2 -c $file |") || die "Cannot open file: $file ($!)\n";
|
|
} else {
|
|
open(FILE, $file) || die "Cannot open file: $file ($!)\n";
|
|
}
|
|
$FOUND = 1;
|
|
}
|
|
}
|
|
next if ($FOUND == 0);
|
|
|
|
# Parse file line at a time
|
|
READ_LOOP: while($_ = <FILE>) {
|
|
s/[\0\e\f\r\a]//g; chomp; # strip control characters if any
|
|
|
|
# START OF SAMPLE Time: Parse hires timestamp, ignore timezone
|
|
if (/time:\s+(\w+)\s+(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})\.(\d{9})\s+\w+\s+\S+\s+uptime:\s+(\S+)\s+(\S+)/) {
|
|
$wday = $1; $yy = $2; $month = $3; $day = $4; $hh = $5; $mm = $6; $ss = $7; $ns = $8; $uptime = $9; $idle = $10;
|
|
$FOUND = -1 if (($FOUND == 1) || ($FOUND == -2));
|
|
$timestamps{$iter} = [($wday,$month,$day,$hh,$mm,$ss,$yy)];
|
|
$days{$iter} = timelocal_nocheck($ss, $mm, $hh, $day, $month-1, $yy-1900)/$seconds_in_day - $first_time;
|
|
($ftotal, $files, $sockets, $pipes) = (0, 0, 0, 0);
|
|
%files_current = ();
|
|
next READ_LOOP;
|
|
}
|
|
|
|
## ls -l /proc/*/fd
|
|
#TOTAL FILES SOCKETS NULL PIPES
|
|
#4955 3385 1071 499
|
|
|
|
if (/^MEMINFO:/ || /^ls -l \/proc\/*\/fd/) {
|
|
# handle case where we detect the sample is incomplete, and delete
|
|
if ($FOUND != -1) {
|
|
close(FILE);
|
|
delete $days{$iter} if (defined $days{$iter});
|
|
delete $timestamps{$iter} if (defined $timestamps{$iter});
|
|
delete $data{$iter} if (defined $data{$iter});
|
|
delete $overall{$iter} if (defined $overall{$iter});
|
|
next FILE_LIST;
|
|
}
|
|
my $hcnt = 0;
|
|
# find headings line
|
|
HEADER_LOOP: while($_ = <FILE>) {
|
|
s/[\0\e\f\r\a]//g; chomp; # strip control characters if any
|
|
$hcnt++;
|
|
last HEADER_LOOP if (/\bTOTAL\b/); # end titles-line
|
|
next READ_LOOP if (($hcnt == 1) && /^\s*$/); # end at blank-line (eg no 'ps' output)
|
|
}
|
|
next if ! $_; # sometimes $_ can be null (at EOF) and causes subsequent warnings
|
|
# Process all entries of MEMINFO
|
|
MEMINFO_LOOP: while($_ = <FILE>) {
|
|
s/[\0\e\f\r\a]//g; chomp; # strip control characters if any
|
|
last MEMINFO_LOOP if (/^\s*$/); # end at blank-line
|
|
($ftotal, $files, $sockets, $pipes) = /^(\d+)\s+(\d+)\s+(\d+)\s+(\d+).*/
|
|
|
|
}
|
|
|
|
}
|
|
next if ! $_; # sometimes $_ can be null (at EOF) and causes subsequent warnings
|
|
|
|
# EXPECTED RAW FORMAT
|
|
## lsof
|
|
#PID TOTAL U W R CWD RTD TXT MEM DEL TCP CMD
|
|
#20830 299 16 4 3 1 1 1 16 0 1 lcore-sla
|
|
|
|
if (/^PROCESS SUMMARY:/ || /^# lsof/) {
|
|
my $hcnt = 0;
|
|
# find headings line
|
|
HEADER_LOOP: while($_ = <FILE>) {
|
|
s/[\0\e\f\r\a]//g; chomp; # strip control characters if any
|
|
$hcnt++;
|
|
last HEADER_LOOP if (/\bPID\b/); # end titles-line
|
|
next READ_LOOP if (($hcnt == 1) && /^\s*$/); # end at blank-line (eg no 'ps' output)
|
|
}
|
|
next if ! $_; # sometimes $_ can be null (at EOF) and causes subsequent warnings
|
|
|
|
# Parse file line at a time
|
|
PROCESS_LOOP: while($_ = <FILE>) {
|
|
my $found_pid = 0;
|
|
s/[\0\e\f\r\a]//g; chomp; # strip control characters if any
|
|
last PROCESS_LOOP if (/^\s*$/); # end at blank-line
|
|
if (($PID ,$TOTAL, $FD, $U, $W, $R, $CWD, $RTD, $TXT, $MEM, $DEL, $TCP, $COMMAND) = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(.*)/) {
|
|
$found_pid = 1;
|
|
}
|
|
if ($found_pid == 1) {
|
|
# Match on multiple regular expressions or multiple pids
|
|
my $match = (defined $arg_all ? 1: 0);
|
|
foreach my $pid (@arg_pids) {if ($pid == $PID) {$match = 1; goto FOUND_CMD;} } # inclusions
|
|
foreach my $nre (@nre_commands) {if ($COMMAND =~ $nre) {$match = 0; goto FOUND_CMD;} } # exclusions
|
|
foreach my $re (@re_commands) {if ($COMMAND =~ $re) {$match = 1; goto FOUND_CMD;} } # inclusions
|
|
foreach my $re (@re_exact_commands) {if ($COMMAND =~ /^$re$/) {$match = 1; goto FOUND_CMD;} } # inclusions
|
|
FOUND_CMD: if ($match == 1) {
|
|
if ($arg_match) {
|
|
|
|
$matched{'MATCH'}{$PID} = 1;
|
|
$data{$iter}{'MATCH'}{$PID} = [($TOTAL, $FD, $U, $W, $R, $CWD, $RTD, $TXT, $MEM, $DEL, $TCP)];
|
|
} else {
|
|
$matched{$COMMAND}{$PID} = 1;
|
|
$data{$iter}{$COMMAND}{$PID} = [($TOTAL, $FD, $U, $W, $R, $CWD, $RTD, $TXT, $MEM, $DEL, $TCP)];
|
|
}
|
|
$files_current{$COMMAND} += $FD;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
next if ! $_; # sometimes $_ can be null (at EOF) and causes subsequent warnings
|
|
|
|
if (/^END OF SAMPLE:/ || /^----/ || / done$/) {
|
|
next if !(defined $uptime);
|
|
|
|
$FOUND = -2; # flag that we parsed complete file descriptor sample
|
|
|
|
# Handle incomplete sample in case there was no END OF SAMPLE
|
|
if (!(defined $days{$iter}) || !(defined $timestamps{$iter})) {
|
|
delete $days{$iter} if (defined $days{$iter});
|
|
delete $timestamps{$iter} if (defined $timestamps{$iter});
|
|
delete $data{$iter} if (defined $data{$iter});
|
|
delete $overall{$iter} if (defined $overall{$iter});
|
|
next;
|
|
}
|
|
|
|
$overall{$iter} = [($ftotal, $files, $sockets, $pipes, $uptime)];
|
|
|
|
# Store data for hirunner stats for current day only
|
|
my $day = $days{$iter};
|
|
if (!(defined $day)) {
|
|
printf "iter = %s\n", $iter;
|
|
printf "day=%s, last_time=%s\n", $day, $last_time;
|
|
$Data::Dumper::Indent = 1;
|
|
print Data::Dumper->Dump([\%overall], [qw(overall)]);
|
|
#printf "overall:%s = %s\n", $iter, "@{$overall{$iter}}";
|
|
}
|
|
if ($day >= ($last_time - 1.0)) {
|
|
foreach my $cmd (keys %files_current) {
|
|
my ($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY) = (0,0,0,0,0,0,0);
|
|
if (defined $file_stats{$cmd}) {
|
|
($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY) = @{$file_stats{$cmd}};
|
|
}
|
|
$count++;
|
|
$cur_files = $files_current{$cmd};
|
|
$max_files = ($cur_files > $max_files) ? $cur_files : $max_files;
|
|
$sum_X += $day;
|
|
$sum_Y += $cur_files;
|
|
$sum_XX += ($day * $day);
|
|
$sum_XY += ($cur_files * $day);
|
|
$file_stats{$cmd} = [($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY)];
|
|
}
|
|
}
|
|
$iter++;
|
|
$uptime = ();
|
|
next READ_LOOP;
|
|
}
|
|
}
|
|
close(FILE);
|
|
|
|
# Check that last sample was completed, else delete last hash key if defined
|
|
# -- no use in showing a message to user, will just confuse
|
|
if ($FOUND != -2 || !$overall{$iter}) {
|
|
delete $days{$iter} if (defined $days{$iter});
|
|
delete $timestamps{$iter} if (defined $timestamps{$iter});
|
|
delete $data{$iter} if (defined $data{$iter});
|
|
delete $overall{$iter} if (defined $overall{$iter});
|
|
}
|
|
}
|
|
|
|
# PRINT SUMMARY FOR THIS HOSTNAME
|
|
my ($idx_nlwp, $idx_rss, $idx_vsz) = (0, 1, 2);
|
|
my ($idx_TOTAL, $idx_FD, $idx_U, $idx_W, $idx_R, $idx_CWD, $idx_RTD, $idx_TXT, $idx_MEM, $idx_DEL, $idx_TCP) = (0,1,2,3,4,5,6,7,8,9,10);
|
|
my ($idx_ftotal, $idx_files, $idx_sockets, $idx_pipes, $idx_uptime) = (0,1,2,3,4);
|
|
my ($idx_wday, $idx_month, $idx_day, $idx_hh, $idx_mm, $idx_ss, $idx_yy) = (0, 1, 2, 3, 4, 5, 6);
|
|
my @iters = sort {$a <=> $b} keys %timestamps;
|
|
if (scalar(@iters) == 0) {
|
|
# do not continue processing for this host if no valid data
|
|
print "\n", "="x80, "\n", "NO VALID DATA FOR: $host\n\n";
|
|
next;
|
|
}
|
|
$last_time = $days{ $iters[-1] };
|
|
|
|
# Calculate statistics (only on first pass)
|
|
my $idx_mem = $idx_FD;
|
|
if ((defined $arg_report) && ($pass == 1)) {
|
|
$pass++;
|
|
my %num = ();
|
|
foreach my $iter (@iters) {
|
|
my $avail = ${ $overall{$iter} }[$idx_ftotal];
|
|
my $unlinked = ${ $overall{$iter} }[$idx_files];
|
|
my $fs_root = ${ $overall{$iter} }[$idx_sockets];
|
|
my $uptime = ${ $overall{$iter} }[$idx_uptime];
|
|
my $slab = ${ $overall{$iter} }[$idx_pipes];
|
|
my $day = $days{$iter};
|
|
my @keys = ();
|
|
# Cumulate stats for regression
|
|
if ($days{$iter} >= ($last_time - 1.0)) { push @keys, '1day';}
|
|
foreach my $key (@keys) {
|
|
$stats{$key}{'DAY'}{'sum_X'} += $day;
|
|
$stats{$key}{'DAY'}{'sum_XX'} += ($day * $day);
|
|
$stats{$key}{'DAY'}{'sum_XY'} += ($day * $day);
|
|
$num{$key}++;
|
|
}
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
# Sum up key values for commands that match
|
|
my $SUM_fd = 0.0;
|
|
foreach my $pid (keys %{ $matched{$cmd} }) {
|
|
if (defined $data{$iter}{$cmd}{$pid}) {
|
|
$SUM_fd += ${ $data{$iter}{$cmd}{$pid} }[$idx_FD];
|
|
}
|
|
}
|
|
# Cumulate stats for regression
|
|
foreach my $key (@keys) {
|
|
$stats{$key}{'fd_' .$cmd}{'sum_X'} += $SUM_fd;
|
|
$stats{$key}{'fd_' .$cmd}{'sum_XX'} += ($SUM_fd * $SUM_fd);
|
|
$stats{$key}{'fd_' .$cmd}{'sum_XY'} += ($SUM_fd * $day);
|
|
}
|
|
}
|
|
}
|
|
|
|
# Perform simple linear regression on all variables
|
|
foreach my $key (keys %stats) {
|
|
foreach my $var (keys %{ $stats{$key} }) {
|
|
($stats{$key}{$var}{'int'}, $stats{$key}{$var}{'slope'}) = &linear_fit(
|
|
$stats{$key}{'DAY'}{'sum_X'}, # 'DAY' is 'x' variable
|
|
$stats{$key}{$var}{'sum_X'},
|
|
$stats{$key}{$var}{'sum_XY'},
|
|
$stats{$key}{'DAY'}{'sum_XX'},
|
|
$num{$key}
|
|
);
|
|
}
|
|
}
|
|
%file_slope = ();
|
|
my $max_iter = $iters[-1];
|
|
|
|
# Compile regular expressions command string patterns
|
|
# -- store list of expressions to INCLUDE
|
|
@re_exact_commands = ();
|
|
foreach my $arg (@arg_exact) {
|
|
push @re_exact_commands, qr/\Q$arg\E/;
|
|
}
|
|
foreach my $cmd (keys %{file_stats}) {
|
|
my ($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY) = @{$file_stats{$cmd}};
|
|
my ($intercept, $slope) = &linear_fit($sum_X, $sum_Y, $sum_XY, $sum_XX, $count);
|
|
$file_slope{$cmd} = $slope; # slope (MiB/d)
|
|
# sneaky check for ignoring transient processes
|
|
# i.e. specific process exists less than half a day,
|
|
# or is in less than half the samples
|
|
if (($file_slope{$cmd} >= $arg_thresh) &&
|
|
((defined $arg_transient) ||
|
|
((($max_iter >= 142) && ($count > 70)) ||
|
|
(($max_iter < 142) && ($max_iter/2 < $count))
|
|
))) {
|
|
push @re_exact_commands, qr/\Q$cmd\E/;
|
|
}
|
|
}
|
|
goto REPEAT_CALCULATION;
|
|
}
|
|
|
|
print "\n", "="x80, "\n", "SUMMARY: host:$host ($first_date to $last_date)\n\n";
|
|
|
|
if (keys %matched) {
|
|
|
|
# PRINT HEADING OF MATCHED COMMAND PATTERNS AND PIDS
|
|
my %labels = ();
|
|
my $ele = 0;
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
my @pids = keys %{$matched{$cmd}};
|
|
# Create short command name
|
|
my $name = "";
|
|
$_ = $cmd;
|
|
if (/^\[(.*)\]/) {
|
|
$name = $1;
|
|
} else {
|
|
my @array = split(/\s+/, $cmd);
|
|
$name = shift @array; $name =~ s/^.*\///;
|
|
}
|
|
$labels{$cmd} = sprintf("%d:%s", $ele, $name);
|
|
printf "label: %s (%s)\n", $labels{$cmd}, $cmd;
|
|
printf " pids:(";
|
|
foreach my $pid (sort {$a <=> $b} keys %{ $matched{$cmd} }) {
|
|
printf "%d,", $pid;
|
|
}
|
|
printf ")\n";
|
|
$ele++;
|
|
}
|
|
|
|
# PRINT COLUMN HEADINGS FOR EACH PATTERN
|
|
printf "%10s %9s", "", "";
|
|
if (!(defined $arg_report)) {
|
|
my $width = 26;
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
my @pids = keys %{$matched{$cmd}};
|
|
printf " | %26s", substr $labels{$cmd}, 0, $width;
|
|
}
|
|
} else {
|
|
my $width = 15;
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
my @pids = keys %{$matched{$cmd}};
|
|
printf " %15s", substr $labels{$cmd}, 0, $width;
|
|
}
|
|
}
|
|
print "\n";
|
|
}
|
|
printf "%10s %9s", "DATE", "TIME";
|
|
if (!(defined $arg_report)) {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " | %8s %8s %8s", "U", "R", "W";
|
|
}
|
|
} else {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " %8s", "FD";
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
print "\n";
|
|
my %num = ();
|
|
my $uptime_last = 0.0;
|
|
my $num_reboots = 0;
|
|
foreach my $iter (@iters) {
|
|
my $ftotal = ${ $overall{$iter} }[$idx_ftotal];
|
|
my $f_files = ${ $overall{$iter} }[$idx_files];
|
|
my $sockets = ${ $overall{$iter} }[$idx_sockets];
|
|
my $uptime = ${ $overall{$iter} }[$idx_uptime];
|
|
my $pipes = ${ $overall{$iter} }[$idx_pipes];
|
|
my $day = $days{$iter};
|
|
my @keys = ('all');
|
|
if ($uptime < $uptime_last) {
|
|
$num_reboots++;
|
|
if (defined $arg_detail) {
|
|
printf "--reboot detected----%28s", '-'x28;
|
|
if (!(defined $arg_report)) {
|
|
foreach (keys %matched) {printf "%25s", '-'x25;}
|
|
} else {
|
|
foreach (keys %matched) {printf "%12s", '-'x12;}
|
|
}
|
|
print "\n";
|
|
}
|
|
}
|
|
if ((defined $arg_detail) || ($iter == $iters[-1])) {
|
|
printf "%04d-%02d-%02d %02d:%02d:%02d",
|
|
${ $timestamps{$iter} }[$idx_yy],
|
|
${ $timestamps{$iter} }[$idx_month],
|
|
${ $timestamps{$iter} }[$idx_day],
|
|
${ $timestamps{$iter} }[$idx_hh],
|
|
${ $timestamps{$iter} }[$idx_mm],
|
|
${ $timestamps{$iter} }[$idx_ss];
|
|
}
|
|
# Cumulate stats for regression
|
|
if ($days{$iter} >= ($last_time - 1.0)) { push @keys, '1day';}
|
|
foreach my $key (@keys) {
|
|
$stats{$key}{'DAY'}{'sum_X'} += $day;
|
|
$stats{$key}{'TOTAL'}{'sum_X'} += $ftotal;
|
|
$stats{$key}{'PIPES'}{'sum_X'} += $pipes;
|
|
$stats{$key}{'FILES'}{'sum_X'} += $f_files;
|
|
$stats{$key}{'SOCKETS'}{'sum_X'} += $sockets;
|
|
$stats{$key}{'DAY'}{'sum_XX'} += ($day * $day);
|
|
$stats{$key}{'TOTAL'}{'sum_XX'} += ($ftotal * $ftotal);
|
|
$stats{$key}{'PIPES'}{'sum_XX'} += ($pipes * $pipes);
|
|
$stats{$key}{'FILES'}{'sum_XX'} += ($f_files * $f_files);
|
|
$stats{$key}{'SOCKETS'}{'sum_XX'} += ($sockets * $sockets);
|
|
$stats{$key}{'DAY'}{'sum_XY'} += ($day * $day);
|
|
$stats{$key}{'TOTAL'}{'sum_XY'} += ($ftotal * $day);
|
|
$stats{$key}{'PIPES'}{'sum_XY'} += ($pipes * $day);
|
|
$stats{$key}{'FILES'}{'sum_XY'} += ($f_files * $day);
|
|
$stats{$key}{'SOCKETS'}{'sum_XY'} += ($sockets * $day);
|
|
$num{$key}++;
|
|
}
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
# Sum up key values for commands that match
|
|
my ($SUM_u, $SUM_r, $SUM_w, $SUM_FD) = (0.0, 0.0, 0.0, 0.0);
|
|
foreach my $pid (keys %{ $matched{$cmd} }) {
|
|
if (defined $data{$iter}{$cmd}{$pid}) {
|
|
$SUM_u += ${ $data{$iter}{$cmd}{$pid} }[$idx_U];
|
|
$SUM_r += ${ $data{$iter}{$cmd}{$pid} }[$idx_R];
|
|
$SUM_w += ${ $data{$iter}{$cmd}{$pid} }[$idx_W];
|
|
$SUM_FD += ${ $data{$iter}{$cmd}{$pid} }[$idx_FD];
|
|
}
|
|
}
|
|
if ((defined $arg_detail) || ($iter == $iters[-1])) {
|
|
if (!(defined $arg_report)) {
|
|
printf " | %8d %8d %8d", $SUM_u, $SUM_r, $SUM_w;
|
|
} else {
|
|
printf " %8d", $SUM_FD;
|
|
}
|
|
}
|
|
# Cumulate stats for regression
|
|
foreach my $key (@keys) {
|
|
$stats{$key}{'u_'.$cmd}{'sum_X'} += $SUM_u;
|
|
$stats{$key}{'r_' .$cmd}{'sum_X'} += $SUM_r;
|
|
$stats{$key}{'w_' .$cmd}{'sum_X'} += $SUM_w;
|
|
$stats{$key}{'u_'.$cmd}{'sum_XX'} += ($SUM_u * $SUM_u);
|
|
$stats{$key}{'r_' .$cmd}{'sum_XX'} += ($SUM_r * $SUM_r);
|
|
$stats{$key}{'w_' .$cmd}{'sum_XX'} += ($SUM_w * $SUM_w);
|
|
$stats{$key}{'u_'.$cmd}{'sum_XY'} += ($SUM_u * $day);
|
|
$stats{$key}{'r_' .$cmd}{'sum_XY'} += ($SUM_r * $day);
|
|
$stats{$key}{'w_' .$cmd}{'sum_XY'} += ($SUM_w * $day);
|
|
$stats{$key}{'fd_' .$cmd}{'sum_X'} += $SUM_FD;
|
|
$stats{$key}{'fd_' .$cmd}{'sum_XX'} += ($SUM_FD * $SUM_FD);
|
|
$stats{$key}{'fd_' .$cmd}{'sum_XY'} += ($SUM_FD * $day);
|
|
}
|
|
}
|
|
if ((defined $arg_detail) || ($iter == $iters[-1])) {
|
|
printf "\n";
|
|
}
|
|
# save uptime for comparison
|
|
$uptime_last = $uptime;
|
|
}
|
|
|
|
|
|
# Perform simple linear regression on all variables
|
|
foreach my $key (keys %stats) {
|
|
foreach my $var (keys %{ $stats{$key} }) {
|
|
($stats{$key}{$var}{'int'}, $stats{$key}{$var}{'slope'}) = &linear_fit(
|
|
$stats{$key}{'DAY'}{'sum_X'}, # 'DAY' is 'x' variable
|
|
$stats{$key}{$var}{'sum_X'},
|
|
$stats{$key}{$var}{'sum_XY'},
|
|
$stats{$key}{'DAY'}{'sum_XX'},
|
|
$num{$key}
|
|
);
|
|
}
|
|
}
|
|
|
|
|
|
|
|
%file_slope = ();
|
|
foreach my $cmd (keys %{file_stats}) {
|
|
|
|
my ($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY) = @{$file_stats{$cmd}};
|
|
my ($intercept, $slope) = &linear_fit($sum_X, $sum_Y, $sum_XY, $sum_XX, $count);
|
|
$file_slope{$cmd} = $slope;
|
|
}
|
|
|
|
# Print out linear trends
|
|
# [ OBSOLETE ] printf "%20s %8s %7s %6s %5s", '-'x20, '-'x8, '-'x7, '-'x6, '-'x5;
|
|
printf "%20s", '-'x20;
|
|
if (!(defined $arg_report)) {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " | %8s %8s %8s", '-'x8, '-'x8, '-'x8;
|
|
}
|
|
} else {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " %15s", '-'x15;
|
|
}
|
|
}
|
|
print "\n";
|
|
foreach my $key (sort {$b cmp $a} keys %stats) {
|
|
printf "%20s", ($key eq 'all') ? "LONG TREND: (FD/d)" : "1 DAY TREND: (FD/d)";
|
|
if (!(defined $arg_report)) {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " | %8.3f %8.3f %8.3f",
|
|
$stats{$key}{'u_'.$cmd}{'slope'},
|
|
$stats{$key}{'w_' .$cmd}{'slope'},
|
|
$stats{$key}{'r_' .$cmd}{'slope'};
|
|
}
|
|
} else {
|
|
foreach my $cmd (sort {$b cmp $a} keys %matched) {
|
|
printf " %8.3f", $stats{$key}{'fd_' .$cmd}{'slope'};
|
|
}
|
|
}
|
|
if (($key eq 'all') && ($num_reboots > 0)) {
|
|
printf " (%d reboots)", $num_reboots;
|
|
}
|
|
print "\n";
|
|
}
|
|
|
|
|
|
|
|
my $n = 0;
|
|
# Print out hirunner process growth
|
|
printf "\nPROCESSES WITH HIGHEST GROWTH (1 DAY TREND: > %.1f FD's/day):\n", $arg_thresh;
|
|
printf "%9s %9s %9s %s\n", 'CurFDs', 'HiFDs', 'Rate', 'COMMAND';
|
|
printf "%9s %9s %9s %s\n", '-'x8, '-'x8, '-'x8, '-'x9;
|
|
foreach my $cmd (sort {$file_slope{$b} <=> $file_slope{$a} } keys %file_slope) {
|
|
last if ($file_slope{$cmd} < $arg_thresh);
|
|
my $max_iter = $iters[-1];
|
|
my ($count, $cur_files, $max_files, $sum_X, $sum_Y, $sum_XX, $sum_XY) = @{$file_stats{$cmd}};
|
|
if ((defined $arg_transient) || ((($max_iter >= 142) && ($count > 70)) ||
|
|
(($max_iter < 142) && ($max_iter/2 < $count)))) { # print only processes seen most of the time
|
|
printf "%9.3f %9.3f %9.3f %s\n", $cur_files, $max_files, $file_slope{$cmd}, $cmd;
|
|
$n++;
|
|
}
|
|
}
|
|
print "none\n" if ($n == 0);
|
|
print "\n";
|
|
}
|
|
|
|
exit 0;
|
|
|
|
#######################################################################################################################
|
|
# Lightweight which(), derived from CPAN File::Which
|
|
sub which {
|
|
my ($exec) = @_;
|
|
return undef unless $exec;
|
|
my $all = wantarray;
|
|
my @results = ();
|
|
my @path = File::Spec->path;
|
|
foreach my $file ( map { File::Spec->catfile($_, $exec) } @path ) {
|
|
next if -d $file;
|
|
if (-x _) { return $file unless $all; push @results, $file; }
|
|
}
|
|
$all ? return @results : return undef;
|
|
}
|
|
|
|
# Process "parse_filestats" command line arguments and set defaults
|
|
sub get_parse_filestats_args {
|
|
# Returned parameters
|
|
local (*::arg_all, *::arg_match, *::arg_name,
|
|
*::arg_pids, *::arg_commands, *::arg_exact, *::arg_commands_excl,
|
|
*::arg_list, *::arg_detail, *::arg_thresh, *::arg_transient,
|
|
*::arg_path, *::arg_dur, *::arg_report, *::arg_files) = @_;
|
|
|
|
# Local variables
|
|
my ($fail, $arg_help) = ();
|
|
my @tmp = ();
|
|
|
|
# Use the Argument processing module
|
|
use Getopt::Long;
|
|
|
|
# Print usage if no arguments
|
|
if (!@ARGV) {
|
|
&Usage();
|
|
exit 0;
|
|
}
|
|
|
|
# Process input arguments
|
|
$fail = 0;
|
|
GetOptions(
|
|
"all", \$::arg_all, # CURRENTLY UNUSED
|
|
"match", \$::arg_match,
|
|
"name=s", \$::arg_name,
|
|
"pid=i", \@::arg_pids,
|
|
"cmd=s", \@::arg_commands,
|
|
"exact=s", \@::arg_exact,
|
|
"excl=s", \@::arg_commands_excl,
|
|
"list", \$::arg_list,
|
|
"detail", \$::arg_detail,
|
|
"thresh=f", \$::arg_thresh,
|
|
"transient", \$::arg_transient,
|
|
"dir=s", \$::arg_path,
|
|
"dur=f", \$::arg_dur,
|
|
"report", \$::arg_report,
|
|
"help|?", \$arg_help
|
|
) || GetOptionsMessage();
|
|
|
|
# Print help documentation if user has selected -help
|
|
&ListHelp() if (defined $arg_help);
|
|
|
|
# Listify @::arg_pids
|
|
@tmp = ();
|
|
if (@::arg_pids) {
|
|
@tmp = @::arg_pids; @::arg_pids = ();
|
|
foreach my $pid (@tmp) { push @::arg_pids, (split /,/, $pid); }
|
|
}
|
|
# Listify @::arg_commands
|
|
@tmp = ();
|
|
if (@::arg_commands) {
|
|
@tmp = @::arg_commands; @::arg_commands = ();
|
|
foreach my $cmd (@tmp) { push @::arg_commands, (split /,/, $cmd); }
|
|
}
|
|
# Listify @::arg_exact
|
|
@tmp = ();
|
|
if (@::arg_exact) {
|
|
@tmp = @::arg_exact; @::arg_exact = ();
|
|
foreach my $cmd (@tmp) { push @::arg_exact, (split /,/, $cmd); }
|
|
}
|
|
# Listify @::arg_commands_excl
|
|
@tmp = ();
|
|
if (@::arg_commands_excl) {
|
|
@tmp = @::arg_commands_excl; @::arg_commands_excl = ();
|
|
foreach my $cmd (@tmp) { push @::arg_commands_excl, (split /,/, $cmd); }
|
|
}
|
|
|
|
# Give warning messages and usage when parameters are specified incorrectly.
|
|
my $cnt = 0;
|
|
$cnt++ if (defined $::arg_name);
|
|
$cnt++ if (defined $::arg_list);
|
|
##$cnt++ if (defined $::arg_all);
|
|
# [ JGAULD - maybe add $::arg_match]
|
|
if ($cnt > 1) {
|
|
warn "$SCRIPT: Input error: cannot specify more than one of {--list} or {--name <pattern>} options.\n";
|
|
$fail = 1;
|
|
}
|
|
if ($fail == 1) {
|
|
# touch variables here to make silly warning go away
|
|
$::arg_all = ""; $::arg_match = "";
|
|
$::arg_name = ""; $::arg_list = ""; $::arg_detail = "";
|
|
$::arg_thresh = 0; $::arg_transient = "";
|
|
&Usage();
|
|
exit 1;
|
|
}
|
|
|
|
# Assume remaining options are filenames
|
|
@::arg_files = @ARGV;
|
|
|
|
# Set defaults for options requiring values
|
|
if (!(defined $::arg_thresh)) {
|
|
$::arg_thresh = 1.0; # Default to 10 FD/d
|
|
}
|
|
if (!(defined $::arg_dur)) {
|
|
$::arg_dur = 7.0; # Default to 7.0 days worth of data
|
|
} else {
|
|
$::arg_dur = 1.0 if ($::arg_dur < 1.0); # minimum 1 day worth of data
|
|
}
|
|
$::arg_path ||= $DEFAULT_PATH;
|
|
$::arg_detail = 1 if (defined $::arg_report); # print details if 'report' option chosen
|
|
}
|
|
|
|
sub GetOptionsMessage {
|
|
# Print out a warning message and then print program usage.
|
|
warn "$SCRIPT: Error processing input arguments.\n";
|
|
&Usage();
|
|
exit 1;
|
|
}
|
|
|
|
sub Usage {
|
|
# Print out program usage.
|
|
printf "Usage: $SCRIPT file1 file2 file3.gz ...\n";
|
|
printf "\t[--list | --all | --name <glob_pattern>]\n";
|
|
printf "\t[--pid <pid> ] ...\n";
|
|
printf "\t[--cmd <pattern>] ...\n";
|
|
printf "\t[--exact <pattern>] ...\n";
|
|
printf "\t[--excl <pattern>] ...\n";
|
|
printf "\t[--detail]\n";
|
|
printf "\t[--thresh <MiB/d>]\n";
|
|
printf "\t[--transient]\n";
|
|
printf "\t[--dir <path>]\n";
|
|
printf "\t[--dur <days>]\n";
|
|
printf "\t[--report]\n";
|
|
printf "\t[--help | -?]\n";
|
|
}
|
|
|
|
sub ListHelp {
|
|
# Print out tool help
|
|
printf "$SCRIPT -- parses 'filestats' data and prints processes with hirunner file growth\n";
|
|
&Usage();
|
|
printf "\nOptional input arguments:\n";
|
|
printf " --list : list all available 'file' data\n";
|
|
printf " --all : summarize all blades\n";
|
|
printf " --name <glob_pattern> : match files with pattern (file globbing allowed if in \"quotes\")\n";
|
|
printf " --pid <pid> : match 'pid' (can specify multiple pids)\n";
|
|
printf " --cmd <command|pattern> : match command name 'string' pattern (can specify multiple patterns)\n";
|
|
printf " --exact <command|pattern> : exact match command name 'string' pattern (can specify multiple patterns)\n";
|
|
printf " --excl <command|pattern> : match command name 'string' pattern to exclude (can specify multiple patterns)\n";
|
|
printf " --detail : time-series output\n";
|
|
printf " --thresh : set threshold for hirunner growth processes : default > 0.005 MiB/d\n";
|
|
printf " --transient : include transient processes (i.e., do not filter out short-lived processes)\n";
|
|
printf " --dir <path> : path to filestat files : default: $DEFAULT_PATH\n";
|
|
printf " --dur <days> : number of days of data to process : default: 7.0\n";
|
|
printf " --report : summarize details for hi-runner file growth\n";
|
|
printf " --help : this help information\n";
|
|
printf "\nfile data storage: %s/%s\n", $DEFAULT_PATH, "<hostname>_<yyyy>-<mm>-<dd>_<hh>00_filestats{.gz}";
|
|
printf "(most data files are gzip compressed)\n";
|
|
printf "\nExamples:\n";
|
|
printf " $SCRIPT --all (i.e., summarize all hosts\n";
|
|
printf " $SCRIPT --name 2014-02-22 (i.e., match files containing 2014-02-22)\n";
|
|
printf " $SCRIPT --name compute-0 --cmd python (i.e., compute-0, specify process(es))\n";
|
|
printf " $SCRIPT --name compute-0 --pid 1 --pid 2 --detail (i.e., slot 1, specify PIDs 1 and 2 time-series)\n";
|
|
printf " $SCRIPT --name controller-0 --cmd python --excl blah --detail (i.e., time-series for 'python', but not 'blah')\n";
|
|
|
|
printf "\nReported Memory Headings:\n";
|
|
printf " FD (#) - Total File descriptors for process\n";
|
|
printf " U (#) - Read Write files open for process\n";
|
|
printf " W (#) - Write Only files open for process\n";
|
|
printf " R (#) - Read Only files open for process\n";
|
|
printf " TREND - reported in change of Files per day";
|
|
printf "\n";
|
|
exit 0;
|
|
}
|
|
|
|
# Calculate linear regression coefficients for linear equation, y = a + b*x
|
|
sub linear_fit {
|
|
my ($sum_X, $sum_Y, $sum_XY, $sum_XX, $n) = @_;
|
|
my ($a, $b, $s1_sq) = ();
|
|
|
|
# Prevent doing regression with less than 2 points
|
|
return (0,0) if ($n < 2);
|
|
$s1_sq = ($sum_XX - $sum_X / $n * $sum_X) / ($n - 1);
|
|
|
|
# Prevent divide by zero
|
|
return (0,0) if ($s1_sq <= 0.0);
|
|
|
|
$b = ($n * $sum_XY - $sum_X * $sum_Y) / $n / ($n - 1) / $s1_sq;
|
|
$a = ($sum_Y - $b * $sum_X)/$n;
|
|
return ($a, $b);
|
|
}
|
|
|
|
1;
|