summarize_cprnc_diffs

#!/usr/bin/env perl
#
# Summarize cprnc output from all tests in a CESM test suite.
# See usage message for details.
#
# Bill Sacks
# 4-10-13

use strict;
use Getopt::Long;
use File::Basename;
use List::Util qw(max sum);

#----------------------------------------------------------------------
# Get arguments and check them
#----------------------------------------------------------------------

my %opts;
# set defaults
$opts{'output_suffix'} = '';
# set up options
GetOptions(
           "basedir=s"       => \$opts{'basedir'},
           "testid=s"        => \$opts{'testid'},
           "output_suffix=s" => \$opts{'output_suffix'},
           "narrow"          => \$opts{'narrow'},
           "h|help"          => \$opts{'help'},
)  or usage();

usage() if $opts{'help'};

if (@ARGV) {
   print "ERROR: unrecognized arguments: @ARGV\n";
   usage();
}

if (!$opts{'basedir'}) {
   print "ERROR: -basedir must be provided\n";
   usage();
}
if (!$opts{'testid'}) {
   print "ERROR: -testid must be provided\n";
   usage();
}

#----------------------------------------------------------------------
# Main script
#----------------------------------------------------------------------

# Create hash containing summary of cprnc differences. This is a reference to a hash, with:
# Keys: "Directory Filename Variable"
# Values: Reference to a hash containing:
#    Dir => directory (gives test name)
#    Filename  => cprnc filename
#    Variable  => variable
#    RMS       => rms value            [may or may not be present]
#    RMS_NORM  => normalized rms value [may or may not be present]
#    FILLDIFF  => ' '                  [may or may not be present]
#    DIMSIZEDIFF => ' '                [may or may not be present]
my ($summary_hash) =
   process_cprnc_output($opts{'basedir'}, $opts{'testid'}, $opts{'output_suffix'});

my $outbase="cprnc.summary.$opts{'testid'}";
if ($opts{'output_suffix'}) {
   $outbase = "$outbase.$opts{'output_suffix'}";
}

# set widths of output strings
my $widths_hash;
if ($opts{'narrow'}) {
   $widths_hash = { Dir => 40, Filename => 40, Variable => 40 };
}
else {
   $widths_hash = max_widths($summary_hash);
}


print_results_by_test("${outbase}.by_test", $summary_hash, $widths_hash);
print_results_by_varname("${outbase}.by_varname", $summary_hash, $widths_hash);
print_results_by_rms("${outbase}.by_rms", $summary_hash, $widths_hash);



#----------------------------------------------------------------------
# Subroutines
#----------------------------------------------------------------------

sub usage {
   die <<EOF;
SYNOPSIS
    summarize_cprnc_diffs -basedir <basedir> -testid <testid> [OPTIONS]

    <basedir> is the base directory in which test directories can be found

    <testid> is the testid of the tests to summarize
             (can contain shell wildcards)

    This script can be used to post-process and summarize baseline comparison
    output from one or more CESM test suites.

    The script finds all directories in basedir whose name ends with the given
    testid; these are the test directories of interest. It then examines the
    'run' subdirectory of each test directory of interest, looking for files of
    the form *.nc.cprnc.out. Or, if the -output_suffix argument is given, then
    it looks for files of the form *.nc.cprnc.out.SUFFIX. (With this naming
    convention [i.e., looking for files of the form *.nc.cprnc.out], note that
    it only looks at output for baseline comparisons - NOT output from the test
    itself, such as cprnc output files from the exact restart test.) (Actually,
    we also allow for files of the form *.nc_[0-9][0-9][0-9][0-9].cprnc.out,
    such as *.nc_0001.cprnc.out and *.nc_0002.cprnc.out, to pick up
    multi-instance files.)

    Summaries of cprnc differences (RMS and normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs)
    are placed in three output files beginning with the name 'cprnc.summary', in
    the current directory.  These files contain the same information, but one is
    sorted by test name, one is sorted by variable name, and is one sorted from
    largest to smallest normalized RMS differences.


OPTIONS
    -output_suffix <suffix>  If provided, look for files of the form *.nc.cprnc.out.SUFFIX
                             rather than just *.nc.cprnc.out

    -narrow                  Use generally-narrower output field widths to aid readability,
                             at the expense of truncated strings

    -help [or -h]            Display this help

EOF
}


# process_cprnc_output
# Read through all cprnc files, and build hashes of instances of RMS, normalized RMS, FILLDIFF and DIMSIZEDIFF
# Inputs:
#  - basedir
#  - testid
#  - output_suffix
# Output: hash reference
# Dies with an error if no cprnc output files are found
sub process_cprnc_output {
   my ($basedir, $testid, $output_suffix) = @_;

   my %diffs;
   my $num_files = 0;

   my @test_dirs = glob "${basedir}/*${testid}";

   foreach my $test_dir (@test_dirs) {
      my $test_dir_base = basename($test_dir);

      my @cprnc_files;
      if ($output_suffix) {
         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix} ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out.${output_suffix}";
      }
      else {
         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out";
      }

      foreach my $cprnc_file (@cprnc_files) {
         my $cprnc_file_base = basename($cprnc_file);
         $num_files++;

         open IN, "<", $cprnc_file or die "ERROR opening ${cprnc_file}";

         while (my $line = <IN>) {
            chomp $line;

            process_line($line, $test_dir_base, $cprnc_file_base, \%diffs);
         }  # while <IN>

         close IN;
      }  # foreach cprnc_file

   }  # foreach test_dir

   if ($num_files == 0) {
      die "ERROR: no cprnc.out files found\n";
   }

   return \%diffs;
}


# process_line: Process one line from one file
# Inputs:
#  - line
#  - test_dir
#  - cprnc_file
#  - diffs hash reference (MODIFIED)
sub process_line {
   my ($line, $test_dir, $cprnc_file, $diffs) = @_;

   my $diff_type;
   my $varname;
   my $rms;
   my $ignore;
   my $rms_normalized;

   if ($line =~ /^ *RMS /) {
      ($diff_type, $varname, $rms, $ignore, $rms_normalized) = split " ", $line;
   } elsif ($line =~ /^ *FILLDIFF /) {
      ($diff_type, $varname) = split " ", $line;
      $rms = "";
      $rms_normalized = "";
   } elsif ($line =~ /^ *DIMSIZEDIFF /) {
      ($diff_type, $varname) = split " ", $line;
      $rms = "";
      $rms_normalized = "";
   } else {
      $diff_type = "";
   }

   if ($diff_type eq 'RMS' || $diff_type eq 'FILLDIFF' || $diff_type eq 'DIMSIZEDIFF') {
      # We have found a cprnc difference

      my $key = "$test_dir $cprnc_file $varname";

      # For RMS errors, keep the highest error found
      if ($diff_type eq "RMS") {
         if (exists $diffs->{$key} && exists $diffs->{$key}{'RMS_NORM'}) {
            if ($diffs->{$key}{'RMS_NORM'} > $rms_normalized) {
               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{'RMS_NORM'}\n";
               return;
            }
            else {
               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{'RMS_NORM'}\n";
            }
         }
      }

      # If the diffs hash doesn't already contain information about this
      # directory/filename/variable combo, then we need to create a hash
      # reference with the appropriate basic metadata.
      if (!exists $diffs->{$key}) {
         $diffs->{$key} = {
                         Dir      => $test_dir,
                         Filename => $cprnc_file,
                         Variable => $varname,
                        };
      }

      # Whether or not the hash already contained the given key, we need to add
      # the value of interest -- either the RMS and normalized RMS errors, or
      # the fact that there is a FILLDIFF or DIMSIZEDIFF.
      if ($diff_type eq "RMS") {
         $diffs->{$key}{'RMS'} = $rms;
         $diffs->{$key}{'RMS_NORM'} = $rms_normalized;
      } else {
         # No meaningful value here - just record the fact that we saw a
         # FILLDIFF or DIMSIZEDIFF
         $diffs->{$key}{$diff_type} = "";
      }
   } elsif ($diff_type ne '') {
      die "Unexpected diff_type: $diff_type";
   }
}


# max_widths
# Inputs:
#  - summary_hash (hash reference)
# Output: reference to a hash containing the maximum width of each of
# the following in the summary hash:
#  - Dir
#  - Filename
#  - Variable
sub max_widths {
   my $summary_hash = shift;

   my %maxes;

   foreach my $var ('Dir','Filename','Variable') {
      $maxes{$var} = max (map { length($summary_hash->{$_}{$var}) } keys %$summary_hash);
   }

   return \%maxes;
}


# print_results_by_test: Print sorted hash entries to a file, sorted by test name
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_test {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort{ $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                        or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
      keys %$summary_hash;

   my $last_dir;
   my $last_filename;

   my $separator_width = sum(values %$widths) + 57;

   for my $key (@sorted_keys) {

      # Print a separator line between different files
      if ($summary_hash->{$key}{'Dir'}      ne $last_dir ||
          $summary_hash->{$key}{'Filename'} ne $last_filename) {
         if ($last_dir && $last_filename) {
            print OUT "=" x $separator_width . "\n";
         }
         $last_dir      = $summary_hash->{$key}{'Dir'};
         $last_filename = $summary_hash->{$key}{'Filename'};
      }

      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   }

   close OUT;
}


# print_results_by_varname: Print sorted hash entries to a file, sorted by variable name
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_varname {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort{ $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'}
                        or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'} }
      keys %$summary_hash;

   my $last_variable;

   my $separator_width = sum(values %$widths) + 57;

   for my $key (@sorted_keys) {

      # Print a separator line between different variables
      if ($summary_hash->{$key}{'Variable'} ne $last_variable) {
         if ($last_variable) {
            print OUT "=" x $separator_width . "\n";
         }
         $last_variable = $summary_hash->{$key}{'Variable'};
      }

      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   }

   close OUT;
}



# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS_NORM
# Inputs:
#  - outfile: name of output file
#  - summary_hash: hash reference containing results to print
#  - widths: hash reference giving widths of output strings
sub print_results_by_rms {
   my ($outfile, $summary_hash, $widths) = @_;

   open OUT, ">", "$outfile" or die "ERROR opening $outfile";

   my @sorted_keys = sort {$summary_hash->{$b}{'RMS_NORM'} <=> $summary_hash->{$a}{'RMS_NORM'}
                        or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                        or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                        or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
      keys %$summary_hash;

   for my $key (@sorted_keys) {
      my $line = format_line($summary_hash->{$key}, $widths);

      print OUT "$line\n";
   }

   close OUT;
}


# Inputs:
# - reference to a hash containing:
#   - Dir
#   - Filename
#   - Variable
#   - RMS (optional)
#   - RMS_NORM (optional)
#   - FILLDIFF (optional)
#   - DIMSIZEDIFF (optional)
# - widths: hash reference giving widths of output strings
# Return a formatted line for printing
sub format_line {
   my ($hash_ref, $widths) = @_;

   my $dir = $hash_ref->{'Dir'};
   my $filename = $hash_ref->{'Filename'};
   my $variable = $hash_ref->{'Variable'};
   my $rms = "";
   my $rms_normalized = "";
   my $filldiff = "";
   my $dimsizediff = "";
   if (exists $hash_ref->{'RMS'}) {
      $rms = sprintf(" : RMS %-16g", $hash_ref->{'RMS'});
   }
   if (exists $hash_ref->{'RMS_NORM'}) {
      $rms_normalized = sprintf(" : RMS_NORM %-16g", $hash_ref->{'RMS_NORM'});
   }
   if (exists $hash_ref->{'FILLDIFF'}) {
      $filldiff = " : FILLDIFF";
   }
   if (exists $hash_ref->{'DIMSIZEDIFF'}) {
      $dimsizediff = " : DIMSIZEDIFF";
   }

   # for width=40, the format string will contain '%-40.40s'
   my $format = '%-' . $widths->{'Dir'} . '.' . $widths->{'Dir'} . 's : ' .
                '%-' . $widths->{'Filename'} . '.' . $widths->{'Filename'} . 's : ' .
                '%-' . $widths->{'Variable'} . '.' . $widths->{'Variable'} . 's' .
                '%s%s%s%s';

   sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms, $rms_normalized);
}

#=======================================================================
# Notes about testing: unit tests
#=======================================================================

#-----------------------------------------------------------------------
# Testing process_line
#-----------------------------------------------------------------------

# use Data::Dumper;

# my %diffs;

# # shouldn't do anything
# process_line("hello", "test_dir1", "file_a", \%diffs);

# # test basic filldiff
# process_line("FILLDIFF var1", "test_dir1", "file_b", \%diffs);

# # add an RMS to existing filldiff
# process_line("RMS var1 4200 NORMALIZED 42", "test_dir1", "file_b", \%diffs);

# # test basic rms error
# process_line("RMS var17 0.314 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);

# # add a filldiff to existing rms error
# process_line("FILLDIFF var17", "test_dir1", "file_b", \%diffs);

# # add a filldiff without RMS
# process_line("FILLDIFF var42", "test_dir2", "file_c", \%diffs);

# # add a dimsizediff
# process_line("DIMSIZEDIFF var43", "test_dir2", "file_c", \%diffs);

# # add an RMS error without filldiff
# process_line("RMS var100 99 NORMALIZED 100", "test_dir2", "file_d", \%diffs);

# # test a warning: should issue a warning and replace the above setting
# process_line("RMS var100 9 NORMALIZED 200", "test_dir2", "file_d", \%diffs);

# # test a warning: should issue a warning but NOT replace the above setting
# # (normalized RMS is smaller even though standard RMS is bigger: the normalized
# # one should be considered in deciding whether to replace the previous setting)
# process_line("RMS var100 999 NORMALIZED 50", "test_dir2", "file_d", \%diffs);

# print Dumper(\%diffs);


# THE ABOVE SHOULD PRINT SOMETHING LIKE THIS (though the output from Dumper will
# likely appear in a different order):

# WARNING: Replacing RMS with higher value: test_dir2 file_d var100 : 200 > 100
# WARNING: Ignoring lower RMS value:        test_dir2 file_d var100 : 50 < 200
# $VAR1 = {
#           'test_dir1 file_b var17' => {
#                                         'RMS' => '0.314',
#                                         'Variable' => 'var17',
#                                         'Filename' => 'file_b',
#                                         'FILLDIFF' => '',
#                                         'Dir' => 'test_dir1',
#                                         'RMS_NORM' => '3.14'
#                                       },
#           'test_dir2 file_d var100' => {
#                                          'Dir' => 'test_dir2',
#                                          'RMS_NORM' => 200,
#                                          'Filename' => 'file_d',
#                                          'Variable' => 'var100',
#                                          'RMS' => '9'
#                                        },
#           'test_dir1 file_b var1' => {
#                                        'Filename' => 'file_b',
#                                        'RMS_NORM' => '42',
#                                        'FILLDIFF' => '',
#                                        'Dir' => 'test_dir1',
#                                        'RMS' => '4200',
#                                        'Variable' => 'var1'
#                                      },
#           'test_dir2 file_c var43' => {
#                                         'Variable' => 'var43',
#                                         'DIMSIZEDIFF' => '',
#                                         'Dir' => 'test_dir2',
#                                         'Filename' => 'file_c'
#                                       },
#           'test_dir2 file_c var42' => {
#                                         'Filename' => 'file_c',
#                                         'Dir' => 'test_dir2',
#                                         'FILLDIFF' => '',
#                                         'Variable' => 'var42'
#                                       }
#         };


#-----------------------------------------------------------------------
# Testing the print routines
#-----------------------------------------------------------------------

# Add the following to the above test code:

# my $widths_hash = { Dir => 40, Filename => 40, Variable => 40 };
# print_results_by_test("testout.by_test", \%diffs, $widths_hash);
# print_results_by_rms("testout.by_rms", \%diffs, $widths_hash);

# This should give:

# $ cat testout.by_rms
# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200
# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
# test_dir2                                : file_c                                   : var42                                    : FILLDIFF
# test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
# $ cat testout.by_test
# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
# =================================================================================================================================================================================
# test_dir2                                : file_c                                   : var42                                    : FILLDIFF
# test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
# =================================================================================================================================================================================
# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200



#=======================================================================
# Notes about testing: integration tests
#=======================================================================

# Test the following

# Note: can do these tests by running the cprnc tests and organizing
# outputs into particular directories.
#
# For each of these tests, sort the different output files and compare
# the sorted files to make sure the same info is in all output files;
# then look at one of the output files.
#
# - no RMS or FILLDIFFs at all (testid that just contains output from
#   comparing control & copy)
#
# - some RMS and some FILLDIFFs, split across 2 directories, each with
#   2 cprnc files (this can be done by comparing the control file with
#   diffs_in_fill.nc, diffs_in_vals.nc, diffs_in_vals_and_diffs_in_fill.nc
#   and diffs_in_vals_and_fill.nc)
#
# - multiple RMS errors to test RMS sorting, split across 2 directories
#   (this can be done by comparing the control file with four of the
#   vals_differ_by_* files)