Unidata · lesserwhirls · Mar 24, 2017 · Mar 16, 2017
diff --git a/src/main/java/edu/ucar/unidata/rosetta/controller/TemplateController.java b/src/main/java/edu/ucar/unidata/rosetta/controller/TemplateController.java
@@ -321,6 +321,7 @@ public String parseFile(AsciiFile file, BindingResult result) {
                     HashMap<String, String> coords = new HashMap<String, String>();
 
                     // fix where session storage where necessairy
+                    // FIXME: hardcoded stuff that should not be in here?
                     if (file.getCfType().equals("trajectory")) {
                         // only the time variable should be a coordinate variable, so
                         // clean up session storage to reflect that.
@@ -345,6 +346,7 @@ public String parseFile(AsciiFile file, BindingResult result) {
                     }
                     List<List<String>> parseFileData = fileParserManager
                             .getParsedFileData();
+                    List<String> header = fileParserManager.getHeader();
 
                     // Create the NCML file using the file data
                     String downloadDir = FilenameUtils.concat(getDownloadDir(),
@@ -381,14 +383,17 @@ public String parseFile(AsciiFile file, BindingResult result) {
                         for (NetcdfFileManager potentialDsgWriter : netcdfFileManager.asciiToDsg()) {
                             if (potentialDsgWriter.isMine(file.getCfType())) {
                                 netcdfFile = potentialDsgWriter.createNetcdfFile(file,
-                                        parseFileData, downloadDir);
+                                        parseFileData, header, downloadDir);
                                 break;
                             }
                         }
                     } catch (IOException e) {
                         System.err.println("Caught IOException: "
                                 + e.getMessage());
                         return netcdfFile;
+                    } catch (IllegalArgumentException e) {
+                        //FIXME: possible security issue with revealing too much?
+                        return e.getMessage();
                     }
 
                     String fileOut = FilenameUtils.concat(downloadDir,

diff --git a/src/main/java/edu/ucar/unidata/rosetta/domain/AsciiFile.java b/src/main/java/edu/ucar/unidata/rosetta/domain/AsciiFile.java
@@ -40,6 +40,8 @@ public class AsciiFile {
     private Map<String, String> variableNameMap = new HashMap<String, String>();
     private String variableMetadata = null;
     private Map<String, HashMap> variableMetadataMap = new HashMap<String, HashMap>();
+    private String parseHeaderForMetadata = null;
+    private List<String> parseHeaderForMetadataList = new ArrayList<String>();
     private String jsonStrSessionStorage = null;
     private Map otherInfo = null;
 
@@ -238,6 +240,28 @@ public void setHeaderLineList(String headerLineNumbers) {
         this.headerLineList = Arrays.asList(headerLineNumbers.split(","));
     }
 
+    public void setParseHeaderForMetadataList(String parseHeaderForMetadata){
+        String[] headers = parseHeaderForMetadata.split(",");
+        for (String s: headers){
+            String[] header = s.split(":");
+            if (header.length == 2 && header[1].equals("true"))
+                parseHeaderForMetadataList.add(header[0]);
+        }
+    }
+
+    public void setParseHeaderForMetadata(String parseHeaderForMetadata){
+        this.parseHeaderForMetadata = parseHeaderForMetadata;
+        setParseHeaderForMetadataList(parseHeaderForMetadata);
+    }
+
+    public String getParseHeaderForMetadata(){
+        return parseHeaderForMetadata;
+    }
+
+    public List<String> getParseHeaderForMetadataList(){
+        return parseHeaderForMetadataList;
+    }
+
     /**
      * Returns the platform metadata in String format.
      *
@@ -317,8 +341,6 @@ public Map<String, String> getGeneralMetadataMap() {
     public void setGeneralMetadataMap(String generalMetadata) {
         String regexComma = "(?<!\\\\)" + Pattern.quote(",");
         String regexColon = "(?<!\\\\)" + Pattern.quote(":");
-        System.out.println("setGeneralMetadataMap:");
-        System.out.println(generalMetadata);
         List<String> pairs = Arrays.asList(generalMetadata.split(regexComma));
         Iterator<String> pairsIterator = pairs.iterator();
         while (pairsIterator.hasNext()) {

diff --git a/src/main/java/edu/ucar/unidata/rosetta/dsg/NetcdfFileManager.java b/src/main/java/edu/ucar/unidata/rosetta/dsg/NetcdfFileManager.java
@@ -15,6 +15,8 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import edu.ucar.unidata.rosetta.domain.AsciiFile;
 import edu.ucar.unidata.rosetta.dsg.util.DateTimeBluePrint;
@@ -44,6 +46,8 @@ public abstract class NetcdfFileManager {
     private Map<String, String> platformMetadataMap;
     private Map<String, String> generalMetadataMap;
     private Map<String, String> otherInfo;
+    private List<String> header;
+
     private HashMap<String, Integer> nameCounts;
     private String coordAttr;
     private List<String> usedVarNames = new ArrayList<String>();
@@ -55,6 +59,7 @@ public abstract class NetcdfFileManager {
     private ArrayList<String> buildTimeTriggers;
     private String relTimeVarName;
     private List<String> nonCoordVarList;
+    private List<String> parseHeaderForMetadataList;
 
     public Map<String, String> getOtherInfo() {
         return this.otherInfo;
@@ -63,6 +68,13 @@ public Map<String, String> getOtherInfo() {
     public void setOtherInfo(Map<String, String> otherInfo) {
         this.otherInfo = otherInfo;
     }
+    public List<String> getHeader() {
+        return header;
+    }
+
+    public void setHeader(List<String> header) {
+        this.header = header;
+    }
 
     public String getMyDsgType() {
         return myDsgType;
@@ -176,6 +188,14 @@ public void setVariableMetadataMap(Map<String, HashMap> variableMetadataMap) {
         this.variableMetadataMap = variableMetadataMap;
     }
 
+    private void setParseHeaderForMetadataList(List<String> parseHeaderForMetadataList) {
+        this.parseHeaderForMetadataList = parseHeaderForMetadataList;
+    }
+
+    private List<String> getParseHeaderForMetadataList() {
+        return this.parseHeaderForMetadataList;
+    }
+
     public Map<String, String> getVariableNameMap() {
         return variableNameMap;
     }
@@ -189,14 +209,37 @@ public Map<String, String> getPlatformMetadataMap() {
     }
 
     public void setPlatformMetadataMap(Map<String, String> platformMetadataMap) {
+        processPatternsOnMetadataMap(platformMetadataMap);
         this.platformMetadataMap = platformMetadataMap;
     }
 
+    private void processPatternsOnMetadataMap(Map<String, String> metadataMap) {
+        for (String key : this.getParseHeaderForMetadataList()) {
+            String pattern = metadataMap.get(key);
+            if (pattern != null){
+                boolean found = false;
+                for (String line : header){
+                    Pattern p = Pattern.compile(pattern);
+                    Matcher m = p.matcher(line);
+                    if (m.find() && m.groupCount() > 0){
+                        metadataMap.put(key, m.group(1));
+                        found = true;
+                        break;
+                    }
+                }
+                if (!found){
+                    throw new IllegalArgumentException("The pattern for '"+key+"':'"+pattern+"' did not match any line");
+                }
+            }
+        }
+    }
+
     public Map<String, String> getGeneralMetadataMap() {
         return generalMetadataMap;
     }
 
     public void setGeneralMetadataMap(Map<String, String> generalMetadataMap) {
+        processPatternsOnMetadataMap(generalMetadataMap);
         this.generalMetadataMap = generalMetadataMap;
     }
 
@@ -208,11 +251,13 @@ public boolean isMine(String reqType) {
         return mine;
     }
 
-    protected void init(AsciiFile file) {
+    protected void init(AsciiFile file, List<String> header) {
 
         setUsedVarNames(new ArrayList<String>());
         setAllVarNames(new ArrayList<String>());
 
+        setHeader(header);
+        setParseHeaderForMetadataList(file.getParseHeaderForMetadataList());
         setVariableNameMap(file.getVariableNameMap());
         setVariableMetadataMap(file.getVariableMetadataMap());
         setPlatformMetadataMap(file.getPlatformMetadataMap());
@@ -388,6 +433,7 @@ protected NetcdfFileWriter createNcfVariable(NetcdfFileWriter ncFileWriter, Stri
 
         if (getCoordVars().containsKey(coordVarType)) {
             if (getCoordVars().get(coordVarType).contains(sessionStorageKey)) {
+                // FIXME: efficient line
                 name = name;
                 shape = coordVarType;
             }
@@ -681,15 +727,15 @@ public List<NetcdfFileManager> asciiToDsg() {
         return dsgWriters;
     }
 
-    public String createNetcdfFile(AsciiFile file, List<List<String>> parseFileData, String downloadDirPath) throws IOException {
+    public String createNetcdfFile(AsciiFile file, List<List<String>> parseFileData, List<String> header, String downloadDirPath) throws IOException {
         try {
             String ncFilePath = downloadDirPath + File.separator + FilenameUtils.removeExtension(file.getFileName()) + ".nc";
             logger.warn("create ncFilePath: " + ncFilePath);
 
             // make sure downloadDir exists and, if not, create it
             checkDownloadDir(downloadDirPath);
 
-            init(file);
+            init(file, header);
 
             // look for coordinate and non-coordinate variables
             Boolean hasRelTime = findCoordAndNonCoordVars();
@@ -740,7 +786,10 @@ public String createNetcdfFile(AsciiFile file, List<List<String>> parseFileData,
                 return null;
             }
 
+        } catch (IllegalArgumentException e) {
+            throw e;
         } catch (Exception e) {
+            //TODO: Using this broad catch is not very good practice
             logger.error(e.getMessage());
             logger.error(e.getStackTrace());
             return null;

diff --git a/src/main/java/edu/ucar/unidata/rosetta/service/FileParserManager.java b/src/main/java/edu/ucar/unidata/rosetta/service/FileParserManager.java
@@ -65,6 +65,19 @@ public interface FileParserManager {
      */
     public int getBlankLines(File file);
 
+    /**
+     * Returns the lines in the header as List<String>
+     *
+     * @return The header.
+     */
+    public List<String> getHeader();
+
+    /**
+     * Sets the lines in the header as List<String>
+     *
+     * @param header The header.
+     */
+    public void setHeader(List<String> header);
 }
 
 
diff --git a/src/main/java/edu/ucar/unidata/rosetta/service/FileParserManagerImpl.java b/src/main/java/edu/ucar/unidata/rosetta/service/FileParserManagerImpl.java
@@ -22,6 +22,7 @@ public class FileParserManagerImpl implements FileParserManager {
     protected static Logger logger = Logger.getLogger(FileParserManagerImpl.class);
 
     public List<List<String>> parsedFileData = new ArrayList<List<String>>();
+    public List<String> header = new ArrayList<>();
 
     /**
      * Returns each line of the file data parsed by delimiter into a
@@ -86,6 +87,7 @@ public String parseByLine(String filePath) {
      */
     public String normalizeDelimiters(String filePath, String selectedDelimiter, List<String> delimiterList, List<String> headerLineList) {
         List<List<String>> parsedData = new ArrayList<List<String>>();
+        List<String> headerData = new ArrayList<>();
         StringBuffer stringBuffer = new StringBuffer();
         int lineCount = 0;
         String currentLine;
@@ -97,6 +99,7 @@ public String normalizeDelimiters(String filePath, String selectedDelimiter, Lis
                     // If a header line we don't have to deal with the delimiter
                     if (headerLineList.contains(new Integer(lineCount).toString())) {
                         stringBuffer.append(currentLine + "\n");
+                        headerData.add(currentLine);
                     } else {
                         // Parse line data based on delimiter count
                         if (delimiterList.size() != 1) { // more than one delimiter
@@ -135,13 +138,22 @@ public String normalizeDelimiters(String filePath, String selectedDelimiter, Lis
 
             }
             setParsedFileData(parsedData);
+            setHeader(headerData);
         } catch (IOException e) {
             logger.error(e.getMessage());
             return null;
         }
         return stringBuffer.toString();
     }
 
+    public List<String> getHeader() {
+        return header;
+    }
+
+    public void setHeader(List<String> header) {
+        this.header = header;
+    }
+
     /**
      * A simple method that reads each line of a file, and looks for blank lines.
      * Blank line = empty, only whitespace, or null (as per StringUtils).

diff --git a/src/main/webapp/WEB-INF/views/jspf/specifyGeneralMetadata.jspf b/src/main/webapp/WEB-INF/views/jspf/specifyGeneralMetadata.jspf
@@ -39,11 +39,12 @@
                             </c:when>
                         </c:choose>
                         <br/>
+                        <input type='checkbox' name="${globalMetadataItem.tagName}"> is a regex
+                        <br/>
                         <input type="text" name="<c:out value="${globalMetadataItem.tagName}" />"
                                value=""/>
                     </label>
-                    <label for="<c:out value="${globalMetadataItem.tagName}" />"
-                           class="error"></label>
+                    <label class="error"></label>
                 </li>
             </c:forEach>
             <div id="containerForCustomAttributes"/>