Page History

package nl.wldelft.fews.system.plugin.dataImport;

import nl.wldelft.util.ExceptionUtils;
import nl.wldelft.util.FastGregorianCalendar;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.io.FileParser;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.RelativeEquidistantTimeStep;
import nl.wldelft.util.timeseries.SimpleEquidistantTimeStep;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import nl.wldelft.util.timeseries.TimeStep;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.GregorianCalendarList;
import java.util.ListLocale;
import java.util.regex.Pattern;


/**
 * TimeSeries parser for SHEF version 2.0. Note: only the .E and .A messages are implemented and
 * http://www.nws.noaa.gov/os/hod/SHManual/SHMan051_shef.htm
 * http://www.nws.noaa.gov/om/water/resources/SHEF_CodeManual_5July2012.pdf
 */
public class ShefTimeSeriesParser implements FileParser<TimeSeriesContentHandler> {
    private static final Logger log = LoggerLogManager.getLogger(ShefTimeSeriesParser.class);

    public static final String readerType = "SHEF";
    private static final String SINGLE_LOCATION_MULTIPLE_PARAMETERS_TYPE = ".A";

    private static final PatternString COMPILEEND_PATTERNTOKEN_A_CONTINUATIONTYPE = Pattern.compile(".A\\d+END"); // Start with a .A followed by one or more digits.
    

    private static final StringPattern SINGLECOMPILE_LOCATIONPATTERN_MULTIPLE_PARAMETERS_CONTINUATION_TYPEB_HEADER = Pattern.compile(".B|.ARBR"); // B type record
    private static final Pattern COMPILE_PATTERN_ERA_ARCONTINUATION = Pattern.compile(".A\\.ER?$|.AR?$d+");
 // Start with a private.A staticfollowed finalby Pattern COMPILE_PATTERN_ER = Pattern.compile("\\.ER?$");one or more digits.
    private static final Pattern COMPILE_PATTERN_ER_D_AR_DB_CONTINUATION = Pattern.compile("\\.ER?.B\\d+|\\.AR?BR\\d+"); // Start with a .B followed by one or more digits.
    private static final Pattern COMPILE_PATTERN_D_STARString SINGLE_LOCATION_MULTIPLE_PARAMETERS_CONTINUATION_TYPE = Pattern.compile("DS.*|DN.*|DH.*|DD.*|DM.*|DY.*|DJ.*|DR.*");.AR";    
    private static final Pattern COMPILE_PATTERN_DER_AR = Pattern.compile("D.*|\\.ER?$|.AR?$");
    private static final Pattern COMPILE_PATTERN_DIER = Pattern.compile("DI.*\\.ER?$");
    private static final Pattern COMPILE_PATTERN_ER_DHD_DDAR_D = Pattern.compile("DH\\.ER?\\d+|\\.AR?\\d.*+");
    private static final Pattern COMPILE_PATTERN_STARTD_DIGITSTAR = Pattern.compile("[^0-9]DS.*|DN.*|DH.*|DD.*|DM.*|DY.*|DJ.*|DR.*");
    private static final Pattern COMPILE_PATTERN_DIHD = Pattern.compile("D.*DIH.*|");
    private static final Pattern COMPILE_PATTERN_DINDU = Pattern.compile("DU.*DIN.*|");
    private static final Pattern COMPILE_PATTERN_DIDDI = Pattern.compile(".*DIDDI.*");

    private static char quoteCharfinal Pattern COMPILE_PATTERN_DH_DD = '\"';

Pattern.compile("DH\\d\\d.*");
    private Calendarstatic calendarfinal = new GregorianCalendar();

    private TimeSeriesContentHandler contentHandler = null;

    // Variables for parsing the file
    private long time = 0;
    private long dtime = 0;
    private boolean separatorOnLastLine = false;

    private static final int E_TYPE = 0;
    private static final int A_TYPE = 1;
    private int messageType = 0;

    private String aContinuationLocationId = null;
    
    // string array contains all fields before the first / separator
    private int firstSlashSeparatorIdx = 0;


    @Override
    public void parse(File file, TimeSeriesContentHandler contentHandler) throws IOException {
        this.contentHandler = contentHandler;
        calendar.setTimeZone(this.contentHandler.getDefaultTimeZone());
Pattern COMPILE_PATTERN_START_DIGIT = Pattern.compile("[^0-9]");
    private static final Pattern COMPILE_PATTERN_DIH = Pattern.compile(".*DIH.*");
    private static final Pattern COMPILE_PATTERN_DIN = Pattern.compile(".*DIN.*");
    private static final Pattern COMPILE_PATTERN_DID = Pattern.compile(".*DID.*");
    private static final Pattern COMPILE_PATTERN_DR_SHIFT = Pattern.compile("DRS*[+-]\\d+|DRN*[+-]\\d+|DRH*[+-]\\d+|DRD*[+-]\\d+|DRM*[+-]\\d+|DRD*[+-]\\d+");

    private static final char quoteChar = '\"';

    private FastGregorianCalendar calendar = null;

    private TimeSeriesContentHandler contentHandler = null;

    // Variables for parsing the file
    private long time = 0;
    private long dtime = 0;
    private boolean separatorOnLastLine = false;

    private static final int E_TYPE = 0;
    private static final int A_TYPE = 1;
    private int messageType = 0;

    private String aContinuationLocationId = null;
    
    // string array contains all fields before the first / separator
    private int firstSlashSeparatorIdx = 0;

    @Override
    public void parse(File file, TimeSeriesContentHandler contentHandler) throws Exception {
        this.contentHandler = contentHandler;
        calendar = new FastGregorianCalendar(this.contentHandler.getDefaultTimeZone(), Locale.US);

        boolean isValid = readFile(file);
        if (!isValid) {
            throw new IOException("Error parsing: " + file.getName());
        }
    }

    private boolean readFile(File file) throws Exception {

        BufferedReader reader = null;
        boolean done = false;

        //Open file
        try {
            //noinspection resource
            reader = new BufferedReader(new FileReader(file));
            done = true;
        } catch (FileNotFoundException e) {
            log.error(file + " could not be opened.", e);
        }
        //Read/parse the file
        if (done) {

            try {
                if (!parseFile(reader)) {
                    done = false;
                    log.error("The file " + file + " has unknown format.");
                }
            } catch (IOException e) {
                done = false;
                log.error("Error while reading the file " +
                        file + " : " + ExceptionUtils.getMessage(e), e);
            }

            closeReader(reader);

        }

        return done;
    }

    private static void closeReader(BufferedReader reader) {

        try {
            reader.close();
        } catch (IOException e) {
            log.error("Cannot close file " + reader +
                    " : " + ExceptionUtils.getMessage(e), e);
        }
    }

    /**
     * Read file content and store it into the memory
     * Comments on the SHEF file format:
     * - The method recognises .ER and .Ed lines only (d=digit)
     * - All other lines are ignored at the moment
     * <p/>
     * Note:
     * We assume that the fields are separated by spaces and that the
     * .ER and .E records do not contain timeseries data!
     *
     * @return
     * @throws IOException
     */
    private boolean parseFile(BufferedReader reader) throws Exception {

        boolean okay = true;
        String line;

        while ((line = reader.readLine()) != null && okay) {
            StringBuilder commentFreeLine = removeCommentsFromLine(line);
            String[] pieces = TextUtils.split(commentFreeLine.toString(), ':', '\0', quoteChar, false);
            //noinspection UnusedAssignment
            String[] fields = TextUtils.split(pieces[0], ' ', '\0', quoteChar, true);

            // get fields, first part is split by space next part by '/'
            // first split with separator '/' will split up line in first part containing spaces (position part)
            // followed by the datastring fields (separated by '/')
            String[] tmpPieces = pieces[0].split("/");
            String[] positionFields = TextUtils.split(tmpPieces[0], ' ', '\"');
            firstSlashSeparatorIdx = positionFields.length;
            fields = new String[tmpPieces.length + positionFields.length - 1];
            System.arraycopy(positionFields, 0, fields, 0, positionFields.length);
            System.arraycopy(tmpPieces, 1, fields, positionFields.length, tmpPieces.length - 1);

            //Header lines start with .E or .ER (but may comtain data)
            //Data-only lines start with .Ed or .ERd - d a digit
            if (fields.length > 0) {
                if (COMPILE_PATTERN_A_CONTINUATION.matcher(fields[0]).matches()) {
                    parseMultiParameterContinuationSeriesData(fields);
                } else if (COMPILE_PATTERN_ER_AR.matcher(fields[0]).matches()) {
                    this.messageType = COMPILE_PATTERN_ER.matcher(fields[0]).matches() ? E_TYPE : A_TYPE;
                    // bug in OHD output for .A messages. missing '/' slash between parameter id and
                    // value. Remove this when fixed
                    if (this.messageType == A_TYPE) {
                        String[] splitfields = TextUtils.split(fields[fields.length - 1], ' ', '\"');
                        if (splitfields.length > 1) {
                            String[] tmpFields = new String[fields.length + 1];
                            System.arraycopy(fields, 0, tmpFields, 0, fields.length - 1);
                            System.arraycopy(splitfields, 0, tmpFields, fields.length - 1, 2);
                            fields = tmpFields;
                        }
                    }
                    if (SINGLE_LOCATION_MULTIPLE_PARAMETERS_TYPE.equals(fields[0])) {
                        // .A type no headers all data is on one line with multiple parameters.
                        parseMultiParameterSeriesData(fields);
                    } else {
                        // .AR type. Revision on earlier measurement. Assumption is that only one parameter at a time is passed.
                        okay = getSeriesParameters(fields);
                        if (okay) {
                            // see if there are any values on this row, if so start fill series data
                            // because parameter and timestep are mandatory values can be started
                            // from firstSlashSeparator untill end of fields
                            if (fields.length == firstSlashSeparatorIdx + 2 && SINGLE_LOCATION_MULTIPLE_PARAMETERS_CONTINUATION_TYPE.equals(fields[0])) {
                                String value = fields[fields.length - 1];
                                okay = getContinuationSeriesData(value);
                            }
                            if (fields.length > firstSlashSeparatorIdx + 2) {

                                for (int i = firstSlashSeparatorIdx + 2; i < fields.length; i++) {
                                    if (isFloat(fields[i])) {
                                        String[] values = new String[fields.length - i];
                                        System.arraycopy(fields, i, values, 0, fields.length - i);
                                        okay = getSeriesData(values);
                                        break;
                                    }
                                }
                            }
                        }
                    }
                } else if (COMPILE_PATTERN_B_HEADER.matcher(fields[0]).matches()) {
                    // parse complete .B type record
                    parseMultipleLocationMultipleParametersSeries(fields, reader);
                } else if (COMPILE_PATTERN_ER_D_AR_D.matcher(fields[0]).matches()) {
                    // continued line
                    String[] datafields;
                    if (positionFields.length == 1 && separatorOnLastLine) {
                        // slash between rowcontinuation and first value. If previous line ended with a slash
                        // a null value is assumed
                        fields[0] = null;
        boolean isValid = readFile(file);
        if (!isValid) {
   datafields = fields;
       throw new IOException("Error parsing: " + file.getName());
        }


    } else {

    private boolean readFile(File file) {

        BufferedReader reader = null;
     // skip first booleancolumn donewith =normal false;linecontinuation

        //Open file
        try {
       datafields = new String[fields.length  //noinspection resource- 1];
            reader = new BufferedReader(new FileReader(file));
        System.arraycopy(fields, 1, datafields, 0, donefields.length =- true1);
          }  catch (FileNotFoundException e) {
     }
       log.error(file + " could not be opened.", e);
      okay = }getSeriesData(datafields);
         //Read/parse the file
     }
   if (done) {

           separatorOnLastLine try {
= line.endsWith("/");
            }
     if (!parseFile(reader)) {   }
        return okay;
    }

    private static StringBuilder done = false;removeCommentsFromLine(String line) {
        //Split the line into separate fields:
       log.error("The file " + file + " has unknown format.");
          //Remove any comment first
        String[] piecesWithComments = TextUtils.split(line, ':', ':', quoteChar, true);
       }
 StringBuilder commentFreeLine = new StringBuilder(line.length());
       } catchif (IOException e(line.contains(":")) {
            int togglePosition   done = false0;
                log.error("Error while reading the file " +
if (line.startsWith(":")) {
                togglePosition = 1; // if the line starts with a : , the filefirst + " : " + ExceptionUtils.getMessage(e), e);entry should be skipped.
            }

            closeReader(reader);

if (piecesWithComments.length > 0) {
           }

     // we found returnsome done;comments.
    }

    private static void closeReader(BufferedReader reader) {

   for (int i = 0; i  try< piecesWithComments.length; i++) {
              reader.close();
      if (i }% catch2 (IOException== etogglePosition) {
            log.error("Cannot close file " + reader +
         //  the comment toggle is off.
     " : " + ExceptionUtils.getMessage(e), e);
        }

    }


  // see: /**http://www.nws.noaa.gov/om/water/resources/SHEF_CodeManual_5July2012.pdf
     * Read file content and store it into the memory
     * Comments on the SHEF file format: commentFreeLine.append(piecesWithComments[i]);
     * - The method recognises .ER and .Ed lines only (d=digit)
     * - All other lines are ignored at the moment
}
         * <p/>
     * Note:}
     * We assume that the fields are separated}
 by spaces and that the
   } else *{
 .ER and .E records do not contain timeseries data!
   commentFreeLine.append(line);
  *
     * @return}
       * @throwsreturn IOExceptioncommentFreeLine;
     */}

    private static boolean parseFileisFloat(BufferedReaderString readervalue) throws IOException{
        try {

          boolean okay = true;
//noinspection UnusedDeclaration
            Float f  String line;
= TextUtils.parseFloat(value);
        while ((line = reader.readLine()) != null && okay} catch (NumberFormatException e) {
            StringBuilder commentFreeLine = removeCommentsFromLine(line)return false;
        }
    String[] pieces = TextUtils.split(commentFreeLine.toString(), ':', '\0', quoteChar, false) return true;
       }

     //noinspection UnusedAssignment
Read a complete set of .B type records spanning multiple lines
   String[] fieldsprivate =void TextUtils.splitparseMultipleLocationMultipleParametersSeries(piecesString[0], ' ', '\0' fields, quoteChar,BufferedReader truereader);

 throws Exception  {
        // get fieldstime, as firsta partcombination isof splitdate, by space next part by '/'

[observation time]
        String date = fields[2];

     // first split withint separatorifield '/' will split up line in first part containing spaces (position part)= 3;
        while (!COMPILE_PATTERN_D_STAR.matcher(fields[ifield]).matches()) {
            // followed by the datastring fields (separated by '/')
ifield++;
        }
         String[] tmpPiecesobservationTime = piecesfields[0].split("/");ifield];

        // The start  String[] positionFields = TextUtils.split(tmpPieces[0], ' ', '\"');
date/time, and the time step:
        time = parseDate(date, observationTime);

    firstSlashSeparatorIdx = positionFields.length;
  // skip optional D* fields
      fields = new String[tmpPieces.length + positionFields.length - 1];
while (COMPILE_PATTERN_D.matcher(fields[ifield]).matches()) {
            ifield++;
    System.arraycopy(positionFields, 0, fields, 0, positionFields.length); }

        // The list  System.arraycopy(tmpPieces, 1, fields, positionFields.length, tmpPieces.length - 1);

of parameter/time shift headers, including .Bp continuation lines, if any
        String line;
    //Header lines start with .E or .ER (but may comtain data)
String header;
        List<String> headerList = new ArrayList<>();
     //Data-only lines start withdo .Ed{
 or .ERd - d a digit
      for (int i = ifield; i if< (fields.length > 0; i++) {
                if (!COMPILE_PATTERN_A_CONTINUATIONDU.matcher(fields[0i]).matches()) {
headerList.add(fields[i]);
            }
           parseMultiParameterContinuationSeriesData(fields);
      // continue on next line ?
           } elseline if= (COMPILE_PATTERN_ER_AR.matcher(fields[0]).matches()) {removeCommentsFromLine(reader.readLine()).toString();
                    this.messageType = COMPILE_PATTERN_ER.matcher(fields[0]).matches() ? E_TYPE : A_TYPEline = line.replace("  ", " ");
            int ix = line.indexOf(" ");
    // bug in OHD output for .A messages. missingheader '/'= slashix between> parameter0 id and
   ? line.substring(0, ix) : "";
            String newline = line.substring(ix  // value. Remove this when fixed
+ 1).replace(" ", "");
            fields =   TextUtils.split(newline, '/');
      if (this.messageType == A_TYPE) {
  // some inconsistency in '/' separators on continuation lines !
             String[] splitfieldsifield = TextUtils.splitequals(fields[fields.length - 1], ' ', '\"')0].trim(), "") ? 1 : 0;
        } while (COMPILE_PATTERN_B_CONTINUATION.matcher(header).matches());

        // The data lines
     if (splitfields.length > 1)do {
            line     = removeCommentsFromLine(line).toString();
           String[] tmpFieldsline = new String[fields.length + 1];
line.replace("  ", " ");
            if (!line.isEmpty()) {
               System.arraycopy(fields, 0, tmpFields, 0, fields.length - 1 int ix = line.indexOf(" ");
                String location           System.arraycopy(splitfields, 0, tmpFields, fields.length - 1, 2= line.substring(0, ix);
                line = line.substring(ix +         fields = tmpFields1).replace(" ", "");
                fields = TextUtils.split(line, '/');
      }
          writeBFormatLineContent(time, location, headerList, fields);
       }
     }
        } while ((line = reader.readLine()) != null if (SINGLE_LOCATION_MULTIPLE_PARAMETERS_TYPE.equals(fields[0])) {&& !line.contains(END_TOKEN_TYPE));
    }

    private void writeBFormatLineContent(long time, String location, List<String> headerList, String[] fields) throws Exception {
        DefaultTimeSeriesHeader //timeSeriesHeader no= headers all data is on one line with multiple parameters.new DefaultTimeSeriesHeader();
        timeSeriesHeader.setLocationId(location);
        long startTime = time;
        int iHeader    parseMultiParameterSeriesData(fields)= 0;
        for (int iField = 0; iField <      } else fields.length; iField++) {
            // in addition to parameter names, header may contain "DRx"   date//time .AR type. Revision on earlier measurement. Assumption is that only one parameter at a time is passed.shift codes
            while (COMPILE_PATTERN_DR_SHIFT.matcher(headerList.get(iHeader)).matches()) {
                time =       okay = getSeriesParameters(fieldsapplyDateTimeShift(startTime, headerList.get(iHeader));
                iHeader++;
        if (okay) {
  }
            // in addition to location code, line may contain date-time override code
   // see if there are any values on this row, if so start fill series datawhile (COMPILE_PATTERN_D_STAR.matcher(fields[iField]).matches()) {
                time =     applyDateTimeOverride(startTime, fields[iField]);
      // because parameter and timestep are mandatory values can be startediField++;
            }
            timeSeriesHeader.setParameterId(headerList.get(iHeader));
    // from firstSlashSeparator untill end of fields
  String text = fields[iField].trim();
            float value = text.isEmpty() || TextUtils.equals("+", text) || TextUtils.equals("M", text) ? if (fields.length == (firstSlashSeparatorIdx + 2) && SINGLE_LOCATION_MULTIPLE_PARAMETERS_CONTINUATION_TYPE.equals(fields[0])) {
Float.NaN : Float.parseFloat(text);
            contentHandler.setTimeSeriesHeader(timeSeriesHeader);
            contentHandler.setTime(time);
         String value = fields[fields.length - 1]contentHandler.setValue(value);
            contentHandler.applyCurrentFields();
            iHeader++;
        okay}
   = getContinuationSeriesData(value); }

    private long applyDateTimeOverride(long time, String token) throws Exception {
        if (!COMPILE_PATTERN_D_STAR.matcher(token).matches()) return time;

         }calendar.setTimeInMillis(time);
        String value = token.substring(2);
        String code = token.substring(0, 2);
     if (fields.length > firstSlashSeparatorIdx +switch 2(code) {

            case "DY":
                time =  for (int i = firstSlashSeparatorIdx + 2; i < fields.length; i++) {
applydateTimeYear(value, true);
                break;
            case "DM":
                time = if (isFloat(fields[i])) {
applyDateTimeMonth(value, true);
                break;
            case "DD":
               String[] valuestime = new String[fields.length - i]applyDateTimeDay(value);
                break;
            case "DH":
            System.arraycopy(fields, i, values, 0, fields.lengthtime -= iapplyDateTimeHour(value);
                break;
            case "DN":
                okaytime = getSeriesDataapplyDateTimeMinute(valuesvalue);
                break;
            case "DS":
               break time = applyDateTimeSeconds(value);
                break;
            default:
        }
        throw new Exception("ShefTimeSeriesParser: invalid 'Date/Time override code' : " + token);
        }
        return }time;
    }

    private long applydateTimeYear(String value, boolean fixCentury) {
        int i =    }Integer.parseInt(value.substring(0, 2));
        if (fixCentury) {
            // see }
SHEF coding manual 4.1.4 : a 10 year in the future and 90 year in the past window is used }

to assign the century
            int }y else if (COMPILE_PATTERN_ER_D_AR_D.matcher(fields[0]).matches()) {
= calendar.get(Calendar.YEAR);
            int c = y / 100;
     // continued line
     if (100 * c + i > y + 10) c--;
     String[] datafields;
      i = 100 * c + i;
        if (positionFields.length == 1 && separatorOnLastLine) {
        }
        calendar.set(Calendar.YEAR, i);
        return applyDateTimeMonth(value.substring(2), false);
    }

    private long applyDateTimeMonth(String value, boolean fixYear) {
      // slash betweenint rowcontinuationi and first = Integer.parseInt(value. If previous line ended with a slash
substring(0, 2));
        if (fixYear) {
            // see SHEF coding manual  //4.1.4 : a null12-month valuewindow is assumed
used to assign the year that causes the date code to be nearest the system/start date
         fields[0] = null;
 int m = calendar.get(Calendar.MONTH) +1;
            int y = calendar.get(Calendar.YEAR);
    datafields = fields;
      if (i > m + 6) {
        } else {
      y--;
            } else if (i <= m //- skip6) first{
 column with normal linecontinuation
            y++;
            datafields}
 = new String[fields.length - 1];
       calendar.set(Calendar.YEAR, y);
        }
        Systemcalendar.arraycopyset(fieldsCalendar.MONTH, 1, datafields, 0, fields.length i - 1);
        return applyDateTimeDay(value.substring(2));
    }

    private long applyDateTimeDay(String value) }{
        calendar.set(Calendar.DATE, Integer.parseInt(value.substring(0, 2)));
          okay = getSeriesData(datafieldsreturn applyDateTimeHour(value.substring(2));
    }

    private long applyDateTimeHour(String value) {
    }
    calendar.set(Calendar.HOUR_OF_DAY, Integer.parseInt(value.substring(0, 2)));
          separatorOnLastLine = line.endsWith("/"return applyDateTimeMinute(value.substring(2));
    }

    private long applyDateTimeMinute(String value) {
  }
      calendar.set(Calendar.MINUTE,  }

Integer.parseInt(value.substring(0, 2)));
        return okayapplyDateTimeSeconds(value.substring(2));
    }

    private StringBuilderlong removeCommentsFromLineapplyDateTimeSeconds(String linevalue) {
        //Split the line into separate fields:if (!value.isEmpty()) calendar.set(Calendar.SECOND, Integer.parseInt(value.substring(0, 2)));
        //Remove any comment firstreturn calendar.getTimeInMillis();
    }

    String[] piecesWithComments = TextUtils.split(line, ':', ':', quoteChar, true);private long applyDateTimeShift(long time, String token) {
        StringBuilder// commentFreeLine = new StringBuilder(line.length());see SHEF_CodeManual_5July2012.pdf table 13a
        if (!linetoken.containsindexOf(":DR")) {
!= 0) return time;

         commentFreeLinecalendar.appendsetTimeInMillis(linetime);
        String } else {unit = token.substring(2, 3).toUpperCase();
        int increment =  int togglePosition = 0;Integer.parseInt(token.substring(3));
        switch (unit) {
            if (line.startsWith(":")) {case "S":
                togglePosition = 1; // if the line starts with a : , the first entry should be skipped.calendar.add(Calendar.SECOND, increment);
                break;
            }case "N":
            if (piecesWithComments.length > 0) {    calendar.add(Calendar.MINUTE, increment);
                // we found some comments.break;
            case "H":
   for (int i = 0; i < piecesWithComments.length; i++) {
         calendar.add(Calendar.HOUR_OF_DAY, increment);
          if (i % 2 == togglePosition) {break;
            case "D":
           // the comment toggle is off.calendar.add(Calendar.DATE, increment);
                break;
        // see: http://www.nws.noaa.gov/om/water/resources/SHEF_CodeManual_5July2012.pdf    case "M":
                calendar.add(Calendar.MONTH, increment);
       commentFreeLine.append(piecesWithComments[i]);
         break;
            }default:
                }
throw new IllegalArgumentException("ShefTimeSeriesParser: invalid 'Date Relative code' : "    }+ token);
        }
        return commentFreeLinecalendar.getTimeInMillis();
    }

    //  private static boolean isFloat(String value) {Continuation of .A field with multiple parameters per line.
    private void parseMultiParameterContinuationSeriesData(String[] fields) try {
        DefaultTimeSeriesHeader timeSeriesHeader =  //noinspection UnusedDeclarationnew DefaultTimeSeriesHeader();
        timeSeriesHeader.setLocationId(aContinuationLocationId);
        FloatList<String> fparameterValueList = new TextUtils.parseFloatArrayList<>(value);
        for (int i = 1; i } catch (NumberFormatException e< fields.length; i++) {
            if  return false(fields[i] == null) continue;
        }
    String[] result =  return truefields[i].split(" ");
     }

    // Continuation of .A field with multiple parameters per line.parameterValueList.addAll(Arrays.asList(result));
    private void parseMultiParameterContinuationSeriesData(String[] fields) {}
        DefaultTimeSeriesHeader writeMultipleParametersSeries(timeSeriesHeader = new DefaultTimeSeriesHeader(, parameterValueList);

        timeSeriesHeader.setLocationId(aContinuationLocationId);}

    private void writeMultipleParametersSeries(DefaultTimeSeriesHeader timeSeriesHeader, List<String> parameterValueList = new ArrayList<>();) {
        forif (int i = 1; i < fields.length; i++(parameterValueList.size() % 2 != 0) {
            if (fields[i] == null) continue;(!parameterValueList.isEmpty()) {
            String[] result = fields[i].split(" ");
 // Check on special symbols like DC     for (int j = 0; j < result.length; j++) {
(date creation).
                String code = parameterValueList.add(result[j]get(0);
            }
    if (code.startsWith("DC")) return; // }
Creation date. Can      writeMultipleParametersSeries(timeSeriesHeader, parameterValueList);

be ignored.
    }

    private void writeMultipleParametersSeries(DefaultTimeSeriesHeader timeSeriesHeader, List<String> parameterValueList) {}
        if (parameterValueList.size() % 2 != 0) {
            if (!parameterValueList.isEmpty()) {
                // Check on special symbols like DC (date creation).
log.warn("SHEF import line of type .A[0-9]* (single station, multiple parameters) doesn't contain a consistent number of parameters and values. Skipping line");
            return;
     String code = parameterValueList.get(0); }
        for (int i = 0; i <  if (code.startsWith("DC")) return; // Creation date. Can be ignored.parameterValueList.size() / 2; i++) {
            }
String paramId = parameterValueList.get(i         
   * 2);
         log.warn("SHEF import line of type .A[0-9]* (single station, multiple parameters) doesn't contain a consistent number of parameters and values. Skipping line"   float value = parseValue(parameterValueList.get(i * 2 + 1));
            timeSeriesHeader.setParameterId(paramId);
            returncontentHandler.setTimeSeriesHeader(timeSeriesHeader);
        }
    contentHandler.setTime(time);
    for  (int i = 0; i < parameterValueListcontentHandler.sizesetValue(value);
 / 2; i++) {
        contentHandler.applyCurrentFields();
    String paramId = parameterValueList.get(i * 2);}
    }


    private void parseMultiParameterSeriesData(String[] fields) float{
 value = parseValue(parameterValueList.get(i * 2 + 1));
  // .A ANAW1 20170215 P DH2400 /DH08 /HGIRX 8.37 /QRIRX timeSeriesHeader.setParameterId(paramId);41.69
        // Couting fields  contentHandler.setTimeSeriesHeader(timeSeriesHeader);
from 0:
        // Field 1 is the  contentHandler.setTime(time);name of the location
        // Field 2 is the contentHandler.setValue(value);date (possibly without a year)

        // Skip all /D contentHandlerparts.applyCurrentFields();
        }
    }


    private void parseMultiParameterSeriesData(String[] fields) {// parameter code 1
        // .A ANAW1 20170215 P DH2400 /DH08 /HGIRX 8.37 /QRIRX 41.69 parameter value 1
        // ..
        // Coutingparameter fields from 0:code N
        // Fieldparameter 1value is the name of the location N

        // Fieldget 2location isid
 the date (possibly without a year)

  String locationId     // Skip all /D parts.
= fields[1];
        String date = fields[2];
        //String parameterobservationTime code 1= "";

        // parameter value 1 get time, as a combination of date, [observation time]
        // .. get observation time if exist
        //int parameterifield code= N3;
         // parameter value  N

for (; ifield < fields.length; ifield++) {
        // get location id
 if (COMPILE_PATTERN_D_STAR.matcher(fields[ifield]).matches()) {
     String locationId = fields[1];
        String dateobservationTime = fields[2ifield];
        String observationTime = "";

     break;
   // get time, as a combination of date, [observation time]}
        //}
 get observation time if exist
   // The start date/time, and intthe ifieldtime = 3;
step:
        time for= parseDate(; ifield < fields.length; ifield++) {date, observationTime);
        aContinuationLocationId    if (COMPILE_PATTERN_D_STAR.matcher(fields[ifield]).matches()) {
       =locationId; // Keep location id in case .A continutations are found.

        DefaultTimeSeriesHeader observationTimetimeSeriesHeader = fields[ifield] new DefaultTimeSeriesHeader();
        timeSeriesHeader.setLocationId(locationId);
        ifield = breakfirstSlashSeparatorIdx;
        // get parameter  }
        }
        // The start date/time, and the time step:
        time = parseDate(date, observationTime);id, i.e. the first field without a D in prefix
        for (; ifield < fields.length; ifield++) {
            if (!COMPILE_PATTERN_D.matcher(fields[ifield]).matches()) {
        aContinuationLocationId =locationId; // Keep location id in case .A continutationsbreak;
 are found.

        DefaultTimeSeriesHeader timeSeriesHeader = new DefaultTimeSeriesHeader();
  // Index of first parameter was timeSeriesHeaderfound.setLocationId(locationId);
          ifield = firstSlashSeparatorIdx;}
        //}
 get parameter id, i.e. the first field withoutList<String> aparameterValueList D= in prefixnew ArrayList<>();
        for (int i = ifield; ifieldi < fields.length; ifieldi++) {
            if (!COMPILE_PATTERN_D.matcher (fields[ifield]).matches()) {i] == null) continue;
            String[] result =  breakfields[i].split(" ");
            parameterValueList.addAll(Arrays.asList(result));
       // Index}
 of first parameter was found.
   writeMultipleParametersSeries(timeSeriesHeader, parameterValueList);
    }
    }
    private boolean getSeriesParameters(String[] fields) }{
        List<String>// parameterValueListCouting =fields new ArrayList<>();from 0:
        for// (intField i1 =is ifield; i < fields.length; i++) {the name of the location
        // Field 2 is the ifdate (fields[i] == null) continue;possibly without a year)
        // Field 3 is String[] result = fields[i].split(" ");the timezone (optional!)
        // observation time (optional)
   for (int j = 0; j// < result.length; j++) {creation date (optional)
        // units code (optional)
      parameterValueList.add(result[j]);  // Data string qualifier (optional)
        // Duration code (optional)
  }
      // parameter }code
        // the writeMultipleParametersSeries(timeSeriesHeader, parameterValueList);time interval (only for .E messages)

    }
    
// get location id
 private boolean getSeriesParameters(String[] fields) {
   String locationId = fields[1];
  // Couting fields from 0:
  String date = fields[2];
   // Field 1 is the nameString ofobservationTime the= location"";
        // Fieldget 2time, isas thea datecombination (possiblyof withoutdate, a year)[observation time]
        // Fieldget 3observation istime the timezone (optional!)if exist
        //int observationifield time (optional)
= 3;
         // creation date (optional)for (; ifield < fields.length; ifield++) {
        //   units code (optional)
if (COMPILE_PATTERN_D_STAR.matcher(fields[ifield]).matches()) {
         // Data string qualifier (optional)
   observationTime = fields[ifield];
   // Duration code (optional)
        // parameter codebreak;
        // the time interval (only for .E messages)

}
        }
 // get location id
    // The start date/time, Stringand locationIdthe = fields[1];
time step:
        time String= parseDate(date = fields[2], observationTime);
        String observationTimeifield = ""firstSlashSeparatorIdx;
        // get time, as a combination of date, [observation time]
        // get observation time if existparameter id, i.e. the first field without a D in prefix
        intString ifieldparameterId = 3null;
        for (; ifield < fields.length; ifield++) {
            if (!COMPILE_PATTERN_D_STAR.matcher(fields[ifield]).matches()) {
                observationTimeparameterId = fields[ifield];
                break;
            }
        }
        // Theget starttimestep date/timefield, mandatory andonly thefor time.E step:messages
        time =for parseDate(date, observationTime);
 ifield       ifield = firstSlashSeparatorIdx;
< fields.length; ifield++) {
         // get parameter id, i.e. the first field without a D in prefix
if (COMPILE_PATTERN_DI.matcher(fields[ifield]).matches()) {
                String parameterIdtimestep = nullfields[ifield];
        for (; ifield < fields.length; ifield++) {
             dtime if= (!COMPILE_PATTERN_D.matcher(fields[ifield]).matches()) {
parseTimeStep(timestep);
                break;
  parameterId = fields[ifield];
        }
        break;}

        DefaultTimeSeriesHeader timeSeriesHeader = new }DefaultTimeSeriesHeader();

        }timeSeriesHeader.setLocationId(locationId);
        timeSeriesHeader.setParameterId(parameterId);

  // get timestep field, mandatory only forif (this.messageType == E_TYPE) messages{
         for   (;TimeStep ifieldrelativeEqTimeStep <= fields.length; ifield++) {RelativeEquidistantTimeStep.getInstance(dtime, time);
            if (COMPILE_PATTERN_DI.matcher(fields[ifield]).matches()) {timeSeriesHeader.setTimeStep(SimpleEquidistantTimeStep.getInstance(relativeEqTimeStep.getMaximumStepMillis())); //supports only equidistant time steps, see parseTimeStep
            timeSeriesHeader.setForecastTime(time);
     String timestep = fields[ifield];
 }

        contentHandler.setTimeSeriesHeader(timeSeriesHeader);

        if dtime(this.messageType = parseTimeStep(timestep);= E_TYPE) {
            return time != 0 break;
&& dtime != 0; //AM: error conditions?
      }
  } else {
    }

        DefaultTimeSeriesHeaderreturn timeSeriesHeadertime != new DefaultTimeSeriesHeader()0;

        timeSeriesHeader.setLocationId(locationId);
}
    }

    private boolean timeSeriesHeader.setParameterId(parameterId);getSeriesData(String[] fields) {

        iffor (this.messageType == E_TYPEint i = 0; i < fields.length; i++) {
            float value = timeSeriesHeader.setTimeStep(RelativeEquidistantTimeStep.getInstance(dtime, time));parseValue(fields[i]);

            timeSeriesHeadercontentHandler.setForecastTimesetTime(time);
        }
        contentHandler.setTimeSeriesHeadersetValue(timeSeriesHeadervalue);

         if (this.messageType == E_TYPE) { contentHandler.applyCurrentFields();

            return time !+= dtime;
 0 && dtime != 0; //AM: error conditions?}
        } else {return true;
    }

    private boolean getContinuationSeriesData(String valueString) return{
 time != 0;
     float value = }parseValue(valueString);
    }

    private boolean getSeriesData(String[] fields) {

contentHandler.setTime(time);
         for (int i = 0; i < fields.length; i++) {
contentHandler.setValue(value);
        contentHandler.applyCurrentFields();
        return true;
     float value = parseValue(fields[i]);}


    private long parseDate(String str, String timestr) {

        int year;
   contentHandler.setTime(time);
     int month;
      contentHandler.setValue(value);
  int day;

        if contentHandler(str.applyCurrentFields();
length() == 8) {
            timeyear += dtimeInteger.parseInt(str.substring(0, 4));
        }
    month = Integer.parseInt(str.substring(4, 6));
   return true;
    }

    privateday boolean getContinuationSeriesData(String valueString) {= Integer.parseInt(str.substring(6, 8));
        float} value = parseValue(valueString);else {

        contentHandler.setTime(time);
       if contentHandler(str.setValuelength(value);
 == 6) {
     contentHandler.applyCurrentFields();
        return true;
  year = }


100 * (calendar.get(Calendar.YEAR) / private100) long+ parseDateInteger.parseInt(String str.substring(0, String timestr) {

2));
         int year;
      month  int month= Integer.parseInt(str.substring(2, 4));
        int day;

       day if= Integer.parseInt(str.lengthsubstring(4, 6));
   == 8) {
       } else {
    year = Integer.parseInt(str.substring(0, 4));
         // SHEF code monthmanual = Integer.parseInt(str.substring(4, 6));2012 explicitly states:
            day = Integer.parseInt(str.substring(6, 8));
        } else {

          // When “yy” (year) is not explicitly coded, a 12-month window is used to assign the year that causes the
      if (str.length() == 6) {
      // date code to be nearest the current system date yearat =the 100time *of (calendar.get(Calendar.YEAR) / 100) + Integer.parseInt(str.substring(0, 2));
decoding.
                // Outside the 12-month current monthdate-centered = Integer.parseInt(str.substring(2, 4));
   default window, a year other than the
             day = Integer.parseInt(str.substring(4, 6));
            } else { // default year must be explicitly specified. Also, exercise caution when choosing not to explicitly
                // onlycode onlyyear monthin andSHEF daymessages. areIf giventhese accordingmessages toare shefarchived 2.0in specraw takeform, theheader lastrecords 12 monthsmust
                // beforebe currentadded andin take the yeararchive thatfunction matches the month day. i.e. check if date is in future
       to make future determination of the correct year possible for
          // with current year, if so take// lastretrieval yearsoftware.
                year = calendar.get(Calendar.YEAR);
                month = Integer.parseInt(str.substring(0, 2));
                day = Integer.parseInt(str.substring(2, 4));
       
     }

            if (month > calendar.get(Calendar.MONTH) + 5) {
                    year--;
                }
                if (month < calendar.get(Calendar.MONTH) - 6) {
                    year++;
                }
            }
        }

        // The time string (actually a composite thing)
        // We expect something like: DH12/... If not, ignore this field
        // -- AM: TODO!
        int hour = 0;
        int minute = 0;
        int second = 0;
        if (COMPILE_PATTERN_DH_DD.matcher(timestr).matches()) {
            hour = Integer.parseInt(timestr.substring(2, 4));
            if (timestr.length() == 6) {
                minute = TextUtils.tryParseInt(timestr.substring(4, 6), 0);
            }
            if (timestr.length() == 8) {
                second = TextUtils.tryParseInt(timestr.substring(6, 8), 0);
            }

        }

        calendar.clear();
        //noinspection MagicConstant
        calendar.set(year, month - 1, day, hour, minute, second); /* Correct for the offset: month numbers start at 0*/
        return calendar.getTimeInMillis();
    }

    private static long parseTimeStep(String str) {

        long scale;
        long value = Long.parseLong(COMPILE_PATTERN_START_DIGIT.matcher(str).replaceAll(""));

        if (COMPILE_PATTERN_DIH.matcher(str).matches()) {
            scale = 3600 * 1000;
        } else if (COMPILE_PATTERN_DIN.matcher(str).matches()) {
            scale = 60 * 1000;
        } else if (COMPILE_PATTERN_DID.matcher(str).matches()) {
            scale = 86400 * 1000;
        } else {
            scale = 0;
        }

        return scale * value;
    }

    private static float parseValue(String valueText) {

        float value = Float.NaN;

        if (valueText != null) {
            valueText = valueText.trim();
            if (!valueText.isEmpty()) {
                // Missing value can be marked +, -, m, mm, M, MM, -9999
                if (!(valueText.equalsIgnoreCase("M") || valueText.equalsIgnoreCase("MM") || valueText.equalsIgnoreCase("-") ||
                        valueText.equalsIgnoreCase("+") || valueText.equalsIgnoreCase("-9999"))) {
                    try {
                        value = TextUtils.parseFloat(valueText);
                    } catch (NumberFormatException e) {
                        // TODO : According to the specs of SHEF:
                        // If no legitimate value is found, the value is treated as a null field or no report.
                        // So we should return missingValue
                        value = Float.NaN;
                    }
                }
            }
        }
        return value;
    }
}