...
package nl.wldelft.timeseriesparsers;
...
import nl.wldelft.util.
...
TextUtils;
...
import nl.wldelft.util.TimeUnit;
...
import nl.wldelft.util.TimeZoneUtils;
...
import nl.wldelft.util.io.LineReader;
...
import nl.wldelft.util.io.TextParser;
...
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
...
import nl.wldelft.util.timeseries.SimpleEquidistantTimeStep;
...
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
...
import nl.wldelft.util.timeseries.TimeStep;
...
import org.apache.logging.log4j.Logger;
...
import org.apache.logging.log4j.LogManager;
import java.io.IOException;
...
import java.text.ParseException;
...
import java.util.TimeZone
...
;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/*
*
...
* Imports time series data in a tabular ASCII format from the Australian Bureau of
...
* Meteorology (HCS). The files consist of a set of header lines and then lines
...
* with a fixed number of fields. The fields are separated by a comma and the order is fixed.
...
* Multiple locations and parameters can be put in a single file.
...
*
...
* Example:
...
* # HEADER: Agency Id: BoM
...
* # HEADER: File Generation Date: 2008-08-01T14:30:45z
...
* # HEADER: File Format: BOM-HCS
...
* # HEADER: File Format Version: 2.0
...
* # HEADER: Generated by (system): TimeStudio
...
* # HEADER: Number of Records: 8
...
* # HEADER: Local ObsTime Offset: 0
...
* # HEADER: Data Fields: IndexNo, SensorType, SensorDataType, SiteIdType, SiteId, ObservationTimestamp, RealValue, Unit, SensorTypeParam1, SensorTypeParam2, Quality, Comment
...
* 1,"WL",1,"SLSR","44198-01-01","2008-02-01T01:07:06z",1.150000,"metres","LGH",,1,""
...
* 2,"WL",1,"SLSR","44198-01-01","2008-02-01T01:08:06z",1.200000,"metres","LGH",,1,""
...
* 3,"WL",1,"SLSR","44198-01-01","2008-02-01T01:43:06z",1.150000,"metres","LGH",,1,""
...
* 4,"WL",1,"SLSR","44198-01-01","2008-02-01T01:46:06z",1.200000,"metres","LGH",,1,""
...
* 5,"WL",1,"SLSR","44198-01-01","2008-02-01T02:04:06z",1.150000,"metres","LGH",,1,""
...
* 6,"WL",1,"SLSR","44198-01-01","2008-02-01T02:11:06z",1.200000,"metres","LGH",,1,""
...
* 7,"WL",1,"SLSR","44198-01-01","2008-02-01T02:16:06z",1.150000,"metres","LGH",,1,""
...
* 8,"WL",1,"SLSR","44198-01-01","2008-02-01T02:22:06z",1.200000,"metres","LGH",,1,""
...
*
...
*/
...
public final class HcsTimeSeriesParser implements TextParser<TimeSeriesContentHandler> {
...
private static final Logger log =
...
LogManager.getLogger(
...
);
...
private TimeZone defaultTimeZone = null;
...
private LineReader reader = null;
...
private TimeSeriesContentHandler handler = null;
...
//Regular expression for timezone in hours
//Matches: can start with - or + or without, the numbers can be 2 digits starting with 0 or 2 digists starting with 1 followed by 0-3, or 1 digit that is 0-9
private static final Pattern pattern = Pattern.compile("[\\-\\+]?(0[0-9]|1[0-3]|[0-9])");
@Override
...
public void parse(LineReader reader, String virtualFileName, TimeSeriesContentHandler handler) throws IOException {
...
this.reader = reader;
...
this.handler = handler;
...
handler.addMissingValue("");
...
DefaultTimeSeriesHeader header = new DefaultTimeSeriesHeader();
...
reader.setSkipEmptyLines(true);
...
parseHeader();
...
for (String[] buffer = new String[12]; reader.readLine(',', '\"', buffer) != -1; ) {
...
header.setParameterId(buffer[1]);
...
header.setLocationId(buffer[4]);
...
String timeText = buffer[5];
...
if (TextUtils.trimToNull(timeText) == null) continue;
...
TimeZone timeZone = defaultTimeZone;
...
if (TextUtils.endsWithIgnoreCase(timeText, "Z")) {
...
timeZone =
...
TimeZoneUtils.GMT;
...
timeText = timeText.substring(0, timeText.length() - 1);
...
}
...
handler.setTime(timeZone, "yyyy-MM-dd'T'HH:mm:ss", timeText);
...
handler.setValue('.', buffer[6]);
...
header.setUnit(buffer[7] + " " + buffer[8]);
...
header.setTimeStep(parseTimeStep(buffer, timeZone));
...
handler.setFlag(buffer[10]);
...
handler.setComment(buffer[11]);
...
handler.setTimeSeriesHeader(header);
...
handler.applyCurrentFields();
...
}
...
}
...
private TimeStep parseTimeStep(String[] buffer, TimeZone timeZone) {
...
String timeStepString
...
=
...
TextUtils.trimToNull(buffer[9]);
if (timeStepString == null) return null;
...
if (TextUtils.equals(buffer[2], "1")) return null;
...
try {
...
int timeStepSeconds = TextUtils.parseInt(timeStepString);
if (timeStepSeconds == 0) return null;
return SimpleEquidistantTimeStep.getInstanceFromTimeZone(
...
timeStepSeconds * TimeUnit.SECOND_MILLIS, timeZone);
...
} catch (NumberFormatException e) {
...
log.
...
warn("Can not parse time step " + buffer[9] + " at " + reader.getFileAndLineNumber());
...
return null;
...
}
...
}
...
private void parseHeader() throws IOException {
...
defaultTimeZone = handler.getDefaultTimeZone();
...
reader.mark(200);
...
for (String line; (line = this.reader.readLine()) != null; reader.mark(200)) {
...
line = line.trim();
...
if (line.charAt(0) != '#') {
...
// this is not a header row, undo read line
...
reader.reset();
...
break;
...
}
...
// Supported
...
formats:
//
...
# HEADER: Local ObsTime Offset: (+/-)(h)h(:mm)
...
String[] elements = TextUtils.split(line, ':');
if (elements.length < 3) continue;
...
...
...
...
...
...
...
TextUtils.trimElements(elements);
...
...
...
if (!elements[1].equalsIgnoreCase("Local ObsTime Offset
...
")) continue;
StringBuilder timeZone = new StringBuilder("");
//
...
If length of elements is 4 or higher the timezone has minutes defined: (+/-)(h)h:mm
...
if (elements.length >=
...
4) {
timeZone.append(elements[2] + ':' + elements[3].substring(0, 2)
...
);
} else {
// If length
...
of ...
elements is less than 4, only hours are given and minutes ":00" will be appended
//A matcher is used to match the pattern to make sure only the correct format is taken
Matcher matcher =
...
pattern.
...
matcher(elements[2]);
...
if (
...
matcher.find()) {
timeZone.append(matcher.group());
...
...
...
}
...
...
...
...
...
...
...
...
...
...
...
...
...
...
timeZone.append(":00"
...
);
}
...
...
//If a '+' or '-' is missing, a '+' will be prepended
...
...
if(timeZone.charAt(0) !=
...
'+'
...
&& timeZone.charAt(0)
...
!= '-')
...
timeZone...
.insert(0, "+");
//If the length is lower than 5 an extra 0 is inserted so there are 2 digits for the hours
if (timeZone.length() <= 5) timeZone.insert(1, "0");
try {
...
...
...
defaultTimeZone = TimeZoneUtils.parseTimeZone("GMT" + timeZone);
...
...
} catch (ParseException e) {
...
log.error("Can not parse time zone " + timeZone + " at " + reader.getFileAndLineNumber());
...
}
...
}
...
}
...
}
...