/* ================================================================
* Delft FEWS
* ================================================================
*
* Project Info: http://www.wldelft.nl/soft/fews/index.html
* Project Lead: Karel Heynert (karel.heynert@wldelft.nl)
*
* (C) Copyright 2003, by WL | Delft Hydraulics
* P.O. Box 177
* 2600 MH Delft
* The Netherlands
* http://www.wldelft.nl
*
* DELFT-FEWS is a sophisticated collection of modules designed
* for building a FEWS customised to the specific requirements
* of individual agencies. An open modelling approach allows users
* to add their own modules in an efficient way.
*
* ----------------------------------------------------------------
* PiTimeSeriesParser.java
* ----------------------------------------------------------------
* (C) Copyright 2003, by WL | Delft Hydraulics
*
* Original Author: Erik de Rooij
* Original Author: Onno van den Akker
*/
package nl.wldelft.fews.pi;
import nl.wldelft.util.BinaryUtils;
import nl.wldelft.util.CharArrayUtils;
import nl.wldelft.util.Clasz;
import nl.wldelft.util.ExceptionUtils;
import nl.wldelft.util.FastDateFormat;
import nl.wldelft.util.FileUtils;
import nl.wldelft.util.FloatArrayUtils;
import nl.wldelft.util.NumberType;
import nl.wldelft.util.Period;
import nl.wldelft.util.Properties;
import nl.wldelft.util.StringArrayUtils;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.TimeUnit;
import nl.wldelft.util.TimeZoneUtils;
import nl.wldelft.util.geodatum.GeoDatum;
import nl.wldelft.util.io.VirtualInputDir;
import nl.wldelft.util.io.VirtualInputDirConsumer;
import nl.wldelft.util.io.XmlParser;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.IrregularTimeStep;
import nl.wldelft.util.timeseries.OutOfDetectionRangeFlag;
import nl.wldelft.util.timeseries.ParameterType;
import nl.wldelft.util.timeseries.RelativeEquidistantTimeStep;
import nl.wldelft.util.timeseries.SimpleEquidistantTimeStep;
import nl.wldelft.util.timeseries.TimeSeriesArray;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import nl.wldelft.util.timeseries.TimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeStep;
import nl.wldelft.util.timeseries.ValueSource;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
public class PiTimeSeriesParser implements XmlParser<TimeSeriesContentHandler>, VirtualInputDirConsumer {
private static final int BUFFER_SIZE = 2048;
private final Properties.Builder propertyBuilder = new Properties.Builder();
private char[] charBuffer = null;
private float minValueResolution = Float.NaN;
private float[][] axisValues = null;
private float[] axisValueResolutions = Clasz.floats.emptyArray();
private boolean axesDirty = false;
private String[] domainParameterIds = Clasz.strings.emptyArray();
private String[] domainUnits = Clasz.strings.emptyArray();
private int domainCount = 0;
@Override
public void setVirtualInputDir(VirtualInputDir virtualInputDir) {
this.virtualInputDir = virtualInputDir;
}
private enum HeaderElement {
type(F.R), moduleInstanceId, locationId(F.R),
parameterId(F.R), qualifierId(F.M), ensembleId, ensembleMemberIndex, ensembleMemberId,
timeStep(F.R | F.A), startDate(F.R | F.A), endDate(F.R | F.A), forecastDate(F.A), approvedDate(F.A),
missVal, longName, stationName, lat, lon, x, y, z, units, domainAxis(F.M | F.A),
sourceOrganisation, sourceSystem, fileDescription,
creationDate, creationTime, region, thresholds,
firstValueTime(F.A), lastValueTime(F.A), maxValue, minValue, valueCount, maxWarningLevelName;
interface F {
int A = 1 << 0; // attributes
int R = 1 << 1; // required;
int M = 1 << 2; // multiple;
}
private final int flags;
HeaderElement() {
this.flags = 0;
}
HeaderElement(int flags) {
this.flags = flags;
}
public boolean isRequired() {
return (flags & F.R) != 0;
}
public boolean hasAttributes() {
return (flags & F.A) != 0;
}
public boolean isMultipleAllowed() {
return (flags & F.M) != 0;
}
}
// fastDateFormat is used to keep track of last time zone and lenient
private FastDateFormat fastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd", "HH:mm:ss", TimeZoneUtils.GMT, Locale.US, null);
private FastDateFormat fastDateFormatWithMillies = FastDateFormat.getInstance("yyyy-MM-dd", "HH:mm:ss.SSS", TimeZoneUtils.GMT, Locale.US, null);
private HeaderElement currentHeaderElement = null;
private static final HeaderElement[] HEADER_ELEMENTS = HeaderElement.class.getEnumConstants();
private PiTimeSeriesHeader header = new PiTimeSeriesHeader();
private List<String> qualifiers = new ArrayList<>();
private long timeStepMillis = 0;
private TimeStep timeStep = null;
private long startTime = Long.MIN_VALUE;
private long endTime = Long.MIN_VALUE;
private float missingValue = Float.NaN;
private String creationDateText = null;
private String creationTimeText = null;
private long lastTime = Long.MIN_VALUE;
private long lastStartTime = Long.MIN_VALUE;
private long lastEndTime = Long.MIN_VALUE;
private boolean timeAmbiguous = false;
private boolean lastTimeAmbiguous = false;
private boolean lastStartTimeAmbiguous = false;
private boolean lastEndTimeAmbiguous = false;
private TimeSeriesContentHandler timeSeriesContentHandler = null;
private PiTimeSeriesSerializer.EventDestination eventDestination = null;
/**
* For performance reasons the pi time series format allows that the values are stored in
* a separate bin file instead of embedded in the xml file.
* The bin file should have same name as the xml file except the extension equals bin
* In this case all time series should be equidistant.
*/
private VirtualInputDir virtualInputDir = VirtualInputDir.NONE;
private InputStream binaryInputStream = null;
private byte[] byteBuffer = null;
private float[] floatBuffer = null;
private int bufferPos = 0;
private int bufferCount = 0;
private XMLStreamReader reader = null;
private String virtualFileName = null;
private static boolean lenient = false;
private double lat = Double.NaN;
private double lon = Double.NaN;
private double z = Double.NaN;
/**
* For backwards compatibility. Earlier versions of the PiTimeSeriesParser were tolerant about the date/time format
* and the case insensitive for header element names.
* This parser should not accept files that are not valid according to pi_timeseries.xsd
* When old adapters are not working you can UseLenientPiTimeSeriesParser temporary till the adapter is fixed
*
* @param lenient
*/
public static void setLenient(boolean lenient) {
PiTimeSeriesParser.lenient = lenient;
}
public static boolean isLenient() {
return lenient;
}
public PiTimeSeriesParser() {
fastDateFormat.setLenient(lenient);
fastDateFormatWithMillies.setLenient(lenient);
}
@Override
public void parse(XMLStreamReader reader, String virtualFileName, TimeSeriesContentHandler timeSeriesContentHandler) throws Exception {
this.reader = reader;
this.virtualFileName = virtualFileName;
this.timeSeriesContentHandler = timeSeriesContentHandler;
String virtualBinFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "bin");
// time zone can be overruled by one or more time zone elements in the pi file
this.fastDateFormat.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone());
this.fastDateFormatWithMillies.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone());
if (!virtualInputDir.exists(virtualBinFileName)) {
eventDestination = PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED;
parse();
return;
}
eventDestination = PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE;
binaryInputStream = virtualInputDir.getInputStream(virtualBinFileName);
try {
if (byteBuffer == null) {
byteBuffer = new byte[BUFFER_SIZE * NumberType.FLOAT_SIZE];
floatBuffer = new float[BUFFER_SIZE];
}
parse();
boolean eof = bufferPos == bufferCount && binaryInputStream.read() == -1;
if (!eof)
throw new IOException("More values available in bin file than expected based on time step and start and end time\n" + FileUtils.getPathWithOtherExtension(virtualFileName, "bin"));
} finally {
bufferPos = 0;
bufferCount = 0;
binaryInputStream.close();
binaryInputStream = null;
}
}
private void parse() throws Exception {
reader.require(XMLStreamConstants.START_DOCUMENT, null, null);
reader.nextTag();
reader.require(XMLStreamConstants.START_ELEMENT, null, "TimeSeries");
reader.nextTag();
while (reader.getEventType() != XMLStreamConstants.END_ELEMENT) {
TimeZone timeZone = PiParserUtils.parseTimeZone(reader); //returns null if the timeZone is not present in the file
if (timeZone != null) {
this.fastDateFormat.setTimeZone(timeZone);
this.fastDateFormatWithMillies.setTimeZone(timeZone);
}
if (noTimeSeries()) continue;
readTimeSeries();
}
reader.require(XMLStreamConstants.END_ELEMENT, null, "TimeSeries");
reader.next();
reader.require(XMLStreamConstants.END_DOCUMENT, null, null);
}
private boolean noTimeSeries() {
return reader.getEventType() == XMLStreamConstants.END_ELEMENT && reader.getLocalName().equals("TimeSeries");
}
private void readTimeSeries() throws Exception {
reader.require(XMLStreamConstants.START_ELEMENT, null, "series");
reader.nextTag();
parseHeader();
lastTime = Long.MIN_VALUE;
timeAmbiguous = false;
lastTimeAmbiguous = false;
lastStartTimeAmbiguous = false;
lastEndTimeAmbiguous = false;
timeSeriesContentHandler.setProperties(Properties.NONE);
if (eventDestination == PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED) {
while (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {
String localName = reader.getLocalName();
if (TextUtils.equals(localName, "event")) {
parseEvent();
} else if (TextUtils.equals(localName, "domainAxisValues")) {
parseDomainAxisValues();
} else if (TextUtils.equals(localName, "properties")) {
parseProperties();
} else {
break;
}
}
} else {
assert eventDestination == PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE;
readValuesFromBinFile();
}
if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {
// skip comment
reader.require(XMLStreamConstants.START_ELEMENT, null, "comment");
reader.getElementText();
reader.nextTag();
}
reader.require(XMLStreamConstants.END_ELEMENT, null, "series");
reader.nextTag();
}
private void parseDomainAxisValues() throws Exception {
String domainParameterId = reader.getAttributeValue(null, "parameterId");
if (domainParameterId == null)
throw new Exception("Attribute parameterId for domainAxisValues is missing");
int index = StringArrayUtils.indexOf(domainParameterIds, 0, domainCount, domainParameterId);
if (index == -1)
throw new Exception("parameterId " + domainParameterId + " for domainAxisValues not defined in header");
if (axisValues == null) axisValues = new float[domainCount][];
if (axisValueResolutions.length < domainCount) axisValueResolutions = new float[domainCount];
int count = parseValueList();
reader.require(XMLStreamConstants.END_ELEMENT, null, "domainAxisValues");
reader.nextTag();
axisValues[index] = Clasz.floats.copyOfArrayRange(floatBuffer, 0, count);
axisValueResolutions[index] = minValueResolution;
axesDirty = true;
}
private int parseValueList() throws Exception {
if (charBuffer == null) charBuffer = new char[100];
char[] charBuffer = this.charBuffer;
if (floatBuffer == null) floatBuffer = new float[100];
minValueResolution = Float.POSITIVE_INFINITY;
int i = 0;
int length = 0;
int j = 0;
while (reader.next() == XMLStreamConstants.CHARACTERS) {
length -= i;
assert length >= 0;
// shift the remaining part of the buffer to the start
if (length > 0) CharArrayUtils.arraycopy(charBuffer, i, charBuffer, 0, length);
int textLength = reader.getTextLength();
if (charBuffer.length < length + textLength) {
charBuffer = Clasz.chars.copyOfArrayRange(charBuffer, 0, length, length + textLength);
this.charBuffer = charBuffer;
}
int n = reader.getTextCharacters(0, charBuffer, length, textLength);
assert n == textLength;
length += textLength;
i = 0;
for (; ; ) {
int start = CharArrayUtils.indexOfNonWhitespace(charBuffer, i, length - i);
if (start == -1) {
i = length;
break;
}
int end = CharArrayUtils.indexOfWhitespace(charBuffer, start, length - start);
if (end == -1) {
i = start;
break;
}
i = end;
parseValue(charBuffer, j++, start, end - start + 1);
}
}
if (i != length) parseValue(charBuffer, j++, i, length - i);
return j;
}
private void parseValue(char[] charBuffer, int pos, int start, int length) {
float value = (float) TextUtils.parseDouble(charBuffer, start, length, '.');
if (value == missingValue) value = Float.NaN;
float valueResolution = TextUtils.getValueResolution(charBuffer, start, length, '.');
if (valueResolution < minValueResolution) minValueResolution = valueResolution;
if (floatBuffer.length == pos) floatBuffer = Clasz.floats.ensureCapacity(floatBuffer, pos + 1);
floatBuffer[pos] = value;
}
private void parseHeader() throws Exception {
domainCount = 0;
Arrays.fill(domainParameterIds, null);
Arrays.fill(domainUnits, null);
if (axisValues != null) Arrays.fill(axisValues, null);
reader.require(XMLStreamConstants.START_ELEMENT, null, "header");
if (reader.getAttributeCount() > 0) {
throw new Exception("Attributes are not allowed for header element ");
}
reader.nextTag();
initHeader();
do {
detectHeaderElement();
parseHeaderElement();
} while (reader.getEventType() != XMLStreamConstants.END_ELEMENT);
if (header.getForecastTime() == Long.MIN_VALUE) header.setForecastTime(startTime);
if (!Double.isNaN(lat)) header.setGeometry(GeoDatum.WGS_1984.createLatLongZ(lat, lon, z));
initiateTimeStep();
header.setTimeStep(timeStep);
if (!qualifiers.isEmpty()) header.setQualifierIds(Clasz.strings.newArrayFrom(qualifiers));
if (creationDateText != null) {
try {
long creationTime = fastDateFormat.parseToMillis(creationDateText, creationTimeText);
header.setCreationTime(creationTime);
} catch (ParseException e) {
throw new Exception("Can not parse creation date/time " + creationDateText + ' ' + creationTimeText);
}
}
if (startTime != Long.MIN_VALUE && endTime != Long.MIN_VALUE) {
timeSeriesContentHandler.setEstimatedPeriod(new Period(startTime, endTime));
}
domainParameterIds = Clasz.strings.resizeArray(domainParameterIds, domainCount);
domainUnits = Clasz.strings.resizeArray(domainUnits, domainCount);
header.setDomainParameterIds(domainParameterIds);
header.setDomainUnits(domainUnits);
timeSeriesContentHandler.setNewTimeSeriesHeader(header);
reader.require(XMLStreamConstants.END_ELEMENT, null, "header");
reader.nextTag();
}
@SuppressWarnings("OverlyLongMethod")
private void parseEvent() throws Exception {
assert binaryInputStream == null;
reader.require(XMLStreamConstants.START_ELEMENT, null, "event");
timeSeriesContentHandler.clearFlagSourceColumns();
String dateText = null;
String timeText = null;
String startDateText = null;
String startTimeText = null;
String endDateText = null;
String endTimeText = null;
String valueText = null;
String valueSource = null;
String minValueText = null;
String maxValueText = null;
String flagText = null;
String flagSource = null;
String comment = null;
String user = null;
String limitText = null;
for (int i = 0, n = reader.getAttributeCount(); i < n; i++) {
String localName = reader.getAttributeLocalName(i);
String attributeValue = reader.getAttributeValue(i);
if (dateText == null && TextUtils.equals(localName, "date")) {
dateText = attributeValue;
} else if (timeText == null && TextUtils.equals(localName, "time")) {
timeText = attributeValue;
} else if (valueText == null && TextUtils.equals(localName, "value")) {
valueText = attributeValue;
} else if (valueSource == null && TextUtils.equals(localName, "valueSource")) {
valueSource = attributeValue;
} else if (flagText == null && TextUtils.equals(localName, "flag")) {
flagText = attributeValue;
} else if (flagSource == null && TextUtils.equals(localName, "flagSource")) {
flagSource = attributeValue;
} else if (comment == null && TextUtils.equals(localName, "comment")) {
comment = attributeValue;
} else if (user == null && TextUtils.equals(localName, "user")) {
user = attributeValue;
} else if (startDateText == null && TextUtils.equals(localName, "startDate")) {
startDateText = attributeValue;
} else if (startTimeText == null && TextUtils.equals(localName, "startTime")) {
startTimeText = attributeValue;
} else if (endDateText == null && TextUtils.equals(localName, "endDate")) {
endDateText = attributeValue;
} else if (endTimeText == null && TextUtils.equals(localName, "endTime")) {
endTimeText = attributeValue;
} else if (minValueText == null && TextUtils.equals(localName, "minValue")) {
minValueText = attributeValue;
} else if (maxValueText == null && TextUtils.equals(localName, "maxValue")) {
maxValueText = attributeValue;
} else if (maxValueText == null && TextUtils.equals(localName, "detection")) {
limitText = attributeValue;
} else if (reader.getAttributePrefix(i) != null && TextUtils.equals(reader.getAttributePrefix(i), "fs")) {
int index = timeSeriesContentHandler.addFlagSourceColumn(localName);
timeSeriesContentHandler.setColumnFlagSource(index, attributeValue);
} else {
if (lenient) continue;
throw new Exception("Unknown or duplicate attribute " + localName + " in event");
}
}
parseTime(dateText, timeText, startDateText, startTimeText, endDateText, endTimeText);
parseFlagsUserComment(flagText, flagSource, comment, user);
parseValue(valueText, valueSource, minValueText, maxValueText, limitText);
timeSeriesContentHandler.applyCurrentFields();
reader.require(XMLStreamConstants.END_ELEMENT, null, "event");
reader.nextTag();
}
private void parseValue(String valueText, String valueSource, String minValueText, String maxValueText, String limit) throws Exception {
if (domainCount == 0) {
try {
float value = valueText == null ? Float.NaN : TextUtils.parseFloat(valueText);
float minValue = minValueText == null ? value : TextUtils.parseFloat(minValueText);
float maxValue = maxValueText == null ? value : TextUtils.parseFloat(maxValueText);
// we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
if (value == missingValue) {
value = Float.NaN;
} else {
timeSeriesContentHandler.setValueResolution(TextUtils.getValueResolution(valueText, '.'));
}
timeSeriesContentHandler.setValueAndRange(value, minValue, maxValue);
timeSeriesContentHandler.setValueSource(TextUtils.equals(valueSource, "MAN") ? ValueSource.MANUAL : ValueSource.AUTOMATIC);
timeSeriesContentHandler.setOutOfDetectionRangeFlag(OutOfDetectionRangeFlag.get(getOutOfDetectionFlag(limit)));
} catch (NumberFormatException e) {
throw new Exception("Value should be a float " + valueText);
}
reader.nextTag();
return;
}
if (valueText != null)
throw new Exception("Attribute value not allowed when having domain parameters, use event element text instead");
int count = parseValueList();
if (count == 0) {
timeSeriesContentHandler.setValues(null);
return;
}
int expectedCount = 1;
for (int i = 0, n = domainCount; i < n; i++) {
float[] axis = axisValues[i];
if (axis == null)
throw new Exception("Domain axis values for domain parameter " + domainParameterIds[i] + " are missing");
expectedCount *= axis.length;
}
if (expectedCount != count)
throw new Exception("Length of value list for event does not matches the axes lengths");
if (axesDirty) {
timeSeriesContentHandler.setDomainAxesValueResolutions(axisValueResolutions);
timeSeriesContentHandler.setDomainAxesValues(axisValues);
axesDirty = false;
}
if (count != floatBuffer.length) floatBuffer = FloatArrayUtils.resize(floatBuffer, expectedCount);
timeSeriesContentHandler.setValueResolution(minValueResolution);
timeSeriesContentHandler.setValues(floatBuffer);
}
private byte getOutOfDetectionFlag(String text) {
if (text == null || text.isEmpty()) return TimeSeriesArray.INSIDE_DETECTION_RANGE;
char ch = text.charAt(0);
if (ch == '<') return TimeSeriesArray.BELOW_DETECTION_RANGE;
if (ch == '>') return TimeSeriesArray.ABOVE_DETECTION_RANGE;
if (ch == '~') return TimeSeriesArray.VARYING;
return TimeSeriesArray.INSIDE_DETECTION_RANGE;
}
private void parseFlagsUserComment(String flagText, String flagSource, String comment, String user) throws Exception {
if (flagText == null) {
timeSeriesContentHandler.setFlag(0);
} else {
try {
timeSeriesContentHandler.setFlag(TextUtils.parseInt(flagText));
} catch (NumberFormatException e) {
throw new Exception("Flag should be an integer " + flagText);
}
}
timeSeriesContentHandler.setComment(comment);
timeSeriesContentHandler.setUser(user);
timeSeriesContentHandler.setFlagSource(flagSource);
}
private void parseTime(String dateText, String timeText, String startDateText, String startTimeText, String endDateText, String endTimeText) throws Exception {
if (timeText == null)
throw new Exception("Attribute time is missing");
if (dateText == null)
throw new Exception("Attribute date is missing");
try {
long time = parseTime(dateText, timeText, null, null, Long.MIN_VALUE, lastTime, lastTimeAmbiguous);
lastTime = time;
lastTimeAmbiguous = timeAmbiguous;
long startTime = parseTime(startDateText, startTimeText, dateText, timeText, time, lastStartTime, lastStartTimeAmbiguous);
lastStartTime = startTime;
lastStartTimeAmbiguous = timeAmbiguous;
long endTime = parseTime(endDateText, endTimeText, dateText, dateText, time, lastEndTime, lastEndTimeAmbiguous);
lastEndTime = endTime;
lastEndTimeAmbiguous = timeAmbiguous;
timeSeriesContentHandler.setTimeAndRange(time, startTime, endTime);
} catch (ParseException e) {
throw new Exception("Can not parse " + dateText + ' ' + timeText);
}
}
private long parseTime(String dateText, String timeText, String defaultDateText, String defaultTimeText, long defaultTime, long lastTime, boolean lastTimeAmbiguous) throws ParseException {
if (dateText == null && timeText == null) return defaultTime;
if (dateText == null) dateText = defaultDateText;
if (timeText == null) timeText = defaultTimeText;
boolean useMillis = timeText.contains(".");
if (!fastDateFormat.getTimeZone().useDaylightTime()) return useMillis? fastDateFormatWithMillies.parseToMillis(dateText, timeText): fastDateFormat.parseToMillis(dateText, timeText);
long t1;
try {
t1 = useMillis?fastDateFormatWithMillies.parseToMillis(dateText, timeText): fastDateFormat.parseToMillis(dateText, timeText);
} catch (ParseException e) {
if (!timeText.equals("02:00:00")) throw e;
// see FEWS-17782 also accept 02:00:00 on daylight saving time switch
try {
return fastDateFormat.parseToMillis(dateText, "03:00:00");
} catch (ParseException e1) {
throw e;
}
}
Calendar calendar = useMillis? fastDateFormatWithMillies.getCalendar() : fastDateFormat.getCalendar();
calendar.setTimeInMillis(t1);
int timeOfDay = getTimeOfDay(calendar);
long t2 = getOtherTime(calendar, t1, timeOfDay);
timeAmbiguous = t1 != t2;
long minTime = Math.min(t1, t2);
long maxTime = Math.max(t1, t2);
long res = minTime > lastTime ? minTime : maxTime;
if (!lastTimeAmbiguous) return res;
// see FEWS-17782 also accept two times 02:00:00 instead of two times 01:00:00 on daylight saving time switch
if (timeStepMillis != 0L && lastTime + timeStepMillis != res) return lastTime + timeStepMillis;
if (timeStep != null && timeStep.isRegular() && timeStep.nextTime(lastTime) != res) return timeStep.nextTime(lastTime);
return res;
}
private static int getTimeOfDay(Calendar calendar) {
return (int) (calendar.get(Calendar.HOUR_OF_DAY) * TimeUnit.HOUR_MILLIS + calendar.get(Calendar.MINUTE) * TimeUnit.MINUTE_MILLIS + calendar.get(Calendar.SECOND) * TimeUnit.SECOND_MILLIS + calendar.get(Calendar.MILLISECOND));
}
private static long getOtherTime(Calendar calendar, long time, int timeOfDay) {
try {
calendar.setTimeInMillis(time + TimeUnit.HOUR_MILLIS);
if (getTimeOfDay(calendar) == timeOfDay) return time + TimeUnit.HOUR_MILLIS;
calendar.setTimeInMillis(time - TimeUnit.HOUR_MILLIS);
if (getTimeOfDay(calendar) == timeOfDay) return time - TimeUnit.HOUR_MILLIS;
return time;
} finally {
calendar.setTimeInMillis(time);
}
}
private long parseTime() throws Exception {
String dateText = reader.getAttributeValue(null, "date");
if (dateText == null) {
throw new Exception("Attribute " + currentHeaderElement +
"-date is missing");
}
String timeText = reader.getAttributeValue(null, "time");
if (timeText == null) {
throw new Exception("Attribute " + currentHeaderElement +
"-time is missing");
}
boolean useMillis = timeText.contains(".");
long time;
try {
time = useMillis? fastDateFormatWithMillies.parseToMillis(dateText, timeText) : fastDateFormat.parseToMillis(dateText, timeText);
} catch (ParseException e) {
throw new Exception("Not a valid data time for "
+ currentHeaderElement + ' ' + dateText + ' ' + timeText, e);
}
reader.nextTag();
return time;
}
private void parseTimeStep() throws Exception {
String times = reader.getAttributeValue(null, "times");
if (times != null) {
timeStep = PiCastorUtils.createTimesOfDayTimeStep(times, getTimeZone());
reader.nextTag();
return;
}
String unit = reader.getAttributeValue(null, "unit");
TimeUnit tu = unit == null ? null : TimeUnit.get(unit);
if (tu != null) {
String multiplierText = reader.getAttributeValue(null, "multiplier");
int multiplier;
if (multiplierText == null) {
multiplier = 1;
} else {
try {
multiplier = Integer.parseInt(multiplierText);
} catch (NumberFormatException e) {
throw new Exception(ExceptionUtils.getMessage(e), e);
}
if (multiplier == 0) {
throw new Exception("Multiplier is 0");
}
}
String dividerText = reader.getAttributeValue(null, "divider");
int divider;
if (dividerText == null) {
divider = 1;
} else {
try {
divider = Integer.parseInt(dividerText);
} catch (NumberFormatException e) {
throw new Exception(ExceptionUtils.getMessage(e), e);
}
if (divider == 0) {
throw new Exception("divider is 0");
}
}
reader.nextTag();
timeStepMillis = tu.getMillis() * multiplier / divider;
timeStep = null;
} else {
reader.nextTag();
timeStepMillis = 0;
timeStep = IrregularTimeStep.INSTANCE;
}
}
private void initHeader() {
header.clear();
header.setFileDescription(virtualFileName);
currentHeaderElement = null;
timeStep = null;
timeStepMillis = 0;
startTime = Long.MIN_VALUE;
endTime = Long.MIN_VALUE;
missingValue = Float.NaN;
creationDateText = null;
creationTimeText = "00:00:00";
qualifiers.clear();
lat = Double.NaN;
lon = Double.NaN;
z = Double.NaN;
}
private void readValuesFromBinFile() throws Exception {
TimeStep timeStep = header.getTimeStep();
if (!timeStep.isRegular()) {
throw new Exception("Only equidistant time step supported when pi events are stored in bin file instead of xml");
}
boolean equidistantMillis = timeStep.isEquidistantMillis();
long stepMillis = equidistantMillis ? timeStep.getStepMillis() : Long.MIN_VALUE;
for (long time = startTime; time <= endTime;) {
timeSeriesContentHandler.setTime(time);
if (bufferPos == bufferCount) fillBuffer();
float value = floatBuffer[bufferPos++];
// we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
if (value == missingValue) value = Float.NaN;
timeSeriesContentHandler.setValue(value);
timeSeriesContentHandler.applyCurrentFields();
if (equidistantMillis) {
time += stepMillis;
continue;
}
time = timeStep.nextTime(time);
}
}
private void fillBuffer() throws IOException {
int byteBufferCount = 0;
while (byteBufferCount % NumberType.FLOAT_SIZE != 0 || byteBufferCount == 0) {
int count = binaryInputStream.read(byteBuffer, byteBufferCount, BUFFER_SIZE * NumberType.FLOAT_SIZE - byteBufferCount);
assert count != 0; // see read javadoc
if (count == -1) throw new EOFException("Bin file is too short");
byteBufferCount += count;
}
bufferCount = byteBufferCount / NumberType.FLOAT_SIZE;
BinaryUtils.copy(byteBuffer, 0, byteBufferCount, floatBuffer, 0, bufferCount, ByteOrder.LITTLE_ENDIAN);
bufferPos = 0;
}
private void initiateTimeStep() {
if (timeStep != null) {
assert timeStepMillis == 0;
return;
}
if (timeStepMillis == 0){
//no timestep in header. Fix for backward compatibility
timeStep = IrregularTimeStep.INSTANCE;
return;
}
if (timeStepMillis >= TimeUnit.HOUR_MILLIS && getTimeZone().useDaylightTime()) {
timeStep = IrregularTimeStep.INSTANCE;
return;
}
long startTime = this.startTime == Long.MIN_VALUE ? 0L : this.startTime;
if (timeStepMillis % TimeUnit.SECOND_MILLIS != 0) {
timeStep = RelativeEquidistantTimeStep.getInstance(timeStepMillis, startTime);
return;
}
long timeZoneOffsetMillis = -startTime % timeStepMillis;
if (timeZoneOffsetMillis % TimeUnit.MINUTE_MILLIS != 0) {
timeStep = RelativeEquidistantTimeStep.getInstance(timeStepMillis, startTime);
return;
}
timeStep = SimpleEquidistantTimeStep.getInstance(timeStepMillis, timeZoneOffsetMillis);
}
@SuppressWarnings({"OverlyLongMethod"})
private void parseHeaderElement() throws Exception {
switch (currentHeaderElement) {
case type:
header.setParameterType(parseType(reader.getElementText()));
break;
case moduleInstanceId:
header.setModuleInstanceId(reader.getElementText());
break;
case locationId:
// see FEWS-9858, when there is no location id the time series are assigned to all locations
// this is a flaw in the pi_timeSeries.xsd, the location element is required but is allowed empty strings
header.setLocationId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none"));
break;
case parameterId:
// see FEWS-9858, when there is no parameter id the time series are assigned to all locations
// this is a flaw in the pi_timeSeries.xsd, the location element is required but is allowed empty strings
header.setParameterId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none"));
break;
case qualifierId:
qualifiers.add(reader.getElementText());
break;
case ensembleId:
header.setEnsembleId(reader.getElementText());
break;
case ensembleMemberIndex:
header.setEnsembleMemberIndex(parseEnsembleMemberIndex(reader.getElementText()));
break;
case ensembleMemberId:
header.setEnsembleMemberId(reader.getElementText());
break;
case timeStep:
parseTimeStep();
break;
case startDate:
startTime = parseTime();
break;
case endDate:
endTime = parseTime();
break;
case forecastDate:
header.setForecastTime(parseTime());
break;
case approvedDate:
header.setApprovedTime(parseTime());
break;
case missVal:
missingValue = parseString(reader.getElementText());
break;
case longName:
header.setLongName(reader.getElementText());
break;
case stationName:
header.setLocationName(reader.getElementText());
break;
case units:
header.setUnit(reader.getElementText());
break;
case domainAxis:
parseDomainAxis();
break;
case sourceOrganisation:
header.setSourceOrganisation(reader.getElementText());
break;
case sourceSystem:
header.setSourceSystem(reader.getElementText());
break;
case fileDescription:
header.setFileDescription(reader.getElementText());
break;
case creationDate:
creationDateText = reader.getElementText();
break;
case creationTime:
creationTimeText = reader.getElementText();
break;
case region:
header.setRegion(reader.getElementText());
break;
case thresholds:
parseThresholds();
break;
case lat:
lat = parseString(reader.getElementText());
break;
case lon:
lon = parseString(reader.getElementText());
break;
case x:
reader.getElementText();
break;
case y:
reader.getElementText();
break;
case z:
z = parseString(reader.getElementText());
break;
case firstValueTime:
reader.getElementText();
break;
case lastValueTime:
reader.getElementText();
break;
case maxValue:
reader.getElementText();
break;
case minValue:
reader.getElementText();
break;
case valueCount:
reader.getElementText();
break;
case maxWarningLevelName:
reader.getElementText();
break;
}
reader.require(XMLStreamConstants.END_ELEMENT, null, currentHeaderElement.name());
reader.nextTag();
}
private void parseDomainAxis() throws Exception {
String parameterId = reader.getAttributeValue(null, "parameterId");
if (parameterId == null)
throw new Exception("Attribute parameterId for domainUnits is missing");
domainParameterIds = Clasz.strings.ensureCapacity(domainParameterIds, domainCount + 1);
domainUnits = Clasz.strings.ensureCapacity(domainUnits, domainCount + 1);
domainParameterIds[domainCount] = parameterId;
domainUnits[domainCount] = reader.getAttributeValue(null, "units");
domainCount++;
reader.nextTag();
}
private void parseThresholds() throws XMLStreamException {
reader.nextTag();
ArrayList<DefaultTimeSeriesHeader.DefaultThreshold> thresholds = new ArrayList<>();
do {
if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {
String id = reader.getAttributeValue(null, "id");
String name = reader.getAttributeValue(null, "name");
float value = TextUtils.parseFloat(reader.getAttributeValue(null, "value"));
String groupId = reader.getAttributeValue(null, "groupId");
String groupName = reader.getAttributeValue(null, "groupName");
String[] groupIds = groupId == null ? Clasz.strings.emptyArray() : new String[] {groupId};
String[] groupNames = groupName == null ? Clasz.strings.emptyArray() : new String[] {groupName};
thresholds.add(new DefaultTimeSeriesHeader.DefaultThreshold(id, name, value, groupIds, groupNames));
}
reader.nextTag();
} while (!reader.getLocalName().equals(currentHeaderElement.name()));
header.setHighLevelThresholds(TimeSeriesHeader.Threshold.clasz.newArrayFrom(thresholds));
}
private void parseProperties() throws XMLStreamException {
reader.require(XMLStreamConstants.START_ELEMENT, null, "properties");
reader.nextTag();
propertyBuilder.clear();
while (!TextUtils.equals(reader.getLocalName(), "properties")){
if (reader.getEventType() != XMLStreamConstants.START_ELEMENT) {
// eg <int key="a" value=12><int>
reader.nextTag();
continue;
}
String key = reader.getAttributeValue(null, "key");
String value = reader.getAttributeValue(null, "value");
String date = reader.getAttributeValue(null, "date");
String time = reader.getAttributeValue(null, "time");
switch (reader.getLocalName()) {
case "string":
propertyBuilder.addString(key, value);
break;
case "int":
propertyBuilder.addInt(key, TextUtils.parseInt(value));
break;
case "float":
propertyBuilder.addFloat(key, TextUtils.parseFloat(value));
break;
case "double":
propertyBuilder.addDouble(key, TextUtils.parseDouble(value));
break;
case "bool":
propertyBuilder.addBoolean(key, Boolean.parseBoolean(value));
break;
case "dateTime":
try {
if (time.contains(".")) {
propertyBuilder.addDateTime(key, fastDateFormatWithMillies.parseToMillis(date, time));
} else {
propertyBuilder.addDateTime(key, fastDateFormat.parseToMillis(date, time));
}
break;
} catch (ParseException e) {
throw new XMLStreamException("Invalid date time "+ date + ' ' + time);
}
default:
throw new XMLStreamException("Invalid property type " + reader.getLocalName());
}
reader.nextTag();
}
timeSeriesContentHandler.setProperties(propertyBuilder.build());
reader.require(XMLStreamConstants.END_ELEMENT, null, "properties");
reader.nextTag();
}
private static float parseString(String gotString) throws Exception {
// <element name="missVal" type="double" default="NaN">
// when default is used in schema for element the consequence is that empty strings are allowed
if (gotString.isEmpty()) return Float.NaN;
try {
return TextUtils.parseFloat(gotString);
} catch (NumberFormatException e) {
throw new Exception(ExceptionUtils.getMessage(e), e);
}
}
private static int parseEnsembleMemberIndex(String gotString) throws Exception {
int index = Integer.parseInt(gotString);
if (index < 0) {
throw new Exception("Negative ensemble member index not allowed " + gotString);
}
return index;
}
private static ParameterType parseType(String gotString) throws Exception {
ParameterType type = ParameterType.get(gotString);
if (type == null) {
throw new Exception("Type in header should be instantaneous or accumulative and not " + gotString);
}
return type;
}
private void detectHeaderElement() throws Exception {
if (reader.getEventType() != XMLStreamConstants.START_ELEMENT)
throw new Exception("header element expected");
String localName = reader.getLocalName();
HeaderElement element;
try {
element = Enum.valueOf(HeaderElement.class, localName);
} catch (Exception e) {
throw new Exception("Unknown header element: " + localName);
}
if (currentHeaderElement == element && currentHeaderElement.isMultipleAllowed()) return;
if (currentHeaderElement != null && element.ordinal() < currentHeaderElement.ordinal()) {
throw new Exception("Header elements in wrong order: " + localName);
}
if (currentHeaderElement == HeaderElement.ensembleMemberIndex && element == HeaderElement.ensembleMemberId) {
throw new Exception("Duplicate header element, both ensembleMemberIndex and ensembleMemberId in header");
}
if (currentHeaderElement == element) {
throw new Exception("Duplicate header element: " + localName);
}
if (reader.getAttributeCount() > 0 && !element.hasAttributes()) {
throw new Exception("Attributes are not allowed for header element " + localName);
}
int nextOrdinal = currentHeaderElement == null ? 0 : currentHeaderElement.ordinal() + 1;
// order is correct and no duplicate so currentHeaderElement can not be last header element
assert nextOrdinal < HEADER_ELEMENTS.length;
HeaderElement nextHeaderElement = HEADER_ELEMENTS[nextOrdinal];
if (nextHeaderElement.isRequired() && nextHeaderElement != element) {
throw new Exception("Required header item missing: " + nextHeaderElement);
}
currentHeaderElement = element;
}
public TimeZone getTimeZone() {
//time zone for both fastDateFormat and fastDateFormatWithMillies should be the same.
return fastDateFormat.getTimeZone();
}
@SuppressWarnings("UnusedDeclaration")
public PiTimeSeriesSerializer.EventDestination getEventDestination() {
return eventDestination;
}
@SuppressWarnings("UnusedDeclaration")
public float getMissingValue() {
return missingValue;
}
}
Overview
Content Tools
