/* ================================================================ * Delft FEWS * ================================================================ * * Project Info: http://www.wldelft.nl/soft/fews/index.html * Project Lead: Karel Heynert (karel.heynert@wldelft.nl) * * (C) Copyright 2003, by WL | Delft Hydraulics * P.O. Box 177 * 2600 MH Delft * The Netherlands * http://www.wldelft.nl * * DELFT-FEWS is a sophisticated collection of modules designed * for building a FEWS customised to the specific requirements * of individual agencies. An open modelling approach allows users * to add their own modules in an efficient way. * * ---------------------------------------------------------------- * PiTimeSeriesParser.java * ---------------------------------------------------------------- * (C) Copyright 2003, by WL | Delft Hydraulics * * Original Author: Erik de Rooij * Original Author: Onno van den Akker */ package nl.wldelft.fews.pi; import nl.wldelft.util.BinaryUtils; import nl.wldelft.util.CharArrayUtils; import nl.wldelft.util.Clasz; import nl.wldelft.util.ExceptionUtils; import nl.wldelft.util.FastDateFormat; import nl.wldelft.util.FileUtils; import nl.wldelft.util.FloatArrayUtils; import nl.wldelft.util.NumberType; import nl.wldelft.util.Period; import nl.wldelft.util.Properties; import nl.wldelft.util.StringArrayUtils; import nl.wldelft.util.TextUtils; import nl.wldelft.util.TimeUnit; import nl.wldelft.util.TimeZoneUtils; import nl.wldelft.util.geodatum.GeoDatum; import nl.wldelft.util.io.VirtualInputDir; import nl.wldelft.util.io.VirtualInputDirConsumer; import nl.wldelft.util.io.XmlParser; import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader; import nl.wldelft.util.timeseries.IrregularTimeStep; import nl.wldelft.util.timeseries.OutOfDetectionRangeFlag; import nl.wldelft.util.timeseries.ParameterType; import nl.wldelft.util.timeseries.RelativeEquidistantTimeStep; import nl.wldelft.util.timeseries.SimpleEquidistantTimeStep; import nl.wldelft.util.timeseries.TimeSeriesArray; import nl.wldelft.util.timeseries.TimeSeriesContentHandler; import nl.wldelft.util.timeseries.TimeSeriesHeader; import nl.wldelft.util.timeseries.TimeStep; import nl.wldelft.util.timeseries.ValueSource; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Locale; import java.util.TimeZone; public class PiTimeSeriesParser implements XmlParser<TimeSeriesContentHandler>, VirtualInputDirConsumer { private static final int BUFFER_SIZE = 2048; private final Properties.Builder propertyBuilder = new Properties.Builder(); private char[] charBuffer = null; private float minValueResolution = Float.NaN; private float[][] axisValues = null; private float[] axisValueResolutions = Clasz.floats.emptyArray(); private boolean axesDirty = false; private String[] domainParameterIds = Clasz.strings.emptyArray(); private String[] domainUnits = Clasz.strings.emptyArray(); private int domainCount = 0; @Override public void setVirtualInputDir(VirtualInputDir virtualInputDir) { this.virtualInputDir = virtualInputDir; } private enum HeaderElement { type(F.R), moduleInstanceId, locationId(F.R), parameterId(F.R), qualifierId(F.M), ensembleId, ensembleMemberIndex, ensembleMemberId, timeStep(F.R | F.A), startDate(F.R | F.A), endDate(F.R | F.A), forecastDate(F.A), approvedDate(F.A), missVal, longName, stationName, lat, lon, x, y, z, units, domainAxis(F.M | F.A), sourceOrganisation, sourceSystem, fileDescription, creationDate, creationTime, region, thresholds, firstValueTime(F.A), lastValueTime(F.A), maxValue, minValue, valueCount, maxWarningLevelName; interface F { int A = 1 << 0; // attributes int R = 1 << 1; // required; int M = 1 << 2; // multiple; } private final int flags; HeaderElement() { this.flags = 0; } HeaderElement(int flags) { this.flags = flags; } public boolean isRequired() { return (flags & F.R) != 0; } public boolean hasAttributes() { return (flags & F.A) != 0; } public boolean isMultipleAllowed() { return (flags & F.M) != 0; } } // fastDateFormat is used to keep track of last time zone and lenient private FastDateFormat fastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd", "HH:mm:ss", TimeZoneUtils.GMT, Locale.US, null); private FastDateFormat fastDateFormatWithMillies = FastDateFormat.getInstance("yyyy-MM-dd", "HH:mm:ss.SSS", TimeZoneUtils.GMT, Locale.US, null); private HeaderElement currentHeaderElement = null; private static final HeaderElement[] HEADER_ELEMENTS = HeaderElement.class.getEnumConstants(); private PiTimeSeriesHeader header = new PiTimeSeriesHeader(); private List<String> qualifiers = new ArrayList<>(); private long timeStepMillis = 0; private TimeStep timeStep = null; private long startTime = Long.MIN_VALUE; private long endTime = Long.MIN_VALUE; private float missingValue = Float.NaN; private String creationDateText = null; private String creationTimeText = null; private long lastTime = Long.MIN_VALUE; private long lastStartTime = Long.MIN_VALUE; private long lastEndTime = Long.MIN_VALUE; private boolean timeAmbiguous = false; private boolean lastTimeAmbiguous = false; private boolean lastStartTimeAmbiguous = false; private boolean lastEndTimeAmbiguous = false; private TimeSeriesContentHandler timeSeriesContentHandler = null; private PiTimeSeriesSerializer.EventDestination eventDestination = null; /** * For performance reasons the pi time series format allows that the values are stored in * a separate bin file instead of embedded in the xml file. * The bin file should have same name as the xml file except the extension equals bin * In this case all time series should be equidistant. */ private VirtualInputDir virtualInputDir = VirtualInputDir.NONE; private InputStream binaryInputStream = null; private byte[] byteBuffer = null; private float[] floatBuffer = null; private int bufferPos = 0; private int bufferCount = 0; private XMLStreamReader reader = null; private String virtualFileName = null; private static boolean lenient = false; private double lat = Double.NaN; private double lon = Double.NaN; private double z = Double.NaN; /** * For backwards compatibility. Earlier versions of the PiTimeSeriesParser were tolerant about the date/time format * and the case insensitive for header element names. * This parser should not accept files that are not valid according to pi_timeseries.xsd * When old adapters are not working you can UseLenientPiTimeSeriesParser temporary till the adapter is fixed * * @param lenient */ public static void setLenient(boolean lenient) { PiTimeSeriesParser.lenient = lenient; } public static boolean isLenient() { return lenient; } public PiTimeSeriesParser() { fastDateFormat.setLenient(lenient); fastDateFormatWithMillies.setLenient(lenient); } @Override public void parse(XMLStreamReader reader, String virtualFileName, TimeSeriesContentHandler timeSeriesContentHandler) throws Exception { this.reader = reader; this.virtualFileName = virtualFileName; this.timeSeriesContentHandler = timeSeriesContentHandler; String virtualBinFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "bin"); // time zone can be overruled by one or more time zone elements in the pi file this.fastDateFormat.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone()); this.fastDateFormatWithMillies.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone()); if (!virtualInputDir.exists(virtualBinFileName)) { eventDestination = PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED; parse(); return; } eventDestination = PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE; binaryInputStream = virtualInputDir.getInputStream(virtualBinFileName); try { if (byteBuffer == null) { byteBuffer = new byte[BUFFER_SIZE * NumberType.FLOAT_SIZE]; floatBuffer = new float[BUFFER_SIZE]; } parse(); boolean eof = bufferPos == bufferCount && binaryInputStream.read() == -1; if (!eof) throw new IOException("More values available in bin file than expected based on time step and start and end time\n" + FileUtils.getPathWithOtherExtension(virtualFileName, "bin")); } finally { bufferPos = 0; bufferCount = 0; binaryInputStream.close(); binaryInputStream = null; } } private void parse() throws Exception { reader.require(XMLStreamConstants.START_DOCUMENT, null, null); reader.nextTag(); reader.require(XMLStreamConstants.START_ELEMENT, null, "TimeSeries"); reader.nextTag(); while (reader.getEventType() != XMLStreamConstants.END_ELEMENT) { TimeZone timeZone = PiParserUtils.parseTimeZone(reader); //returns null if the timeZone is not present in the file if (timeZone != null) { this.fastDateFormat.setTimeZone(timeZone); this.fastDateFormatWithMillies.setTimeZone(timeZone); } if (noTimeSeries()) continue; readTimeSeries(); } reader.require(XMLStreamConstants.END_ELEMENT, null, "TimeSeries"); reader.next(); reader.require(XMLStreamConstants.END_DOCUMENT, null, null); } private boolean noTimeSeries() { return reader.getEventType() == XMLStreamConstants.END_ELEMENT && reader.getLocalName().equals("TimeSeries"); } private void readTimeSeries() throws Exception { reader.require(XMLStreamConstants.START_ELEMENT, null, "series"); reader.nextTag(); parseHeader(); lastTime = Long.MIN_VALUE; timeAmbiguous = false; lastTimeAmbiguous = false; lastStartTimeAmbiguous = false; lastEndTimeAmbiguous = false; timeSeriesContentHandler.setProperties(Properties.NONE); if (eventDestination == PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED) { while (reader.getEventType() == XMLStreamConstants.START_ELEMENT) { String localName = reader.getLocalName(); if (TextUtils.equals(localName, "event")) { parseEvent(); } else if (TextUtils.equals(localName, "domainAxisValues")) { parseDomainAxisValues(); } else if (TextUtils.equals(localName, "properties")) { parseProperties(); } else { break; } } } else { assert eventDestination == PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE; readValuesFromBinFile(); } if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) { // skip comment reader.require(XMLStreamConstants.START_ELEMENT, null, "comment"); reader.getElementText(); reader.nextTag(); } reader.require(XMLStreamConstants.END_ELEMENT, null, "series"); reader.nextTag(); } private void parseDomainAxisValues() throws Exception { String domainParameterId = reader.getAttributeValue(null, "parameterId"); if (domainParameterId == null) throw new Exception("Attribute parameterId for domainAxisValues is missing"); int index = StringArrayUtils.indexOf(domainParameterIds, 0, domainCount, domainParameterId); if (index == -1) throw new Exception("parameterId " + domainParameterId + " for domainAxisValues not defined in header"); if (axisValues == null) axisValues = new float[domainCount][]; if (axisValueResolutions.length < domainCount) axisValueResolutions = new float[domainCount]; int count = parseValueList(); reader.require(XMLStreamConstants.END_ELEMENT, null, "domainAxisValues"); reader.nextTag(); axisValues[index] = Clasz.floats.copyOfArrayRange(floatBuffer, 0, count); axisValueResolutions[index] = minValueResolution; axesDirty = true; } private int parseValueList() throws Exception { if (charBuffer == null) charBuffer = new char[100]; char[] charBuffer = this.charBuffer; if (floatBuffer == null) floatBuffer = new float[100]; minValueResolution = Float.POSITIVE_INFINITY; int i = 0; int length = 0; int j = 0; while (reader.next() == XMLStreamConstants.CHARACTERS) { length -= i; assert length >= 0; // shift the remaining part of the buffer to the start if (length > 0) CharArrayUtils.arraycopy(charBuffer, i, charBuffer, 0, length); int textLength = reader.getTextLength(); if (charBuffer.length < length + textLength) { charBuffer = Clasz.chars.copyOfArrayRange(charBuffer, 0, length, length + textLength); this.charBuffer = charBuffer; } int n = reader.getTextCharacters(0, charBuffer, length, textLength); assert n == textLength; length += textLength; i = 0; for (; ; ) { int start = CharArrayUtils.indexOfNonWhitespace(charBuffer, i, length - i); if (start == -1) { i = length; break; } int end = CharArrayUtils.indexOfWhitespace(charBuffer, start, length - start); if (end == -1) { i = start; break; } i = end; parseValue(charBuffer, j++, start, end - start + 1); } } if (i != length) parseValue(charBuffer, j++, i, length - i); return j; } private void parseValue(char[] charBuffer, int pos, int start, int length) { float value = (float) TextUtils.parseDouble(charBuffer, start, length, '.'); if (value == missingValue) value = Float.NaN; float valueResolution = TextUtils.getValueResolution(charBuffer, start, length, '.'); if (valueResolution < minValueResolution) minValueResolution = valueResolution; if (floatBuffer.length == pos) floatBuffer = Clasz.floats.ensureCapacity(floatBuffer, pos + 1); floatBuffer[pos] = value; } private void parseHeader() throws Exception { domainCount = 0; Arrays.fill(domainParameterIds, null); Arrays.fill(domainUnits, null); if (axisValues != null) Arrays.fill(axisValues, null); reader.require(XMLStreamConstants.START_ELEMENT, null, "header"); if (reader.getAttributeCount() > 0) { throw new Exception("Attributes are not allowed for header element "); } reader.nextTag(); initHeader(); do { detectHeaderElement(); parseHeaderElement(); } while (reader.getEventType() != XMLStreamConstants.END_ELEMENT); if (header.getForecastTime() == Long.MIN_VALUE) header.setForecastTime(startTime); if (!Double.isNaN(lat)) header.setGeometry(GeoDatum.WGS_1984.createLatLongZ(lat, lon, z)); initiateTimeStep(); header.setTimeStep(timeStep); if (!qualifiers.isEmpty()) header.setQualifierIds(Clasz.strings.newArrayFrom(qualifiers)); if (creationDateText != null) { try { long creationTime = fastDateFormat.parseToMillis(creationDateText, creationTimeText); header.setCreationTime(creationTime); } catch (ParseException e) { throw new Exception("Can not parse creation date/time " + creationDateText + ' ' + creationTimeText); } } if (startTime != Long.MIN_VALUE && endTime != Long.MIN_VALUE) { timeSeriesContentHandler.setEstimatedPeriod(new Period(startTime, endTime)); } domainParameterIds = Clasz.strings.resizeArray(domainParameterIds, domainCount); domainUnits = Clasz.strings.resizeArray(domainUnits, domainCount); header.setDomainParameterIds(domainParameterIds); header.setDomainUnits(domainUnits); timeSeriesContentHandler.setNewTimeSeriesHeader(header); reader.require(XMLStreamConstants.END_ELEMENT, null, "header"); reader.nextTag(); } @SuppressWarnings("OverlyLongMethod") private void parseEvent() throws Exception { assert binaryInputStream == null; reader.require(XMLStreamConstants.START_ELEMENT, null, "event"); timeSeriesContentHandler.clearFlagSourceColumns(); String dateText = null; String timeText = null; String startDateText = null; String startTimeText = null; String endDateText = null; String endTimeText = null; String valueText = null; String valueSource = null; String minValueText = null; String maxValueText = null; String flagText = null; String flagSource = null; String comment = null; String user = null; String limitText = null; for (int i = 0, n = reader.getAttributeCount(); i < n; i++) { String localName = reader.getAttributeLocalName(i); String attributeValue = reader.getAttributeValue(i); if (dateText == null && TextUtils.equals(localName, "date")) { dateText = attributeValue; } else if (timeText == null && TextUtils.equals(localName, "time")) { timeText = attributeValue; } else if (valueText == null && TextUtils.equals(localName, "value")) { valueText = attributeValue; } else if (valueSource == null && TextUtils.equals(localName, "valueSource")) { valueSource = attributeValue; } else if (flagText == null && TextUtils.equals(localName, "flag")) { flagText = attributeValue; } else if (flagSource == null && TextUtils.equals(localName, "flagSource")) { flagSource = attributeValue; } else if (comment == null && TextUtils.equals(localName, "comment")) { comment = attributeValue; } else if (user == null && TextUtils.equals(localName, "user")) { user = attributeValue; } else if (startDateText == null && TextUtils.equals(localName, "startDate")) { startDateText = attributeValue; } else if (startTimeText == null && TextUtils.equals(localName, "startTime")) { startTimeText = attributeValue; } else if (endDateText == null && TextUtils.equals(localName, "endDate")) { endDateText = attributeValue; } else if (endTimeText == null && TextUtils.equals(localName, "endTime")) { endTimeText = attributeValue; } else if (minValueText == null && TextUtils.equals(localName, "minValue")) { minValueText = attributeValue; } else if (maxValueText == null && TextUtils.equals(localName, "maxValue")) { maxValueText = attributeValue; } else if (maxValueText == null && TextUtils.equals(localName, "detection")) { limitText = attributeValue; } else if (reader.getAttributePrefix(i) != null && TextUtils.equals(reader.getAttributePrefix(i), "fs")) { int index = timeSeriesContentHandler.addFlagSourceColumn(localName); timeSeriesContentHandler.setColumnFlagSource(index, attributeValue); } else { if (lenient) continue; throw new Exception("Unknown or duplicate attribute " + localName + " in event"); } } parseTime(dateText, timeText, startDateText, startTimeText, endDateText, endTimeText); parseFlagsUserComment(flagText, flagSource, comment, user); parseValue(valueText, valueSource, minValueText, maxValueText, limitText); timeSeriesContentHandler.applyCurrentFields(); reader.require(XMLStreamConstants.END_ELEMENT, null, "event"); reader.nextTag(); } private void parseValue(String valueText, String valueSource, String minValueText, String maxValueText, String limit) throws Exception { if (domainCount == 0) { try { float value = valueText == null ? Float.NaN : TextUtils.parseFloat(valueText); float minValue = minValueText == null ? value : TextUtils.parseFloat(minValueText); float maxValue = maxValueText == null ? value : TextUtils.parseFloat(maxValueText); // we can not use the automatic missing value detection of the content handler because the missing value is different for each time series if (value == missingValue) { value = Float.NaN; } else { timeSeriesContentHandler.setValueResolution(TextUtils.getValueResolution(valueText, '.')); } timeSeriesContentHandler.setValueAndRange(value, minValue, maxValue); timeSeriesContentHandler.setValueSource(TextUtils.equals(valueSource, "MAN") ? ValueSource.MANUAL : ValueSource.AUTOMATIC); timeSeriesContentHandler.setOutOfDetectionRangeFlag(OutOfDetectionRangeFlag.get(getOutOfDetectionFlag(limit))); } catch (NumberFormatException e) { throw new Exception("Value should be a float " + valueText); } reader.nextTag(); return; } if (valueText != null) throw new Exception("Attribute value not allowed when having domain parameters, use event element text instead"); int count = parseValueList(); if (count == 0) { timeSeriesContentHandler.setValues(null); return; } int expectedCount = 1; for (int i = 0, n = domainCount; i < n; i++) { float[] axis = axisValues[i]; if (axis == null) throw new Exception("Domain axis values for domain parameter " + domainParameterIds[i] + " are missing"); expectedCount *= axis.length; } if (expectedCount != count) throw new Exception("Length of value list for event does not matches the axes lengths"); if (axesDirty) { timeSeriesContentHandler.setDomainAxesValueResolutions(axisValueResolutions); timeSeriesContentHandler.setDomainAxesValues(axisValues); axesDirty = false; } if (count != floatBuffer.length) floatBuffer = FloatArrayUtils.resize(floatBuffer, expectedCount); timeSeriesContentHandler.setValueResolution(minValueResolution); timeSeriesContentHandler.setValues(floatBuffer); } private byte getOutOfDetectionFlag(String text) { if (text == null || text.isEmpty()) return TimeSeriesArray.INSIDE_DETECTION_RANGE; char ch = text.charAt(0); if (ch == '<') return TimeSeriesArray.BELOW_DETECTION_RANGE; if (ch == '>') return TimeSeriesArray.ABOVE_DETECTION_RANGE; if (ch == '~') return TimeSeriesArray.VARYING; return TimeSeriesArray.INSIDE_DETECTION_RANGE; } private void parseFlagsUserComment(String flagText, String flagSource, String comment, String user) throws Exception { if (flagText == null) { timeSeriesContentHandler.setFlag(0); } else { try { timeSeriesContentHandler.setFlag(TextUtils.parseInt(flagText)); } catch (NumberFormatException e) { throw new Exception("Flag should be an integer " + flagText); } } timeSeriesContentHandler.setComment(comment); timeSeriesContentHandler.setUser(user); timeSeriesContentHandler.setFlagSource(flagSource); } private void parseTime(String dateText, String timeText, String startDateText, String startTimeText, String endDateText, String endTimeText) throws Exception { if (timeText == null) throw new Exception("Attribute time is missing"); if (dateText == null) throw new Exception("Attribute date is missing"); try { long time = parseTime(dateText, timeText, null, null, Long.MIN_VALUE, lastTime, lastTimeAmbiguous); lastTime = time; lastTimeAmbiguous = timeAmbiguous; long startTime = parseTime(startDateText, startTimeText, dateText, timeText, time, lastStartTime, lastStartTimeAmbiguous); lastStartTime = startTime; lastStartTimeAmbiguous = timeAmbiguous; long endTime = parseTime(endDateText, endTimeText, dateText, dateText, time, lastEndTime, lastEndTimeAmbiguous); lastEndTime = endTime; lastEndTimeAmbiguous = timeAmbiguous; timeSeriesContentHandler.setTimeAndRange(time, startTime, endTime); } catch (ParseException e) { throw new Exception("Can not parse " + dateText + ' ' + timeText); } } private long parseTime(String dateText, String timeText, String defaultDateText, String defaultTimeText, long defaultTime, long lastTime, boolean lastTimeAmbiguous) throws ParseException { if (dateText == null && timeText == null) return defaultTime; if (dateText == null) dateText = defaultDateText; if (timeText == null) timeText = defaultTimeText; boolean useMillis = timeText.contains("."); if (!fastDateFormat.getTimeZone().useDaylightTime()) return useMillis? fastDateFormatWithMillies.parseToMillis(dateText, timeText): fastDateFormat.parseToMillis(dateText, timeText); long t1; try { t1 = useMillis?fastDateFormatWithMillies.parseToMillis(dateText, timeText): fastDateFormat.parseToMillis(dateText, timeText); } catch (ParseException e) { if (!timeText.equals("02:00:00")) throw e; // see FEWS-17782 also accept 02:00:00 on daylight saving time switch try { return fastDateFormat.parseToMillis(dateText, "03:00:00"); } catch (ParseException e1) { throw e; } } Calendar calendar = useMillis? fastDateFormatWithMillies.getCalendar() : fastDateFormat.getCalendar(); calendar.setTimeInMillis(t1); int timeOfDay = getTimeOfDay(calendar); long t2 = getOtherTime(calendar, t1, timeOfDay); timeAmbiguous = t1 != t2; long minTime = Math.min(t1, t2); long maxTime = Math.max(t1, t2); long res = minTime > lastTime ? minTime : maxTime; if (!lastTimeAmbiguous) return res; // see FEWS-17782 also accept two times 02:00:00 instead of two times 01:00:00 on daylight saving time switch if (timeStepMillis != 0L && lastTime + timeStepMillis != res) return lastTime + timeStepMillis; if (timeStep != null && timeStep.isRegular() && timeStep.nextTime(lastTime) != res) return timeStep.nextTime(lastTime); return res; } private static int getTimeOfDay(Calendar calendar) { return (int) (calendar.get(Calendar.HOUR_OF_DAY) * TimeUnit.HOUR_MILLIS + calendar.get(Calendar.MINUTE) * TimeUnit.MINUTE_MILLIS + calendar.get(Calendar.SECOND) * TimeUnit.SECOND_MILLIS + calendar.get(Calendar.MILLISECOND)); } private static long getOtherTime(Calendar calendar, long time, int timeOfDay) { try { calendar.setTimeInMillis(time + TimeUnit.HOUR_MILLIS); if (getTimeOfDay(calendar) == timeOfDay) return time + TimeUnit.HOUR_MILLIS; calendar.setTimeInMillis(time - TimeUnit.HOUR_MILLIS); if (getTimeOfDay(calendar) == timeOfDay) return time - TimeUnit.HOUR_MILLIS; return time; } finally { calendar.setTimeInMillis(time); } } private long parseTime() throws Exception { String dateText = reader.getAttributeValue(null, "date"); if (dateText == null) { throw new Exception("Attribute " + currentHeaderElement + "-date is missing"); } String timeText = reader.getAttributeValue(null, "time"); if (timeText == null) { throw new Exception("Attribute " + currentHeaderElement + "-time is missing"); } boolean useMillis = timeText.contains("."); long time; try { time = useMillis? fastDateFormatWithMillies.parseToMillis(dateText, timeText) : fastDateFormat.parseToMillis(dateText, timeText); } catch (ParseException e) { throw new Exception("Not a valid data time for " + currentHeaderElement + ' ' + dateText + ' ' + timeText, e); } reader.nextTag(); return time; } private void parseTimeStep() throws Exception { String times = reader.getAttributeValue(null, "times"); if (times != null) { timeStep = PiCastorUtils.createTimesOfDayTimeStep(times, getTimeZone()); reader.nextTag(); return; } String unit = reader.getAttributeValue(null, "unit"); TimeUnit tu = unit == null ? null : TimeUnit.get(unit); if (tu != null) { String multiplierText = reader.getAttributeValue(null, "multiplier"); int multiplier; if (multiplierText == null) { multiplier = 1; } else { try { multiplier = Integer.parseInt(multiplierText); } catch (NumberFormatException e) { throw new Exception(ExceptionUtils.getMessage(e), e); } if (multiplier == 0) { throw new Exception("Multiplier is 0"); } } String dividerText = reader.getAttributeValue(null, "divider"); int divider; if (dividerText == null) { divider = 1; } else { try { divider = Integer.parseInt(dividerText); } catch (NumberFormatException e) { throw new Exception(ExceptionUtils.getMessage(e), e); } if (divider == 0) { throw new Exception("divider is 0"); } } reader.nextTag(); timeStepMillis = tu.getMillis() * multiplier / divider; timeStep = null; } else { reader.nextTag(); timeStepMillis = 0; timeStep = IrregularTimeStep.INSTANCE; } } private void initHeader() { header.clear(); header.setFileDescription(virtualFileName); currentHeaderElement = null; timeStep = null; timeStepMillis = 0; startTime = Long.MIN_VALUE; endTime = Long.MIN_VALUE; missingValue = Float.NaN; creationDateText = null; creationTimeText = "00:00:00"; qualifiers.clear(); lat = Double.NaN; lon = Double.NaN; z = Double.NaN; } private void readValuesFromBinFile() throws Exception { TimeStep timeStep = header.getTimeStep(); if (!timeStep.isRegular()) { throw new Exception("Only equidistant time step supported when pi events are stored in bin file instead of xml"); } boolean equidistantMillis = timeStep.isEquidistantMillis(); long stepMillis = equidistantMillis ? timeStep.getStepMillis() : Long.MIN_VALUE; for (long time = startTime; time <= endTime;) { timeSeriesContentHandler.setTime(time); if (bufferPos == bufferCount) fillBuffer(); float value = floatBuffer[bufferPos++]; // we can not use the automatic missing value detection of the content handler because the missing value is different for each time series if (value == missingValue) value = Float.NaN; timeSeriesContentHandler.setValue(value); timeSeriesContentHandler.applyCurrentFields(); if (equidistantMillis) { time += stepMillis; continue; } time = timeStep.nextTime(time); } } private void fillBuffer() throws IOException { int byteBufferCount = 0; while (byteBufferCount % NumberType.FLOAT_SIZE != 0 || byteBufferCount == 0) { int count = binaryInputStream.read(byteBuffer, byteBufferCount, BUFFER_SIZE * NumberType.FLOAT_SIZE - byteBufferCount); assert count != 0; // see read javadoc if (count == -1) throw new EOFException("Bin file is too short"); byteBufferCount += count; } bufferCount = byteBufferCount / NumberType.FLOAT_SIZE; BinaryUtils.copy(byteBuffer, 0, byteBufferCount, floatBuffer, 0, bufferCount, ByteOrder.LITTLE_ENDIAN); bufferPos = 0; } private void initiateTimeStep() { if (timeStep != null) { assert timeStepMillis == 0; return; } if (timeStepMillis == 0){ //no timestep in header. Fix for backward compatibility timeStep = IrregularTimeStep.INSTANCE; return; } if (timeStepMillis >= TimeUnit.HOUR_MILLIS && getTimeZone().useDaylightTime()) { timeStep = IrregularTimeStep.INSTANCE; return; } long startTime = this.startTime == Long.MIN_VALUE ? 0L : this.startTime; if (timeStepMillis % TimeUnit.SECOND_MILLIS != 0) { timeStep = RelativeEquidistantTimeStep.getInstance(timeStepMillis, startTime); return; } long timeZoneOffsetMillis = -startTime % timeStepMillis; if (timeZoneOffsetMillis % TimeUnit.MINUTE_MILLIS != 0) { timeStep = RelativeEquidistantTimeStep.getInstance(timeStepMillis, startTime); return; } timeStep = SimpleEquidistantTimeStep.getInstance(timeStepMillis, timeZoneOffsetMillis); } @SuppressWarnings({"OverlyLongMethod"}) private void parseHeaderElement() throws Exception { switch (currentHeaderElement) { case type: header.setParameterType(parseType(reader.getElementText())); break; case moduleInstanceId: header.setModuleInstanceId(reader.getElementText()); break; case locationId: // see FEWS-9858, when there is no location id the time series are assigned to all locations // this is a flaw in the pi_timeSeries.xsd, the location element is required but is allowed empty strings header.setLocationId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none")); break; case parameterId: // see FEWS-9858, when there is no parameter id the time series are assigned to all locations // this is a flaw in the pi_timeSeries.xsd, the location element is required but is allowed empty strings header.setParameterId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none")); break; case qualifierId: qualifiers.add(reader.getElementText()); break; case ensembleId: header.setEnsembleId(reader.getElementText()); break; case ensembleMemberIndex: header.setEnsembleMemberIndex(parseEnsembleMemberIndex(reader.getElementText())); break; case ensembleMemberId: header.setEnsembleMemberId(reader.getElementText()); break; case timeStep: parseTimeStep(); break; case startDate: startTime = parseTime(); break; case endDate: endTime = parseTime(); break; case forecastDate: header.setForecastTime(parseTime()); break; case approvedDate: header.setApprovedTime(parseTime()); break; case missVal: missingValue = parseString(reader.getElementText()); break; case longName: header.setLongName(reader.getElementText()); break; case stationName: header.setLocationName(reader.getElementText()); break; case units: header.setUnit(reader.getElementText()); break; case domainAxis: parseDomainAxis(); break; case sourceOrganisation: header.setSourceOrganisation(reader.getElementText()); break; case sourceSystem: header.setSourceSystem(reader.getElementText()); break; case fileDescription: header.setFileDescription(reader.getElementText()); break; case creationDate: creationDateText = reader.getElementText(); break; case creationTime: creationTimeText = reader.getElementText(); break; case region: header.setRegion(reader.getElementText()); break; case thresholds: parseThresholds(); break; case lat: lat = parseString(reader.getElementText()); break; case lon: lon = parseString(reader.getElementText()); break; case x: reader.getElementText(); break; case y: reader.getElementText(); break; case z: z = parseString(reader.getElementText()); break; case firstValueTime: reader.getElementText(); break; case lastValueTime: reader.getElementText(); break; case maxValue: reader.getElementText(); break; case minValue: reader.getElementText(); break; case valueCount: reader.getElementText(); break; case maxWarningLevelName: reader.getElementText(); break; } reader.require(XMLStreamConstants.END_ELEMENT, null, currentHeaderElement.name()); reader.nextTag(); } private void parseDomainAxis() throws Exception { String parameterId = reader.getAttributeValue(null, "parameterId"); if (parameterId == null) throw new Exception("Attribute parameterId for domainUnits is missing"); domainParameterIds = Clasz.strings.ensureCapacity(domainParameterIds, domainCount + 1); domainUnits = Clasz.strings.ensureCapacity(domainUnits, domainCount + 1); domainParameterIds[domainCount] = parameterId; domainUnits[domainCount] = reader.getAttributeValue(null, "units"); domainCount++; reader.nextTag(); } private void parseThresholds() throws XMLStreamException { reader.nextTag(); ArrayList<DefaultTimeSeriesHeader.DefaultThreshold> thresholds = new ArrayList<>(); do { if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) { String id = reader.getAttributeValue(null, "id"); String name = reader.getAttributeValue(null, "name"); float value = TextUtils.parseFloat(reader.getAttributeValue(null, "value")); String groupId = reader.getAttributeValue(null, "groupId"); String groupName = reader.getAttributeValue(null, "groupName"); String[] groupIds = groupId == null ? Clasz.strings.emptyArray() : new String[] {groupId}; String[] groupNames = groupName == null ? Clasz.strings.emptyArray() : new String[] {groupName}; thresholds.add(new DefaultTimeSeriesHeader.DefaultThreshold(id, name, value, groupIds, groupNames)); } reader.nextTag(); } while (!reader.getLocalName().equals(currentHeaderElement.name())); header.setHighLevelThresholds(TimeSeriesHeader.Threshold.clasz.newArrayFrom(thresholds)); } private void parseProperties() throws XMLStreamException { reader.require(XMLStreamConstants.START_ELEMENT, null, "properties"); reader.nextTag(); propertyBuilder.clear(); while (!TextUtils.equals(reader.getLocalName(), "properties")){ if (reader.getEventType() != XMLStreamConstants.START_ELEMENT) { // eg <int key="a" value=12><int> reader.nextTag(); continue; } String key = reader.getAttributeValue(null, "key"); String value = reader.getAttributeValue(null, "value"); String date = reader.getAttributeValue(null, "date"); String time = reader.getAttributeValue(null, "time"); switch (reader.getLocalName()) { case "string": propertyBuilder.addString(key, value); break; case "int": propertyBuilder.addInt(key, TextUtils.parseInt(value)); break; case "float": propertyBuilder.addFloat(key, TextUtils.parseFloat(value)); break; case "double": propertyBuilder.addDouble(key, TextUtils.parseDouble(value)); break; case "bool": propertyBuilder.addBoolean(key, Boolean.parseBoolean(value)); break; case "dateTime": try { if (time.contains(".")) { propertyBuilder.addDateTime(key, fastDateFormatWithMillies.parseToMillis(date, time)); } else { propertyBuilder.addDateTime(key, fastDateFormat.parseToMillis(date, time)); } break; } catch (ParseException e) { throw new XMLStreamException("Invalid date time "+ date + ' ' + time); } default: throw new XMLStreamException("Invalid property type " + reader.getLocalName()); } reader.nextTag(); } timeSeriesContentHandler.setProperties(propertyBuilder.build()); reader.require(XMLStreamConstants.END_ELEMENT, null, "properties"); reader.nextTag(); } private static float parseString(String gotString) throws Exception { // <element name="missVal" type="double" default="NaN"> // when default is used in schema for element the consequence is that empty strings are allowed if (gotString.isEmpty()) return Float.NaN; try { return TextUtils.parseFloat(gotString); } catch (NumberFormatException e) { throw new Exception(ExceptionUtils.getMessage(e), e); } } private static int parseEnsembleMemberIndex(String gotString) throws Exception { int index = Integer.parseInt(gotString); if (index < 0) { throw new Exception("Negative ensemble member index not allowed " + gotString); } return index; } private static ParameterType parseType(String gotString) throws Exception { ParameterType type = ParameterType.get(gotString); if (type == null) { throw new Exception("Type in header should be instantaneous or accumulative and not " + gotString); } return type; } private void detectHeaderElement() throws Exception { if (reader.getEventType() != XMLStreamConstants.START_ELEMENT) throw new Exception("header element expected"); String localName = reader.getLocalName(); HeaderElement element; try { element = Enum.valueOf(HeaderElement.class, localName); } catch (Exception e) { throw new Exception("Unknown header element: " + localName); } if (currentHeaderElement == element && currentHeaderElement.isMultipleAllowed()) return; if (currentHeaderElement != null && element.ordinal() < currentHeaderElement.ordinal()) { throw new Exception("Header elements in wrong order: " + localName); } if (currentHeaderElement == HeaderElement.ensembleMemberIndex && element == HeaderElement.ensembleMemberId) { throw new Exception("Duplicate header element, both ensembleMemberIndex and ensembleMemberId in header"); } if (currentHeaderElement == element) { throw new Exception("Duplicate header element: " + localName); } if (reader.getAttributeCount() > 0 && !element.hasAttributes()) { throw new Exception("Attributes are not allowed for header element " + localName); } int nextOrdinal = currentHeaderElement == null ? 0 : currentHeaderElement.ordinal() + 1; // order is correct and no duplicate so currentHeaderElement can not be last header element assert nextOrdinal < HEADER_ELEMENTS.length; HeaderElement nextHeaderElement = HEADER_ELEMENTS[nextOrdinal]; if (nextHeaderElement.isRequired() && nextHeaderElement != element) { throw new Exception("Required header item missing: " + nextHeaderElement); } currentHeaderElement = element; } public TimeZone getTimeZone() { //time zone for both fastDateFormat and fastDateFormatWithMillies should be the same. return fastDateFormat.getTimeZone(); } @SuppressWarnings("UnusedDeclaration") public PiTimeSeriesSerializer.EventDestination getEventDestination() { return eventDestination; } @SuppressWarnings("UnusedDeclaration") public float getMissingValue() { return missingValue; } }