Code Block | ||||
---|---|---|---|---|
| ||||
package nl.wldelft.fews.pi; import com.sun.org.apache.xml.internal.serialize.OutputFormat; import com.sun.org.apache.xml.internal.serialize.XMLSerializer; import nl.wldelft.util.FastDateFormat*; import nl.wldelft.util.coverage.FileUtilsGeometry; import nl.wldelft.util.coverage.PeriodGeometryUtils; import nl.wldelft.util.TimeUnitgeodatum.GeoPoint; import nl.wldelft.util.io.LineWriterVirtualOutputDir; import nl.wldelft.util.io.LittleEndianDataOutputStreamVirtualOutputDirConsumer; import nl.wldelft.util.io.TextSerializerXmlSerializer; import nl.wldelft.util.iotimeseries.VirtualOutputDirTimeSeriesContent; import nl.wldelft.util.iotimeseries.VirtualOutputDirConsumerTimeSeriesHeader; import nl.wldelft.util.timeseries.TimeSeriesContentTimeStep; import nl.wldelft.util.timeseries.TimeSeriesHeaderjavax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; import nljava.wldelft.util.timeseries.TimeStepio.IOException; import java.io.OutputStream; import org.xml.sax.ContentHandlerjava.nio.ByteOrder; import java.util.Arrays; import orgjava.xmlutil.sax.SAXExceptionLocale; import orgjava.xml.sax.helpers.AttributesImplutil.SimpleTimeZone; import java.util.TimeZone; public class PiTimeSeriesSerializer implements TextSerializer<TimeSeriesContent>XmlSerializer<TimeSeriesContent>, VirtualOutputDirConsumer { publicprivate enumstatic EventDestination {XML_EMBEDDED, SEPARATE_BINARY_FILE, ONLY_HEADERS} private EventDestination eventDestination = EventDestination.XML_EMBEDDEDfinal int BUFFER_SIZE = 2048; private PiVersionTimeZone versiontimeZoneNoDst = PiVersion.VERSION_1_2null; privatepublic finalenum FastDateFormatEventDestination dateFormat{ = new FastDateFormat("yyyy-MM-dd"); private final FastDateFormat timeFormat = new FastDateFormat("HH:mm:ss"); XML_EMBEDDED, SEPARATE_BINARY_FILE, ONLY_HEADERS } privatepublic TimeSeriesContentenum timeSeriesContent = null;EnsembleMemberFormat { private LineWriter writer = null; INDEX, ID, HIDE private ContentHandler xmlContentHandler = null; } private VirtualOutputDirEventDestination virtualOutputDireventDestination = nullEventDestination.XML_EMBEDDED; private LittleEndianDataOutputStreamPiVersion binaryOutputSteamversion = null; PiVersion.VERSION_1_2; private finalEnsembleMemberFormat AttributesImplensembleMemberFormat attributesBuffer = new AttributesImpl()EnsembleMemberFormat.INDEX; publicprivate PiTimeSeriesSerializer() { } public EventDestination getEventDestination() {FastDateFormat dateFormat = null; private FastDateFormat timeFormat = null; private TimeSeriesContent timeSeriesContent = return eventDestinationnull; } private XMLStreamWriter writer = publicnull; void setEventDestination(EventDestination eventDestination) { private VirtualOutputDir virtualOutputDir = null; private OutputStream this.eventDestinationbinaryOutputSteam = eventDestinationnull; } public PiVersion getVersion() {private byte[] byteBuffer = null; private float[] floatBuffer = return versionnull; } private int bufferPos = public0; void setVersion(PiVersion version) { private long[] cachedTimes = new long[4]; if (version private String[] cachedDateStrings == null) new String[4]; private String[] cachedTimeStrings = new String[4]; throwprivate new IllegalArgumentException("versionint cachedSeconds == null")-1; private String this.versioncachedSecondsString = versionnull; } public PiTimeSeriesSerializer() { @Override} public void setVirtualOutputDirPiTimeSeriesSerializer(VirtualOutputDirPiVersion virtualOutputDirversion) { this.virtualOutputDirversion = virtualOutputDirversion; } @Override public EventDestination getEventDestination() { public void serialize(TimeSeriesContent timeSeriesContent, LineWriter writer, String virtualFileName) throws Exception { return eventDestination; } public this.timeSeriesContent = timeSeriesContent;void setEventDestination(EventDestination eventDestination) { this.writereventDestination = writereventDestination; } dateFormat.setTimeZone(this.timeSeriesContent.getDefaultTimeZone());public PiVersion getVersion() { timeFormat.setTimeZone(this.timeSeriesContent.getDefaultTimeZone())return version; } public ifvoid setVersion(eventDestination == EventDestination.SEPARATE_BINARY_FILEPiVersion version) { if (virtualOutputDirversion == null) throw new IllegalStateExceptionIllegalArgumentException("virtualOutputDirversion == null"); this.version = version; binaryOutputSteam = new LittleEndianDataOutputStream(virtualOutputDir.getOutputStream(FileUtils.getPathWithOtherExtension(virtualFileName, "bin")));} public EnsembleMemberFormat getEnsembleMemberFormat() { return ensembleMemberFormat; try {} public void setEnsembleMemberFormat(EnsembleMemberFormat ensembleMemberFormat) { if serialize(ensembleMemberFormat == null); }throw finally { new IllegalArgumentException("ensembleMemberFormat == null"); this.ensembleMemberFormat = ensembleMemberFormat; binaryOutputSteam.close();} @Override public void setVirtualOutputDir(VirtualOutputDir virtualOutputDir) }{ this.virtualOutputDir = returnvirtualOutputDir; } binaryOutputSteam = null;@Override public void serialize(); TimeSeriesContent timeSeriesContent, XMLStreamWriter streamWriter, } private void serialize(String virtualFileName) throws Exception { XMLSerializerthis.timeSeriesContent serializer = new XMLSerializer()timeSeriesContent; OutputFormat of = new OutputFormat("XML", "UTF-8", true); serializer.setOutputFormat(ofif (timeSeriesContent.getTimeSeriesCount() <=0) throw new IOException("No TimeSeries to be Written to " + virtualFileName); serializerthis.setOutputCharStream(writer)writer = streamWriter; xmlContentHandlerif (ensembleMemberFormat = serializer.asContentHandler(); = EnsembleMemberFormat.ID && version.getIntId() < PiVersion.VERSION_1_10.getIntId()) { xmlContentHandler.startDocument( throw new IOException("ensembleMemberId not supported for pi version " + version); attributesBuffer.clear();} addAttribute("xmlns", "http://www.wldelft.nl/fews/PI");timeSeriesContent.setRequireEnsembleMemberIndices(ensembleMemberFormat == EnsembleMemberFormat.INDEX); addAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); if (!ObjectUtils.equals(timeZoneNoDst, this.timeSeriesContent.getDefaultTimeZone())) { addAttribute("xsi:schemaLocation", PiSchemaLocations.get("pi_timeseries.xsd")); Arrays.fill(cachedTimes, Long.MIN_VALUE); timeZoneNoDst = timeSeriesContent.getDefaultTimeZone(); } if (timeZoneNoDst.useDaylightTime()){ //Make timeZone not use day light saving time (DST) because the timeZone field does not use DST therefore // all values must also not use DST timeZoneNoDst = new SimpleTimeZone(timeZoneNoDst.getRawOffset(), timeZoneNoDst.getID(), 0, 0, 0, 0, 0, 0, 0, 0); } dateFormat = FastDateFormat.getInstance("yyyy-MM-dd", timeZoneNoDst, Locale.US, dateFormat); timeFormat = FastDateFormat.getInstance("HH:mm:ss", timeZoneNoDst, Locale.US, timeFormat); String binFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "bin"); if (virtualOutputDir != null) virtualOutputDir.delete(binFileName); if (eventDestination == EventDestination.SEPARATE_BINARY_FILE) { if (virtualOutputDir == null) throw new IllegalStateException("virtualOutputDir == null"); binaryOutputSteam = virtualOutputDir.getOutputStream(binFileName); if (byteBuffer == null) { byteBuffer = new byte[BUFFER_SIZE * NumberType.FLOAT_SIZE]; floatBuffer = new float[BUFFER_SIZE]; } try { serialize(); } finally { bufferPos = 0; binaryOutputSteam.close(); } return; } binaryOutputSteam = null; serialize(); } private void serialize() throws Exception { writer.writeStartDocument(); writer.writeStartElement("TimeSeries"); writer.setDefaultNamespace("http://www.wldelft.nl/fews/PI"); writer.writeNamespace("", "http://www.wldelft.nl/fews/PI"); writer.writeAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); // writer.writeAttribute("xmlns", "http://www.wldelft.nl/fews/PI"); writer.writeAttribute("xsi:schemaLocation", PiSchemaLocations.get("pi_timeseries.xsd")); writer.writeAttribute("version", version.toString()); writeElement("timeZone", String.valueOf((double) timeSeriesContent.getDefaultTimeZone().getRawOffset() / (double) TimeUnit.HOUR_MILLIS)); for (int i = 0, n = timeSeriesContent.getTimeSeriesCount(); i < n; i++) { timeSeriesContent.setTimeSeriesIndex(i); writer.writeStartElement("series"); writeHeader(); writeEvents(); writer.writeEndElement(); } writer.writeEndElement(); writer.writeEndDocument(); if (eventDestination == EventDestination.SEPARATE_BINARY_FILE) flushBinEvents(); } private void writeHeader() throws Exception { TimeSeriesHeader header = timeSeriesContent.getTimeSeriesHeader(); PiTimeSeriesHeader piHeader = header instanceof PiTimeSeriesHeader ? (PiTimeSeriesHeader) header : new PiTimeSeriesHeader(); writer.writeStartElement("header"); writeElement("type", header.getParameterType() == null ? "instantaneous" : header.getParameterType().getName()); writeElement("locationId", header.getLocationId() == null ? "unknown" : header.getLocationId()); writeElement("parameterId", header.getParameterId() == null ? "unknown" : header.getParameterId()); if (version.getIntId() >= PiVersion.VERSION_1_4.getIntId()) { for (int i = 0, n = header.getQualifierCount(); i < n; i++) { writeElement("qualifierId", header.getQualifierId(i)); } } if (version.getIntId() >= PiVersion.VERSION_1_4.getIntId() && header.getEnsembleId() != null && !header.getEnsembleId().equals("main")) { if (ensembleMemberFormat != EnsembleMemberFormat.HIDE) { writeOptionalElement("ensembleId", header.getEnsembleId()); writeOptionalElement(ensembleMemberFormat == EnsembleMemberFormat.INDEX ? "ensembleMemberIndex" : "ensembleMemberId", header.getEnsembleMemberId()); } } writeTimeStep(header); writePeriod(); if (version.getIntId() >= PiVersion.VERSION_1_5.getIntId()) writeTime("forecastDate", header.getForecastTime(), 2); writeElement("missVal", Float.toString(timeSeriesContent.getDefaultMissingValue())); writeOptionalElement("longName", piHeader.getLongName()); writeOptionalElement("stationName", header.getLocationName()); if (version.getIntId() >= PiVersion.VERSION_1_7.getIntId()) writeCoordinates(header); writeOptionalElement("units", header.getUnit()); writeOptionalElement("sourceOrganisation", piHeader.getSourceOrganisation()); writeOptionalElement("sourceSystem", piHeader.getSourceSystem()); writeOptionalElement("fileDescription", piHeader.getFileDescription()); if (header.getCreationTime() != Long.MIN_VALUE) { updateDateTimeStringCache(header.getCreationTime(), 3); writeElement("creationDate", cachedDateStrings[3]); writeElement("creationTime", cachedTimeStrings[3]); } writeOptionalElement("region", piHeader.getRegion()); if (header.getHighLevelThresholdCount() > 0) { writeHighThresholds(header); } writer.writeEndElement(); } private void writeProperties(Properties properties) throws XMLStreamException { if (version.getIntId() < PiVersion.VERSION_1_13.getIntId()) return; if (properties.isEmpty()) { writer.writeEmptyElement("properties"); // next events don't have properties return; } writer.writeStartElement("properties"); for (int i = 0, n = properties.size(); i < n; i++) { String key = properties.getKey(i); PropertyType type = properties.getType(i); switch (type) { case STRING: writer.writeEmptyElement("string"); writer.writeAttribute("key", key); writer.writeAttribute("value", properties.getString(i)); continue; case INT: writer.writeEmptyElement("int"); writer.writeAttribute("key", key); writer.writeAttribute("value", TextUtils.toString(properties.getInt(i))); continue; case FLOAT: writer.writeEmptyElement("float"); writer.writeAttribute("key", key); writer.writeAttribute("value", Float.toString(properties.getInt(i))); continue; case DOUBLE: writer.writeEmptyElement("float"); writer.writeAttribute("key", key); writer.writeAttribute("value", Double.toString(properties.getDouble(i))); continue; case BOOLEAN: writer.writeEmptyElement("bool"); writer.writeAttribute("key", key); writer.writeAttribute("value", Boolean.toString(properties.getBool(i))); continue; case DATE_TIME: writer.writeEmptyElement("dateTime"); writer.writeAttribute("key", key); long dateTime = properties.getDateTime(i); writer.writeAttribute("date", dateFormat.format(dateTime)); writer.writeAttribute("time", timeFormat.format(dateTime)); } } writer.writeEndElement(); } private void writeHighThresholds(TimeSeriesHeader header) throws XMLStreamException { boolean thresholdsAvailable = false; boolean firstThreshold = true; for (int i = 0; i < header.getHighLevelThresholdCount(); i++) { String id = header.getHighLevelThresholdId(i); String name = header.getHighLevelThresholdName(i); //Note: here value can be NaN for LevelThresholdValues that have different values for different aggregationTimeSpans configured. float value = header.getHighLevelThresholdValue(i); if (Float.isNaN(value)) { continue; } thresholdsAvailable = true; if (firstThreshold) { writer.writeStartElement("thresholds"); firstThreshold = false; } writer.writeStartElement("highLevelThreshold"); writer.writeAttribute("id", id); writer.writeAttribute("name", name); writer.writeAttribute("value", Float.toString(value)); if (version.getIntId() >= PiVersion.VERSION_1_14.getIntId()) { String groupId = header.getHighLevelThresholdGroupId(i); if (groupId != null){ writer.writeAttribute("groupId", groupId); String groupName = header.getHighLevelThresholdGroupName(i); if (groupName != null) writer.writeAttribute("groupName", groupName); addAttribute("version", version.toString()); } xmlContentHandler.startElement("", "TimeSeries", "TimeSeries", attributesBuffer); } writeElement("timeZone", String.valueOf((double) timeSeriesContent.getDefaultTimeZone().getRawOffset() / (double) TimeUnit.HOUR_MILLIS))writer.writeEndElement(); for (int i = 0, n = timeSeriesContent.getTimeSeriesCount(); i < n; i++} if (thresholdsAvailable) { timeSeriesContentwriter.setTimeSeriesIndexwriteEndElement(i); } } private void xmlContentHandler.startElement(null, null, "series", null);writePeriod() throws XMLStreamException { TimeStep timeStep writeHeader= timeSeriesContent.getTimeSeriesHeader().getTimeStep(); Period period = writeEventstimeSeriesContent.getTimeSeriesPeriod(); Period headerPeriod; xmlContentHandler.endElement(null, null, "series"); if (period == Period.NEVER) { } xmlContentHandler.endElement(null, null, "TimeSeries"); // create a dummy period xmlContentHandler.endDocument(); } long privatenow void writeEvents() throws Exception {= timeStep.nearestTime(System.currentTimeMillis()); for (int i = 0, nheaderPeriod = new timeSeriesContent.getContentTimeCount(Period(now, now); i < n; i++) { } else { timeSeriesContent.setContentTimeIndex(i); headerPeriod = period; if (!timeSeriesContent.isTimeAvailable()) continue; } writeEvent(timeSeriesContent.getTime()writeTime("startDate", headerPeriod.getStartTime(), 0); writeTime("endDate", }headerPeriod.getEndTime(), 1); } private void writeHeaderwriteCoordinates(TimeSeriesHeader header) throws ExceptionXMLStreamException { TimeSeriesHeaderGeometry headergeometry = timeSeriesContentheader.getTimeSeriesHeadergetGeometry(); PiTimeSeriesHeaderif piHeader(geometry == header instanceof PiTimeSeriesHeader ? (PiTimeSeriesHeader) header : new PiTimeSeriesHeader(); null) return; xmlContentHandler.startElement(null, null, "header", nullGeoPoint geoPoint = GeometryUtils.getPoint(geometry, 0); writeElement("typelat", header.getParameterType() == null ? "instantaneous" : header.getParameterType().getName())Double.toString(geoPoint.getLatitude())); writeElement("lon", Double.toString(geoPoint.getLongitude())); writeElement("locationIdx", header.getLocationId() == null ? "unknown" : header.getLocationId())Double.toString(geoPoint.getX())); writeElement("y", Double.toString(geoPoint.getY())); writeElement("parameterIdz", header.getParameterId() == null ? "unknown" : header.getParameterId());Double.toString(geoPoint.getZ())); } private void updateDateTimeStringCache(long time, int cacheIndex) { if (version.ordinal() >= PiVersion.VERSION_1_4.ordinal()) {cachedTimes[cacheIndex] == time) return; cachedTimes[cacheIndex] = time; for (int i = 0, n cachedDateStrings[cacheIndex] = headerdateFormat.getQualifierCountformat(time); i < n; i++) { cachedTimeStrings[cacheIndex] writeElement("qualifierId", header.getQualifierId(i)= timeFormat.format(time); } private void } }writeTime(String name, long time, int cacheIndex) throws XMLStreamException { if (version.ordinal() >time == PiVersionLong.VERSION_1_4.ordinal() && header.getEnsembleId() != null && !header.getEnsembleId().equals("main")) { MIN_VALUE) return; writer.writeEmptyElement(name); updateDateTimeStringCache(time, cacheIndex); writeOptionalElementwriter.writeAttribute("ensembleIddate", header.getEnsembleId())cachedDateStrings[cacheIndex]); writeOptionalElementwriter.writeAttribute("ensembleMemberIndextime", header.getEnsembleMemberIndex()cachedTimeStrings[cacheIndex]); } } private void writeTimeStep(TimeSeriesHeader header) throws XMLStreamException { writeTimeStep(headerwriter.writeEmptyElement("timeStep"); writePeriodTimeStep timeStep = header.getTimeStep(); // todo if (version.ordinal() >= PiVersion.VERSION_1_5.ordinal()) writeTime("forecastDate", header.getForecastTime()); add support for month time step writeElement("missVal", Float.toString(timeSeriesContent.getDefaultMissingValueif (timeStep.isEquidistantMillis())); { writeOptionalElement writer.writeAttribute("longNameunit", piHeader.getLongName())"second"); int seconds = writeOptionalElement("stationName", header.getLocationName()(int) (timeStep.getStepMillis() / 1000); writeOptionalElement("units", header.getUnit()); if (cachedSeconds != seconds) { writeOptionalElement("sourceOrganisation", piHeader.getSourceOrganisation()); writeOptionalElement("sourceSystem", piHeader.getSourceSystem()); cachedSecondsString = TextUtils.toString(seconds); writeOptionalElement("fileDescription", piHeader.getFileDescription()); cachedSeconds = seconds; if (header.getCreationTime() != Long.MIN_VALUE) { } writeElementwriter.writeAttribute("creationDatemultiplier", dateFormat.format(header.getCreationTime()))cachedSecondsString); } else { writeElement writer.writeAttribute("creationTimeunit", timeFormat.format(header.getCreationTime()))"nonequidistant"); } } writeOptionalElement("region", piHeader.getRegion()); private void writeEvents() throws Exception { switch xmlContentHandler.endElement(null, null, "header");(eventDestination) { } private void writePeriod() throws SAXException { case ONLY_HEADERS: TimeStep timeStep = timeSeriesContent.getTimeSeriesHeader().getTimeStep() return; Period period = timeSeriesContent.getTimeSeriesPeriod(); case SEPARATE_BINARY_FILE: Period headerPeriod; if (period == Period.NEVER) {writeBinEvents(); // create a dummy periodreturn; long now = timeStep.nearestTime(System.currentTimeMillis()); case XML_EMBEDDED: headerPeriod = new PeriodwriteXmlEvents(now, now); } else { } private void writeBinEvents() throws headerPeriod = period;Exception { } for (int i = 0, n writeTime("startDate", headerPeriod.getStartTime());= timeSeriesContent.getContentTimeCount(); i < n; i++) { writeTime("endDate", headerPeriod.getEndTime()) timeSeriesContent.setContentTimeIndex(i); } private void writeTime(String name, long time) throws SAXException { if (!timeSeriesContent.isTimeAvailable()) continue; if (timebufferPos == Long.MINBUFFER_VALUESIZE) return; attributesBuffer.clearflushBinEvents(); addAttribute("date", dateFormat.format(time)); addAttribute("time", timeFormat.format(time))floatBuffer[bufferPos++] = timeSeriesContent.getValue(); writeAttributes(name);} } private void writeTimeStepflushBinEvents(TimeSeriesHeader header) throws SAXExceptionIOException { TimeStepif timeStep(bufferPos == header.getTimeStep()0) return; attributesBufferBinaryUtils.clear(); copy(floatBuffer, 0, bufferPos, byteBuffer, 0, bufferPos * NumberType.FLOAT_SIZE, ByteOrder.LITTLE_ENDIAN); // todo add support for month time step binaryOutputSteam.write(byteBuffer, 0, bufferPos * NumberType.FLOAT_SIZE); if (timeStep.isEquidistantMillis()) {bufferPos = 0; } private long seconds = timeStep.getStepMillisvoid writeXmlEvents() / 1000;throws XMLStreamException { Properties properties addAttribute("unit","second")= Properties.NONE; for (int i = addAttribute("multiplier", String.valueOf(seconds));0, n = timeSeriesContent.getContentTimeCount(); i < n; i++) { } else { timeSeriesContent.setContentTimeIndex(i); if addAttribute("unit", "nonequidistant")(!timeSeriesContent.isTimeAvailable()) continue; } Properties newProperties = writeAttributes("timeStep"timeSeriesContent.getProperties(); } private void writeEvent(long time) throws Exceptionif (!newProperties.equals(properties)) { if (eventDestination == EventDestination.ONLY_HEADERS) return; properties = newProperties; if (eventDestination == EventDestination.SEPARATE_BINARY_FILE) { binaryOutputSteam.writeFloat(timeSeriesContent.getValue()); writeProperties(properties); } return; long time } = timeSeriesContent.getTime(); attributesBufferwriter.clearwriteEmptyElement("event"); addAttributewriter.writeAttribute("date", dateFormat.format(time)); addAttribute writer.writeAttribute("time", timeFormat.format(time)); addAttributewriter.writeAttribute("value", timeSeriesContent.getValue('.')); addAttribute writer.writeAttribute("flag", timeSeriesContent.getStringFlag()); if (version.ordinalgetIntId() >= PiVersion.VERSION_1_311.ordinalgetIntId()) { ) { String commentflagSource = timeSeriesContent.getCommentgetFlagSource(); if (commentflagSource != null) addAttributewriter.writeAttribute("commentflagSource", timeSeriesContent.getComment()flagSource); } writeAttributes("event"); if (version.getIntId() >= PiVersion.VERSION_1_3.getIntId()) { } private void writeOptionalElement(String elementName, int index)String throwscomment SAXException {= timeSeriesContent.getComment(); if (index == -1) return; if (comment != null) writeElement(elementName, Integer.toString(index))writer.writeAttribute("comment", comment); } private void writeOptionalElement(String elementName, String} s) throws SAXException { if (version.getIntId(s) == null) return; >= PiVersion.VERSION_1_10.getIntId()) { if (s.trim().length() == 0) return; String user = writeElement(elementName, stimeSeriesContent.getUser(); } private void writeElement(String name, String value) throws SAXExceptionif { (user != xmlContentHandler.startElement(null, null, name, nullnull) writer.writeAttribute("user", user); xmlContentHandler.characters(value.toCharArray(), 0, value.length()); } xmlContentHandler.endElement(null, null, name);} } private void writeAttributeswriteOptionalElement(String elementName, String names) throws SAXExceptionXMLStreamException { xmlContentHandler.startElement("", name, name, attributesBuffer)if (s == null) return; if (s.trim().isEmpty()) return; xmlContentHandler.endElementwriteElement(nullelementName, null, names); } private void addAttributewriteElement(String name, String value) throws XMLStreamException { attributesBufferwriter.addAttribute("", name, name, "CDATA", writeStartElement(name); writer.writeCharacters(value); writer.writeEndElement(); } } |