Page History

Code Block

package nl.wldelft.fews.pi;

import nl.wldelft.util.DateUtils;
import nl.wldelft.util.ExceptionUtils;
import nl.wldelft.util.FastDateFormat;
import nl.wldelft.util.FileUtils;
import nl.wldelft.util.Period;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.TimeUnit;
import nl.wldelft.util.TimeZoneUtils;
import nl.wldelft.util.NumberType;
import nl.wldelft.util.BinaryUtils;
import nl.wldelft.util.io.VirtualInputDir;
import nl.wldelft.util.io.VirtualInputDirConsumer;
import nl.wldelft.util.io.XmlParser;
import nl.wldelft.util.timeseries.IrregularTimeStep;
import nl.wldelft.util.timeseries.ParameterType;
import nl.wldelft.util.timeseries.SimpleEquidistantTimeStep;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import nl.wldelft.util.timeseries.TimeStep;
import org.apache.log4j.Logger;

import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.EOFException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;
import java.nio.ByteOrder;

public class PiTimeSeriesParser implements XmlParser<TimeSeriesContentHandler>, VirtualInputDirConsumer {
    private static final Logger log = Logger.getLogger(PiTimeSeriesParser.class);

    private static final int BUFFER_SIZE = 2048;

    @Override
    public void setVirtualInputDir(VirtualInputDir virtualInputDir) {
        this.virtualInputDir = virtualInputDir;
    }

    private enum HeaderElement {
        type(F.R), locationId(F.R),
        parameterId(F.R), qualifierId(F.M), ensembleId, ensembleMemberIndex,
        timeStep(F.R | F.A), startDate(F.R | F.A), endDate(F.R | F.A), forecastDate(F.A),
        missVal, longName, stationName, units,
        sourceOrganisation, sourceSystem, fileDescription,
        creationDate, creationTime, region, thresholds;

        interface F {
            int A = 1 << 0; // attributes
            int R = 1 << 1; // required/* ================================================================
 * Delft FEWS 
 * ================================================================
 *
 * Project Info:  http://www.wldelft.nl/soft/fews/index.html
 * Project Lead:  Karel Heynert (karel.heynert@wldelft.nl)
 *
 * (C) Copyright 2003, by WL | Delft Hydraulics
 *                        P.O. Box 177
 *                        2600 MH  Delft
 *                        The Netherlands
 *                        http://www.wldelft.nl
 *
 * DELFT-FEWS is a sophisticated collection of modules designed 
 * for building a FEWS customised to the specific requirements 
 * of individual agencies. An open modelling approach allows users
 * to add their own modules in an efficient way.
 *
 * ----------------------------------------------------------------
 * PiTimeSeriesParser.java
 * ----------------------------------------------------------------
 * (C) Copyright 2003, by WL | Delft Hydraulics
 *
 * Original Author:  Erik de Rooij
 * Original Author:  Onno van den Akker
 */

package nl.wldelft.fews.pi;

import nl.wldelft.util.*;
import nl.wldelft.util.coverage.PointGeometry;
import nl.wldelft.util.geodatum.GeoDatum;
import nl.wldelft.util.io.VirtualInputDir;
import nl.wldelft.util.io.VirtualInputDirConsumer;
import nl.wldelft.util.io.XmlParser;
import nl.wldelft.util.timeseries.*;

import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.TimeZone;

public class PiTimeSeriesParser implements XmlParser<TimeSeriesContentHandler>, VirtualInputDirConsumer {
    private static final int BUFFER_SIZE = 2048;
    private final Properties.Builder propertyBuilder = new Properties.Builder();

    @Override
    public void setVirtualInputDir(VirtualInputDir virtualInputDir) {
        this.virtualInputDir = virtualInputDir;
    }

    private enum HeaderElement {
        type(F.R), locationId(F.R),
        parameterId(F.R), qualifierId(F.M), ensembleId, ensembleMemberIndex, ensembleMemberId,
        timeStep(F.R | F.A), startDate(F.R | F.A), endDate(F.R | F.A), forecastDate(F.A),
        missVal, longName, stationName, lat, lon, x, y, z, units,
        sourceOrganisation, sourceSystem, fileDescription,
        creationDate, creationTime, region, thresholds;

        interface F {
            int A = 1 << 0; // attributes
            int R = 1 << 1; // required;
            int M = 1 << 2; // multiple;
        }

        private final int flags;

        HeaderElement() {
            this.flags = 0;
        }

        HeaderElement(int flags) {
            this.flags = flags;
        }

        public boolean isRequired() {
            return (flags & F.R) != 0;
        }

        public boolean hasAttributes() {
            return (flags & F.A) != 0;
        }

        public boolean isMultipleAllowed() {
            return (flags & F.M) != 0;
        }
    }


    // fastDateFormat is used to keep track of last time zone and lenient
    private FastDateFormat fastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd", "HH:mm:ss", TimeZoneUtils.GMT, Locale.US, null);

    private HeaderElement currentHeaderElement = null;

    private static final HeaderElement[] HEADER_ELEMENTS = HeaderElement.class.getEnumConstants();

    private PiTimeSeriesHeader header = new PiTimeSeriesHeader();
    private List<String> qualifiers = new ArrayList<>();
    private long timeStepMillis = 0;
    private TimeStep timeStep = null;
    private long startTime = Long.MIN_VALUE;
    private long endTime = Long.MIN_VALUE;
    private float missingValue = Float.NaN;
    private String creationDateText = null;
    private String creationTimeText = null;

    private TimeSeriesContentHandler timeSeriesContentHandler = null;

    private PiTimeSeriesSerializer.EventDestination eventDestination = null;

    /**
     * For performance reasons the pi time series format allows that the values are stored in
     * a separate bin file instead of embedded in the xml file.
     * The bin file should have same name as the xml file except the extension equals bin
     * In this case all time series should be equidistant.
     */
    private VirtualInputDir virtualInputDir = VirtualInputDir.NONE;
    private InputStream binaryInputStream = null;
    private byte[] byteBuffer = null;
    private float[] floatBuffer = null;
    private int bufferPos = 0;
    private int bufferCount = 0;

    private XMLStreamReader reader = null;
    private String virtualFileName = null;

    private static boolean lenient = false;
    private double lat = Double.NaN;
    private double lon = Double.NaN;
    private double z = Double.NaN;

    /**
     * For backwards compatibility. Earlier versions of the PiTimeSeriesParser were tolerant about the date/time format
     * and the case insensitive for header element names.
     * This parser should not accept files that are not valid according to pi_timeseries.xsd
     * When old adapters are not working you can UseLenientPiTimeSeriesParser temporary till the adapter is fixed
     *
     * @param lenient
     */
    public static void setLenient(boolean lenient) {
        PiTimeSeriesParser.lenient = lenient;
    }

    public static boolean isLenient() {
        return lenient;
    }

    public PiTimeSeriesParser() {
        fastDateFormat.setLenient(lenient);
    }

    @Override
    public void parse(XMLStreamReader reader, String virtualFileName, TimeSeriesContentHandler timeSeriesContentHandler) throws Exception {
        this.reader = reader;
        this.virtualFileName = virtualFileName;
        this.timeSeriesContentHandler = timeSeriesContentHandler;

        String virtualBinFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "bin");

        // time zone can be overruled by one or more time zone elements in the pi file
        this.fastDateFormat.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone());

        if (!virtualInputDir.exists(virtualBinFileName)) {
            eventDestination = PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED;
            parse();
            return;
        }

        eventDestination = PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE;
        binaryInputStream = virtualInputDir.getInputStream(virtualBinFileName);
        try {
            if (byteBuffer == null) {
                byteBuffer = new byte[BUFFER_SIZE * NumberType.FLOAT_SIZE];
                floatBuffer = new float[BUFFER_SIZE];
            }
            parse();
            boolean eof = bufferPos == bufferCount && binaryInputStream.read() == -1;
            if (!eof)
                throw new IOException("More values available in bin file than expected based on time step and start and end time\n" + FileUtils.getPathWithOtherExtension(virtualFileName, "bin"));

        } finally {
            bufferPos = 0;
            bufferCount = 0;
            binaryInputStream.close();
            binaryInputStream = null;
        }
    }

    private void parse() throws Exception {
        reader.require(XMLStreamConstants.START_DOCUMENT, null, null);
        reader.nextTag();
        reader.require(XMLStreamConstants.START_ELEMENT, null, "TimeSeries");
        reader.nextTag();

        while (reader.getEventType() != XMLStreamConstants.END_ELEMENT) {
            parseTimeZone();
            readTimeSeries();
        }

        reader.require(XMLStreamConstants.END_ELEMENT, null, "TimeSeries");
        reader.next();
        reader.require(XMLStreamConstants.END_DOCUMENT, null, null);

    }

    private void readTimeSeries() throws Exception {
        reader.require(XMLStreamConstants.START_ELEMENT, null, "series");
        reader.nextTag();
        parseHeader();
        timeSeriesContentHandler.setProperties(Properties.NONE);
        if (eventDestination == PiTimeSeriesSerializer.EventDestination.XML_EMBEDDED) {
            while (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {
                String localName = reader.getLocalName();
                if (TextUtils.equals(localName, "event")) {
                    parseEvent();
                } else if (TextUtils.equals(localName, "properties")) {
                    parseProperties();
                }  else {
                    break;
                }
            }

            if (reader.getEventType() == XMLStreamConstants.START_ELEMENT) {
                // skip comment
                reader.require(XMLStreamConstants.START_ELEMENT, null, "comment");
                reader.getElementText();
                reader.nextTag();
            }
        } else {
            assert eventDestination == PiTimeSeriesSerializer.EventDestination.SEPARATE_BINARY_FILE;
            readValuesFromBinFile();
        }
        reader.require(XMLStreamConstants.END_ELEMENT, null, "series");
        reader.nextTag();
    }

    private void parseHeader() throws Exception {
        reader.require(XMLStreamConstants.START_ELEMENT, null, "header");
        if (reader.getAttributeCount() > 0) {
            throw new Exception("Attributes are not allowed for header element ");
        }
        reader.nextTag();
        initHeader();
        do {
            detectHeaderElement();
            parseHeaderElement();
        } while (reader.getEventType()  int M = 1 << 2; // multple;
        }

!= XMLStreamConstants.END_ELEMENT);

        if (header.getForecastTime() == Long.MIN_VALUE) header.setForecastTime(startTime);
        if (!Double.isNaN(lat))  private final int flags;
header.setGeometry(new PointGeometry(GeoDatum.WGS_1984.createLatLongZ(lat, lon, z)));
        HeaderElementinitiateTimeStep() {;
            this.flags = 0header.setTimeStep(timeStep);
        }

        HeaderElement(int flags) {if (!qualifiers.isEmpty()) header.setQualifierIds(qualifiers.toArray(new String[qualifiers.size()]));
        if (creationDateText !=  this.flags = flags;null) {
        }

    try {
   public boolean isRequired() {
          long creationTime return= (flags & F.R) != 0fastDateFormat.parseToMillis(creationDateText, creationTimeText);
        }

        public boolean hasAttributes() {
header.setCreationTime(creationTime);
            } returncatch (flags & F.A) != 0;ParseException e) {
        }

        publicthrow booleannew isMultipleAllowed() {
       Exception("Can not parse creation date/time " + creationDateText + ' ' + creationTimeText);
     return   (flags & F.M) != 0;}
        }
    }


    //if fastDateFormat(startTime is used to keep track of last time zone and lenient
!= Long.MIN_VALUE && endTime != Long.MIN_VALUE) {
        private FastDateFormat fastDateFormat = FastDateFormattimeSeriesContentHandler.getInstance("yyyy-MM-dd", "HH:mm:ss", DateUtils.GMT, Locale.US, nullsetEstimatedPeriod(new Period(startTime, endTime));

    private boolean invalidHeaderTimeDetected = false;
}
    private HeaderElement currentHeaderElement = null timeSeriesContentHandler.setNewTimeSeriesHeader(header);

    private static final HeaderElement[] HEADER_ELEMENTS = HeaderElement.class.getEnumConstants( reader.require(XMLStreamConstants.END_ELEMENT, null, "header");

    private PiTimeSeriesHeader header = new PiTimeSeriesHeaderreader.nextTag();
    private}

 List<String> qualfiers = new ArrayList<String>();@SuppressWarnings("OverlyLongMethod")
    private long timeStepMillis = 0;
void parseEvent() throws Exception {
       private TimeStepassert timeStepbinaryInputStream == null;
    private long startTime = Long.MIN_VALUE;
reader.require(XMLStreamConstants.START_ELEMENT, null, "event");

       private longString endTimedateText = Long.MIN_VALUEnull;
    private float missingValue = Float.NaN;
    private String creationDateTexttimeText = null;
       private String creationTimeTextvalueText = null;

    private TimeSeriesContentHandler timeSeriesContentHandler = null;

    /**
String flagText = null;
  * For performance reasions the pi timeString seriesflagSource format= alllowsnull;
 that the values are stored in
  String comment = *null;
 a separate bin file instead of embedded inString theuser xml file.= null;

     *   Thefor bin(int filei should= have0, samen name as the xml file except the extension equals bin
= reader.getAttributeCount(); i < n; i++) {
        * In this case allString timelocalName series should be equidistant.
= reader.getAttributeLocalName(i);
       */
    private VirtualInputDirString virtualInputDirattributeValue = VirtualInputDir.NONEreader.getAttributeValue(i);
    private InputStream binaryInputStream = null;
    private byte[] byteBuffer = null;if (dateText == null && TextUtils.equals(localName, "date")) {
    private   float[] floatBuffer = null;
    private int bufferPosdateText = 0attributeValue;
     private int bufferCount = 0;

   } privateelse XMLStreamReaderif reader(timeText == null; && TextUtils.equals(localName, "time")) {
    private  String virtualFileName = null;

    private static boolean lenienttimeText = falseattributeValue;

    /**
     * For backwards compatibility.} Earlierelse versionsif of(valueText the== PiTimeSeriesParsernull were tollerant about the date/time format
&& TextUtils.equals(localName, "value")) {
       * and the case insensitive for header element names.
 valueText = attributeValue;
  * This parser should not accept files that are not valid} accordingelse to pi_timeseries.xsd
     * When old adapters are not working you can UseLenientPiTimeSeriesParser temporaray till the adapter is fixed
     *if (flagText == null && TextUtils.equals(localName, "flag")) {
                flagText = attributeValue;
     * @param lenient
     */
} else if (flagSource public== staticnull void setLenient(boolean lenient&& TextUtils.equals(localName, "flagSource")) {
        PiTimeSeriesParser.lenient = lenient;
    }

  flagSource  public PiTimeSeriesParser() {= attributeValue;
        fastDateFormat.setLenient(lenient);
    }

 else if (comment @Override
== null   public void parse(XMLStreamReader reader, String virtualFileName, TimeSeriesContentHandler timeSeriesContentHandler) throws Exception && TextUtils.equals(localName, "comment")) {
        this.reader = reader;
       comment this.virtualFileName = virtualFileNameattributeValue;
        this.timeSeriesContentHandler = timeSeriesContentHandler;

  } else if (user == null String virtualBinFileName = FileUtils.getPathWithOtherExtension(virtualFileName&& TextUtils.equals(localName, "binuser"));
 {
        // time zone can be overruled by one oruser more= timeattributeValue;
 zone elements in the pi file
      }  this.fastDateFormat.setTimeZone(timeSeriesContentHandler.getDefaultTimeZone());

else {
         if (!virtualInputDir.exists(virtualBinFileName)) {
     throw new Exception("Unknown attribute " +  parse(localName + " in event");
            return;}
        }

        if binaryInputStream(timeText == virtualInputDir.getInputStream(virtualBinFileNamenull);
        try {
   throw new Exception("Attribute time is missing");

        if (byteBufferdateText == null) {
            throw new Exception("Attribute  byteBuffer = new byte[BUFFER_SIZE * NumberType.FLOAT_SIZE];date is missing");

        if (valueText == null)
            throw new   floatBuffer = new float[BUFFER_SIZE];Exception("Attribute value is missing");

        putValue(dateText, timeText, valueText, flagText, }
flagSource, comment, user);
          parsereader.nextTag();
        reader.require(XMLStreamConstants.END_ELEMENT, null, "event");
  boolean eof = bufferPos == bufferCount && binaryInputStream.readreader.nextTag() == -1;
    }

    private void putValue(String dateText, if (!eof)
                throw new IOException("More values available in bin file than expected based on time step and start and end time\n" + FileUtils.getPathWithOtherExtension(virtualFileName, "bin"));

String timeText, String valueText, String flagText, String flagSource, String comment, String user) throws Exception {
        try {
            timeSeriesContentHandler.setTime(fastDateFormat.parseToMillis(dateText, timeText));
        } finally catch (ParseException e) {
            bufferPosthrow = 0;
            bufferCount = 0;
   new Exception("Can not parse " + dateText + ' ' + timeText);
        }

        if binaryInputStream.close();(flagText == null) {
            binaryInputStream = nulltimeSeriesContentHandler.setFlag(0);
        } else {
    }

    private void parse() throws Exceptiontry {
        reader.require(XMLStreamConstants.START_DOCUMENT, null, null);
        reader.nextTag()timeSeriesContentHandler.setFlag(TextUtils.parseInt(flagText));
        reader.require(XMLStreamConstants.START_ELEMENT, null, "TimeSeries");
  } catch (NumberFormatException e) {
  reader.nextTag();

        while (reader.getEventType() != XMLStreamConstants.END_ELEMENT) {
  throw new Exception("Flag should be an integer " +  parseTimeZone(flagText);
            readTimeSeries();}
        }

        readertimeSeriesContentHandler.require(XMLStreamConstants.END_ELEMENT, null, "TimeSeries"setComment(comment);
        readertimeSeriesContentHandler.nextsetUser(user);
        readertimeSeriesContentHandler.require(XMLStreamConstants.END_DOCUMENT, null, nullsetFlagSource(flagSource);

     }

   try private{
  void readTimeSeries() throws Exception {
      float value reader= TextUtils.require(XMLStreamConstants.START_ELEMENT, null, "series"parseFloat(valueText);
            reader.nextTag();
        parseHeader();
        if (binaryInputStream == null) {// we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
            whileif (reader.getEventType()value == XMLStreamConstants.START_ELEMENT && TextUtils.equals(reader.getLocalName(), "event")) missingValue) {
                value  parseEvent()= Float.NaN;
            }

 else {
               if timeSeriesContentHandler.setValueResolution(readerTextUtils.getEventTypegetValueResolution() == XMLStreamConstants.START_ELEMENT) {valueText, '.'));
            }
    // skip comment
      timeSeriesContentHandler.setValue(value);
          reader.require(XMLStreamConstants.START_ELEMENT, null, "comment"   timeSeriesContentHandler.applyCurrentFields();
        } catch (NumberFormatException e) {
    reader.getElementText();
        throw new Exception("Value should be a float " + reader.nextTag(valueText);
        }
    }

    private long parseTime() throws }Exception else {
        String dateText =  readValuesFromBinFile(reader.getAttributeValue(null, "date");
        }if (dateText == null) {
        reader.require(XMLStreamConstants.END_ELEMENT, null, "series");
    throw new Exception("Attribute " + currentHeaderElement +
            reader.nextTag();
    }

    private"-date voidis parseHeader() throws Exception {
missing");
        }
        String timeText = reader.require(XMLStreamConstants.START_ELEMENT, getAttributeValue(null, "headertime");
        if (reader.getAttributeCount() > 0timeText == null) {
            throw new Exception("Attributes are not allowed for header element ");
"Attribute " + currentHeaderElement +
              }
      "-time  reader.nextTag();
is missing");
        }

        long initHeader()time;
        dotry {
            time = detectHeaderElement(fastDateFormat.parseToMillis(dateText, timeText);
        } catch (ParseException  parseHeaderElement();
e) {
            }throw whilenew (reader.getEventType() != XMLStreamConstants.END_ELEMENT);

Exception("Not a valid data time for "
            if (header.getForecastTime() == Long.MIN_VALUE) header.setForecastTime(startTime);
    + currentHeaderElement + ' initiateTimeStep();
' + dateText + ' ' +  header.setTimeStep(timeSteptimeText, e);
        if (!qualfiers.isEmpty()) header.setQualifierIds(qualfiers.toArray(new String[qualfiers.size()])); }

        if (creationDateText != null) {reader.nextTag();
        return time;
   try {}

    private long parseTimeStep() throws Exception {
        longString creationTimeunit = fastDateFormatreader.parseToMillisgetAttributeValue(creationDateTextnull, creationTimeText"unit");
        if (unit == null) {
    header.setCreationTime(creationTime);
        throw new Exception("Attribute unit is missing }in catch" (ParseException+ ecurrentHeaderElement);
 {
       }

        TimeUnit throwtu new= Exception("Can not parse creation date/time " + creationDateText + ' ' + creationTimeText);TimeUnit.get(unit);
        if (tu != null) {
            }
String multiplierText = reader.getAttributeValue(null, "multiplier");
    }
        timeSeriesContentHandler.setNewTimeSeriesHeader(header)int multiplier;
        if (startTime != Long.MIN_VALUE && endTime != Long.MIN_VALUE if (multiplierText == null) {
                timeSeriesContentHandler.setEstimatedPeriod(new Period(startTime, endTime))multiplier = 1;
        }

    } else {
   reader.require(XMLStreamConstants.END_ELEMENT, null, "header");
        reader.nextTag();
   try }{

      private void parseEvent() throws Exception {
        assert binaryInputStreammultiplier == nullInteger.parseInt(multiplierText);
        reader.require(XMLStreamConstants.START_ELEMENT, null, "event");
      }  String timeText = reader.getAttributeValue(null, "time");
 catch (NumberFormatException e) {
       String dateText = reader.getAttributeValue(null, "date");
        String valueTextthrow =new readerException(ExceptionUtils.getAttributeValuegetMessage(nulle), "value"e);
        String flagText = reader.getAttributeValue(null, "flag");
    }

    String commentText = reader.getAttributeValue(null, "comment");

        if (timeTextmultiplier == null0) {
                    throw new Exception("AttributeMultiplier time is missing0");

        if (dateText == null)
     }
       throw new Exception("Attribute date is missing");

        if (valueText == null)
}

            String throwdividerText new= Exception("Attribute value is missingreader.getAttributeValue(null, "divider");

            tryint {divider;
            if timeSeriesContentHandler.setTime(fastDateFormat.parseToMillis(dateText, timeText));
(dividerText == null) {
         }   catch (ParseException e) {
 divider = 1;
         throw new Exception("Can not} parseelse "{
 + dateText + ' ' + timeText);
        }

 try {
      if (flagText == null) {
          divider = timeSeriesContentHandlerInteger.setFlagparseInt(0dividerText);
        } else {
      } catch (NumberFormatException e) {
  try    {
              throw new timeSeriesContentHandler.setFlagException(TextUtilsExceptionUtils.parseIntgetMessage(flagTexte), e);
                }

  catch   (NumberFormatException e) {
         if (divider == 0) {
   throw new Exception("Flag should be an integer " + flagText);
        throw new Exception("divider is }0");
        }
        timeSeriesContentHandler.setComment(commentText);
}
        try {
   }
         float value = TextUtilsreader.parseFloatnextTag(valueText);
            // we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
return tu.getMillis() * multiplier / divider;
        } else {
            reader.nextTag();
   if (value == missingValue) {
     return 0;
        }
  value = Float.NaN; }

    private void initHeader() {
      } else {
 header.clear();
        header.setFileDescription(virtualFileName);
         timeSeriesContentHandler.setValueResolution(TextUtils.getValueResolution(valueText, '.'));
currentHeaderElement = null;
        timeStep = null;
    }
    timeStepMillis = 0;
      timeSeriesContentHandler.setValue(value)  startTime = Long.MIN_VALUE;
        endTime =   timeSeriesContentHandler.applyCurrentFields()Long.MIN_VALUE;
        }missingValue catch (NumberFormatException e) {
= Float.NaN;
        creationDateText = null;
    throw new Exception("Value should becreationTimeText a float " + valueText)= "00:00:00";
        }
qualifiers.clear();
        lat = readerDouble.nextTag()NaN;
        reader.require(XMLStreamConstants.END_ELEMENT, null, "event");
lon = Double.NaN;
        z reader.nextTag();= Double.NaN; 
    }

    private longvoid parseTimereadValuesFromBinFile() throws Exception {
        StringTimeStep dateTexttimeStep = readerheader.getAttributeValue(null, "date"getTimeStep();
        if (dateText == null!timeStep.isRegular()) {
            throw new Exception("Attribute " + currentHeaderElement +
Only equidistant time step supported when pi events are stored in bin file instead of xml");
        }

        boolean equidistantMillis = timeStep.isEquidistantMillis();
        long stepMillis = equidistantMillis ?  timeStep.getStepMillis() : Long.MIN_VALUE;
       "-date is missing");
        }
for (long time = startTime; time <= endTime;) {
         String timeText = readertimeSeriesContentHandler.getAttributeValuesetTime(null, "time");
            if (timeTextbufferPos == nullbufferCount) {fillBuffer();
            throwfloat new Exception("Attribute " + currentHeaderElement +value = floatBuffer[bufferPos++];
                    "-time is missing");
        }

        long time;
        try {
// we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
            if (value == missingValue) timevalue = fastDateFormat.parseToMillis(dateText, timeText)Float.NaN;
         }  catch (ParseException e) {timeSeriesContentHandler.setValue(value);
            throw new Exception("Not a valid data time for "
timeSeriesContentHandler.applyCurrentFields();
            if (equidistantMillis) {
            + currentHeaderElement + ' 'time + dateText + ' ' + timeText, e)= stepMillis;
        }

        reader.nextTag()continue;
        return time;
    }

    private long parseTimeStep() throws Exception {
        String unittime = readertimeStep.getAttributeValue(null, "unit"nextTime(time);
        if}
 (unit == null) {}

    private void fillBuffer() throws IOException {
   throw new Exception("Attribute unit is missingint inbyteBufferCount " + currentHeaderElement)= 0;
        }

while (byteBufferCount       TimeUnit tu = TimeUnit.get(unit);
        if (tu != null% NumberType.FLOAT_SIZE != 0 || byteBufferCount == 0) {
            Stringint multiplierTextcount = readerbinaryInputStream.getAttributeValue(null, "multiplier"read(byteBuffer, byteBufferCount, BUFFER_SIZE * NumberType.FLOAT_SIZE - byteBufferCount);
            assert count int!= multiplier;0; // see read javadoc
            if (multiplierTextcount == null-1) {
throw new EOFException("Bin file is too short");
          multiplier  byteBufferCount += 1count;
        }
      }  bufferCount else= {
byteBufferCount / NumberType.FLOAT_SIZE;
        BinaryUtils.copy(byteBuffer, 0, byteBufferCount, floatBuffer, 0,  try {bufferCount, ByteOrder.LITTLE_ENDIAN);
        bufferPos = 0;
    }

    private  multiplier = Integer.parseInt(multiplierText);void initiateTimeStep() {
        if (timeStepMillis == 0)  {
   } catch (NumberFormatException e) {
     timeStep = IrregularTimeStep.INSTANCE;
            return;
 throw new Exception(ExceptionUtils.getMessage(e), e);
    }

        long startTime =  }

this.startTime ==  Long.MIN_VALUE ? 0L : this.startTime;
        if (timeStepMillis if (multiplier ==% TimeUnit.SECOND_MILLIS != 0) {
            timeStep = RelativeEquidistantTimeStep.getInstance(timeStepMillis, startTime);
        throw new Exception("Multiplier is 0")return;
        }

        long timeZoneOffsetMillis = -startTime % }timeStepMillis;
        if (timeZoneOffsetMillis %  }

TimeUnit.MINUTE_MILLIS != 0) {
            String dividerTexttimeStep = readerRelativeEquidistantTimeStep.getAttributeValuegetInstance(nulltimeStepMillis, "divider"startTime);
            int dividerreturn;
        }

       if (dividerTexttimeStep == null) { SimpleEquidistantTimeStep.getInstance(timeStepMillis, timeZoneOffsetMillis);
    }

    private void parseTimeZone() throws Exception {
   divider = 1;
   if (reader.getEventType() != XMLStreamConstants.START_ELEMENT) return;
     } else {
 if (!TextUtils.equals(reader.getLocalName(),    "timeZone")) return;
          try {
            String        divider timeZoneText = Integerreader.parseIntgetElementText(dividerText);
            // element name="timeZone" type="fews:TimeZoneSimpleType" default="0.0" minOccurs="0"/>
   } catch (NumberFormatException e) {
     // when default is used in schema for element the consequence is that empty strings throw new Exception(ExceptionUtils.getMessage(e), e);are allowed
            double offset =  }

   timeZoneText.isEmpty() ? 0d : Double.parseDouble(timeZoneText);
            TimeZone iftimeZoneFromDouble (divider == 0) {TimeZoneUtils.createTimeZoneFromDouble(offset);
            this.fastDateFormat.setTimeZone(timeZoneFromDouble);

        throw} newcatch Exception("dividplierNumberFormatException is 0");e) {
            throw new   }
   Exception("Not valid timeZone format", e);
         }
            reader.nextTag(require(XMLStreamConstants.END_ELEMENT, null, "timeZone");
            return tu.getMillis() * multiplier / dividerreader.nextTag();
    }

    } else {@SuppressWarnings({"OverlyLongMethod"})
    private        reader.nextTagvoid parseHeaderElement();
 throws Exception {
        switch return 0;
(currentHeaderElement) {
         }
   case }type:

     private void initHeader() {
        header.clear(.setParameterType(parseType(reader.getElementText()));
         header.setFileDescription(virtualFileName);
       break;
 currentHeaderElement = null;
        timeStep =case null;locationId:
         timeStepMillis  = 0;
    // see FEWS-9858, when startTimethere = Long.MIN_VALUE;
        endTime = Long.MIN_VALUE;is no location id the time series are assigned to all locations
        missingValue  = Float.NaN;
     // this is creationDateTexta =flaw null;
in the pi_timeSeries.xsd, the location element is required creationTimeTextbut = "00:00:00";
  is allowed empty strings
      qualfiers.clear();
    }

    private void readValuesFromBinFile() throws Exception {
 header.setLocationId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none"));
             TimeStep timeStep = header.getTimeStep()break;
        if (!timeStep.isRegular()) {    case parameterId:
            throw new Exception("Only equidistant time// step supportedsee FEWS-9858, when pithere eventsis areno storedparameter inid binthe filetime insteadseries of xml");
     are assigned to all locations
   }

        boolean equidistantMillis = timeStep.isEquidistantMillis();
  // this is a flaw in long stepMillis = equidistantMillis ?  timeStep.getStepMillis() : Long.MIN_VALUE;
   the pi_timeSeries.xsd, the location element is required but is allowed empty strings
     try {
          header.setParameterId(TextUtils.defaultIfNull(TextUtils.trimToNull(reader.getElementText()), "none"));
 for (long time = startTime; time <= endTime;) {
       break;
         timeSeriesContentHandler.setTime(time);
   case qualifierId:
            if (bufferPos == bufferCount) fillBuffer qualifiers.add(reader.getElementText());
                float value = floatBuffer[bufferPos++]break;
            case ensembleId:
   // we can not use the automatic missing value detection of the content handler because the missing value is different for each time series
 header.setEnsembleId(reader.getElementText());
                break;
          if (value == missingValue) value = Float.NaN;case ensembleMemberIndex:
                timeSeriesContentHandler.setValue(valueheader.setEnsembleMemberIndex(parseEnsembleMemberIndex(reader.getElementText()));
                break;
     timeSeriesContentHandler.applyCurrentFields();
       case ensembleMemberId:
        if (equidistantMillis) {
      header.setEnsembleMemberId(reader.getElementText());
              time += stepMillisbreak;
            case timeStep:
       continue;
         timeStepMillis = parseTimeStep();
       }
         break;
       time = timeStep.nextTime(time);
   case startDate:
        }
        }startTime catch= parseTime(IOException e);
 {
            throw new Exception(ExceptionUtils.getMessage(e), e) break;
         }
   case }
endDate:
    private void fillBuffer()  throws IOException {
        int byteBufferCountendTime = 0parseTime();
        while (byteBufferCount % NumberType.FLOAT_SIZE != 0 || byteBufferCount == 0) { break;
            intcase countforecastDate:
 = binaryInputStream.read(byteBuffer, byteBufferCount, BUFFER_SIZE * NumberType.FLOAT_SIZE - byteBufferCount);
        header.setForecastTime(parseTime());
    assert count != 0; // see read javadoc
     break;
       if (count == -1) throw new EOFException("Bin file is too short");
case missVal:
                byteBufferCountmissingValue += countparseValue(reader.getElementText());
        }        break;
        bufferCount  = byteBufferCount / NumberType.FLOAT_SIZE;
case longName:
         BinaryUtils.copy(byteBuffer, 0, byteBufferCount, floatBuffer, 0, bufferCount, ByteOrder.LITTLE_ENDIAN       header.setLongName(reader.getElementText());
        bufferPos   = 0;
    }break;

     private void initiateTimeStep() {
    case stationName:
   timeStep = IrregularTimeStep.INSTANCE; //default timestep

        if (timeStepMillis == 0) {header.setLocationName(reader.getElementText());
            return;
    break;
    }
        if (timeStepMillis % TimeUnit.MINUTE_MILLIS != 0) {
case units:
               if header.setUnit(!this.invalidHeaderTimeDetected) {reader.getElementText());
                if (log.isDebugEnabled()) log.debug("Header timestep and/or start time has not rounded  minutes ! Irregular timestep wil be used.");
break;
            case sourceOrganisation:
                header.setSourceOrganisation(reader.getElementText());
    this.invalidHeaderTimeDetected = true;
            }break;
            timeStepMillis = 0;case sourceSystem:
            return;
    header.setSourceSystem(reader.getElementText());
    }

        long timeZoneOffsetMillis = -startTime % timeStepMillisbreak;
        if   (timeZoneOffsetMillis % TimeUnit.MINUTE_MILLIS != 0) {
case fileDescription:
               if header.setFileDescription(!this.invalidHeaderTimeDetected) {reader.getElementText());
                if (log.isDebugEnabled()) log.debug("Header timestep and/or start time has not rounded  minutes ! Irregular timestep wil be used.");break;
            case creationDate:
                this.invalidHeaderTimeDetectedcreationDateText = truereader.getElementText();
            }
    break;
        timeStepMillis  = 0;
 case creationTime:
          return;
      creationTimeText  }= reader.getElementText();
        timeStep  = SimpleEquidistantTimeStep.getInstance(timeStepMillis, timeZoneOffsetMillis);
    }
break;
    private void parseTimeZone() throws Exception {
   case region:
    if (reader.getEventType() != XMLStreamConstants.START_ELEMENT) return;
        if (!TextUtils.equalsheader.setRegion(reader.getLocalName(), "timeZone"getElementText()) return;
               try {break;
            double offsetcase = Double.parseDouble(reader.getElementText());
thresholds:
             TimeZone timeZoneFromDouble = TimeZoneUtils.createTimeZoneFromDoubleparseThresholds(offset);
                this.fastDateFormat.setTimeZone(timeZoneFromDouble)break;

         } catch (NumberFormatException e)case {lat:
            throw new Exception("Not valid timeZonelat format", e= parseValue(reader.getElementText());
        }
        break;
   reader.require(XMLStreamConstants.END_ELEMENT, null, "timeZone");
        reader.nextTag();
case lon:
       }

     @SuppressWarnings({"OverlyLongMethod"})
    privatelon void parseHeaderElement() throws Exception {
= parseValue(reader.getElementText());
              switch (currentHeaderElement) {break;
            case typex:
                header.setParameterType(parseType(reader.getElementText()));
                break;
            case locationIdy:
                header.setLocationId(reader.getElementText());
                break;
            case parameterIdz:
                header.setParameterIdz = parseValue(reader.getElementText());
                break;
        }
     case qualifierId:
   reader.require(XMLStreamConstants.END_ELEMENT, null, currentHeaderElement.name());
        reader.nextTag();
    }

    private void parseThresholds() throws XMLStreamException {
        qualfiers.add(reader.getElementTextnextTag());
        ArrayList<String> ids =      breaknew ArrayList<>();
        ArrayList<String> names =  case ensembleId:new ArrayList<>();
        ArrayList<String> stringValues =      header.setEnsembleId(reader.getElementText())new ArrayList<>();
        ArrayList<String> groupIds = new ArrayList<>();
    break;
    ArrayList<String> groupNames = new ArrayList<>();
    case  ensembleMemberIndex:
  do {
            if header.setEnsembleMemberIndex(parseEnsembleMemberIndex(reader.getElementTextgetEventType()));
            == XMLStreamConstants.START_ELEMENT) {
    break;
            case timeStep:ids.add(reader.getAttributeValue(null, "id"));
                timeStepMillis = parseTimeStep(names.add(reader.getAttributeValue(null, "name"));
                breakstringValues.add(reader.getAttributeValue(null, "value"));
            case startDate:
   groupIds.add(reader.getAttributeValue(null, "groupId"));
              startTime = parseTime() groupNames.add(reader.getAttributeValue(null, "groupName"));
            }
    break;
        reader.nextTag();
    case endDate:
     } while (!reader.getLocalName().equals(currentHeaderElement.name()));
        float[] endTimevalues = parseTime new float[stringValues.size()];
        for (int i = 0; i < values.length;  break;
i++) {
            values[i] case forecastDate:= Float.parseFloat(stringValues.get(i));
        }

        header.setForecastTime(parseTime());setHighLevelThresholds(ids.toArray(new String[ids.size()]), names.toArray(new String[names.size()]), values,
                breakgroupIds.toArray(new String[groupIds.size()]), groupNames.toArray(new String[groupNames.size()]));
    }

    private void parseProperties() throws caseXMLStreamException missVal:{
        reader.require(XMLStreamConstants.START_ELEMENT, null, "properties");
      missingValue = parseMissingValue(reader.getElementTextnextTag());
                breakpropertyBuilder.clear();
        while    case longName:(!TextUtils.equals(reader.getLocalName(), "properties")){
                header.setLongNameif (reader.getElementTextgetEventType());
 != XMLStreamConstants.START_ELEMENT) {
             break;
   // eg <int       case stationName:key="a" value=12><int>
                header.setLocationName(reader.getElementTextnextTag());
                breakcontinue;
            case units:}
            String key =  header.setUnit(reader.getElementText());
                break;
            case sourceOrganisation:getAttributeValue(null, "key");
            String value  = header.setSourceOrganisation(reader.getElementText()getAttributeValue(null, "value");
            String date =  breakreader.getAttributeValue(null, "date");
            caseString sourceSystem:
time = reader.getAttributeValue(null, "time");
            switch header.setSourceSystem(reader.getElementTextgetLocalName()); {
                case "string" break;:
            case fileDescription:
       propertyBuilder.addString(key, value);
        header.setFileDescription(reader.getElementText());
            break;
    break;
            case creationDate"int":
                creationDateText = reader.getElementText(    propertyBuilder.addInt(key, TextUtils.parseInt(value));
                    break;
                case creationTime"float":
                creationTimeText = reader.getElementText(    propertyBuilder.addFloat(key, TextUtils.parseFloat(value));
                    break;
                case region"double":
                    headerpropertyBuilder.setRegion(reader.getElementText(addDouble(key, TextUtils.parseDouble(value));
                    break;
                case thresholds"bool":
                     parseThresholds();
propertyBuilder.addBoolean(key, Boolean.parseBoolean(value));
                    break;
        }
        reader.require(XMLStreamConstants.END_ELEMENT, null, currentHeaderElement.name());case "dateTime":
        reader.nextTag();
    }

    private void parseThresholds() throws XMLStreamExceptiontry {
             reader.nextTag();
        ArrayList<String> ids = new ArrayList<String>() propertyBuilder.addDateTime(key, fastDateFormat.parseToMillis(date, time));
        ArrayList<String> names = new ArrayList<String>();
        ArrayList<String> stringValues = new ArrayList<String>()break;
        do {
           } ifcatch (reader.getEventType() == XMLStreamConstants.START_ELEMENTParseException e) {
                String id = reader.getAttributeValue(null, "id");
          throw new XMLStreamException("Invalid date time "+ Stringdate + name' = reader.getAttributeValue(null, "name"' + time);
                String stringValue = reader.getAttributeValue(null, "value"); }
                ids.add(id);default:
                names.add(name);
    throw new XMLStreamException("Invalid property type "       stringValues.add(stringValue+ reader.getLocalName());
            }
            reader.nextTag();
        } while (!reader.getLocalName().equals(currentHeaderElement.name()))

        timeSeriesContentHandler.setProperties(propertyBuilder.build());

        reader.require(XMLStreamConstants.END_ELEMENT, null, "properties");
        float[] values = new float[stringValues.sizereader.nextTag()];
    }

    forprivate static float parseValue(intString igotString) =throws 0;Exception i{
 < values.length; i++) {
    // <element name="missVal" type="double" default="NaN">
    values[i] = Float.valueOf(stringValues.get(i));
   // when default is used }
in schema for element the consequence is  header.setHighLevelThresholds(ids.toArray(new String[ids.size()]), names.toArray(new String[names.size()]), values);
    }

that empty strings are allowed 
      private static floatif parseMissingValue(String gotString.isEmpty()) throws Exception {return Float.NaN;
        try {
            return TextUtils.parseFloat(gotString);
        } catch (NumberFormatException e) {
            throw new Exception(ExceptionUtils.getMessage(e), e);
        }
    }

    private static int parseEnsembleMemberIndex(String gotString) throws Exception {
        int index = Integer.parseInt(gotString);
        if (index < 0) {
            throw new Exception("Negative ensemble member index not allowed " + gotString);
        }
        return index;
    }

    private static ParameterType parseType(String gotString) throws Exception {
        ParameterType type = ParameterType.get(gotString);
        if (type == null) {
            throw new Exception("Type in header should be instantaneous or accumulative and not " + gotString);
        }
        return type;
    }

    private void detectHeaderElement() throws Exception {
        if (reader.getEventType() != XMLStreamConstants.START_ELEMENT)
            throw new Exception("header element expected");

        String localName = reader.getLocalName();
        HeaderElement element;
        try {
            element = Enum.valueOf(HeaderElement.class, localName);
;
        try {
       assert element != null; // contractelement of= Enum.valueOf(HeaderElement.class, localName);
        } catch (Exception e) {
            throw new Exception("Unknown header element: " + localName);
        }

        if (currentHeaderElement == element && currentHeaderElement.isMultipleAllowed()) return;

        if (currentHeaderElement != null && element.ordinal() < currentHeaderElement.ordinal())) {
            throw new Exception("Header elements in wrong order: " + localName);
        }

        if (currentHeaderElement == HeaderElement.ensembleMemberIndex && element == HeaderElement.ensembleMemberId) {
            throw new Exception("Header elements in wrong order: " + localNameDuplicate header element, both ensembleMemberIndex and ensembleMemberId in header");
        }

        if (currentHeaderElement == element) {
            throw new Exception("Duplicate header element: " + localName);
        }

        if (reader.getAttributeCount() > 0 && !element.hasAttributes()) {
            throw new Exception("Attributes are not allowed for header element " + localName);
        }

        int nextOrdinal = currentHeaderElement == null ? 0 : currentHeaderElement.ordinal() + 1;

        // order is correct and no duplicate so currentHeaderElement can not be last header element
        assert nextOrdinal < HEADER_ELEMENTS.length;
        HeaderElement nextHeaderElement = HEADER_ELEMENTS[nextOrdinal];
        if (nextHeaderElement.isRequired() && nextHeaderElement != element) {
            throw new Exception("Required header item missing: " + nextHeaderElement);
        }

        currentHeaderElement = element;
    }

    public TimeZone getTimeZone() {
        return fastDateFormat.getTimeZone();
    }

    @SuppressWarnings("UnusedDeclaration")
    public PiTimeSeriesSerializer.EventDestination getEventDestination() {
        return eventDestination;
    }

    @SuppressWarnings("UnusedDeclaration")
    public float getMissingValue() {
        return missingValue;
    }
}
Page tree

Versions Compared

Old Version 3

New Version 4

Key
