package nl.wldelft.timeseriesparsers;
import nl.wldelft.util.FastDateFormat;
import nl.wldelft.util.Properties;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.TimeZoneUtils;
import nl.wldelft.util.io.XmlParser;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import org.apache.log4j.Logger;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import java.io.IOException;
import java.text.DateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
/**
*
* @autor pelgrim
* @author ekkelenk
* Date: 8/9/13
* Time: 9:35 AM
* Import the warnings from the AifsML format
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
public class AifsMLTimeSeriesParser implements XmlParser<TimeSeriesContentHandler> {
private final static Map<String, String[]> PART_OF_DAY_MAPPINGS = new HashMap<>();
// Property keys
public static final String WARNING_NEXT_ISSUE = "Warning_Next_Issue";
public static final String PREDICTION_LEVEL_TYPE = "Prediction_Level_Type";
public static final String WARNING_LIKELIHOOD = "Warning_Likelihood";
public static final String HYDROGRAPH_TYPE = "Hydrograph_Type";
public static final String TARGET_TIME_TYPE = "Target_Time_Type";
public static final String PART_DAY_VALUE = "Part_Day_Value";
public static final String PREDICTION_LEVEL = "Prediction_Level";
public static final String WARNING_PREDICTION_TEXT = "Warning_Prediction_Text";
public static final String WARNING_TIME_ZONE = "Warning_Timezone";
public static final String WARNING_AREA_TYPE = "river-basin";
public static final String WARNING_ID = "Warning_Id";
// target time types
public static final String TARGET_TIME_TYPE_DATE_TIME = "DateTime";
public static final String TARGET_TIME_TYPE_DAY = "Day";
public static final String TARGET_TIME_TYPE_PART_DAY = "PartDay";
// target time values
public static final String TARGET_TIME_VALUE_OVERNIGHT = "Overnight";
public static final String TARGET_TIME_VALUE_NIGHT = "Night";
public static final String TARGET_TIME_VALUE_LATE = "Late";
public static final String TARGET_TIME_VALUE_EVENING = "Evening";
public static final String TARGET_TIME_VALUE_EARLY_EVENING = "Early Evening";
public static final String TARGET_TIME_VALUE_LATE_EVENING = "Late Evening";
public static final String TARGET_TIME_VALUE_LATE_AFTERNOON = "Late Afternoon";
public static final String TARGET_TIME_VALUE_AFTERNOON = "Afternoon";
public static final String TARGET_TIME_VALUE_EARLY_AFTERNOON = "Early Afternoon";
public static final String TARGET_TIME_VALUE_LATE_MORNING = "Late Morning";
public static final String TARGET_TIME_VALUE_MORNING = "Morning";
public static final String TARGET_TIME_VALUE_EARLY = "Early";
public static final String TARGET_TIME_VALUE_EARLY_MORNING = "Early Morning";
public static final String TARGET_TIME_VALUE_DAY = "Day";
private final static String dateFormatString = "yyyy-MM-dd'T'HH:mm:ss";
private final static String dateFormatStringWithTimeZone = "yyyy-MM-dd'T'HH:mm:ss'Z'";
public static final String WARNING_SEQUENCE = "Warning_Sequence";
private static FastDateFormat dateFormat = null;
static {
// The time ranges of the different par day values in HH:mm:ss format
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_EARLY_MORNING, new String[]{"00:00:00", "06:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_EARLY, new String[] {"00:00:00", "08:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_MORNING, new String[] {"00:00:00", "12:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_LATE_MORNING, new String[] {"09:00:00", "12:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_EARLY_AFTERNOON, new String[] {"12:00:00", "15:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_AFTERNOON, new String[]{"12:00:00", "18:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_LATE_AFTERNOON, new String[] {"15:00:00", "18:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_EARLY_EVENING, new String[] { "18:00:00", "21:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_EVENING, new String[] {"18:00:00", "24:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_LATE_EVENING, new String[]{"21:00:00", "24:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_LATE, new String[] {"18:00:00", "24:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_NIGHT, new String[] {"18:00:00", "24:00:00"});
PART_OF_DAY_MAPPINGS.put(TARGET_TIME_VALUE_DAY, new String[] {"00:00:00", "24:00:00"});
}
private static final String HYFS_MARKER = "[HYFS]";
private TimeSeriesContentHandler contentHandler = null;
private XMLStreamReader reader = null;
private DefaultTimeSeriesHeader header = null;
private String warningTitle = null;
private String warningNextIssue = null;
private String productIdentifier = null;
private String externalForecastTime = null;
private String warningSequence=null; // Warning sequence should be used by all area locations.
private static final Logger log = Logger.getLogger(AifsMLTimeSeriesParser.class);
@Override
public void parse(XMLStreamReader reader, String virtualFileName, TimeSeriesContentHandler contentHandler) throws Exception {
clearFieldsFromPreviousParse();
this.reader = reader;
this.contentHandler = contentHandler;
header = new DefaultTimeSeriesHeader();
reader.require(XMLStreamConstants.START_DOCUMENT, null, null);
reader.nextTag();
reader.require(XMLStreamConstants.START_ELEMENT, null, "product");
parseIdentifier();
parseExternalForecastTime();
if (!XmlStreamReaderUtils.goTo(reader, "warning")) {
// There has to be a warning element.
throw new IOException("Input file doesn't contain a warning element: " + virtualFileName);
}
reader.require(XMLStreamConstants.START_ELEMENT, null, "warning");
reader.nextTag();
reader.require(XMLStreamConstants.START_ELEMENT, null, "warning-info");
parseWarningInfo();
parseWarningAreas();
}
private void clearFieldsFromPreviousParse() {
warningTitle = null;
warningNextIssue = null;
productIdentifier = null;
externalForecastTime = null;
}
/**
* Retrieve the identifier from the source element
*
* @throws XMLStreamException;
*/
private void parseIdentifier() throws XMLStreamException {
XmlStreamReaderUtils.goTo(reader, "identifier");
if (TextUtils.equals(reader.getLocalName(), "identifier")) {
this.productIdentifier = reader.getElementText();
}
}
private void parseExternalForecastTime() throws XMLStreamException {
XmlStreamReaderUtils.goTo(reader, "issue-time-utc");
if (TextUtils.equals(reader.getLocalName(), "issue-time-utc")) {
this.externalForecastTime = reader.getElementText();
}
}
/**
* filter the warning sequence and warning title text attributes from the text elements.
* they will be used for all areas.
*
* @throws XMLStreamException
*/
private void parseWarningInfo() throws XMLStreamException {
do {
if (!XmlStreamReaderUtils.goTo(reader, "text", "warning-info")) {
// no more text areas, return.
return;
}
if (this.warningTitle == null || warningNextIssue == null) {
String type = reader.getAttributeValue(null, "type");
if (TextUtils.equals("warning_title", type)) {
this.warningTitle = reader.getElementText();
}
if (TextUtils.equals("warning_next_issue", type)) {
this.warningNextIssue = reader.getElementText();
}
}
if (this.warningTitle != null && this.warningNextIssue != null) {
// we're finished with the text elements, skip to the areas
return;
}
} while (reader.hasNext());
}
/**
* Retrieve the locationId from the area element(s) and call separate method to parse forecast data
*
* @throws XMLStreamException;
*/
private void parseWarningAreas() throws XMLStreamException {
// parse all areas inside a warning element.
warningSequence = null; // Warning sequence should be used by all area locations.
do {
if (!XmlStreamReaderUtils.goTo(reader, "area", "warning")) {
// alle area's have been processed or an end tag warning has been found.
return;
}
String locationId = reader.getAttributeValue(null, "aac");
String areaType = reader.getAttributeValue(null, "type");
// an area can have more that one fore-cast periods.
parseForecastData(locationId, areaType);
} while (reader.hasNext());
}
/**
* helper class to store forecast data per area.
*/
private static class ForecastData {
private Properties.Builder propertiesBuilder = new Properties.Builder();
private String parameter = null;
private String value = null;
private String localTime = null;
private String comment = null;
private Long timeRangeTime = null;
private Long timeRangeStart = null;
private Long timeRangeEnd = null;
private String valueRangeStart = null;
private String valueRangeEnd = null;
}
/**
* This method skips elements until a start element with the corresponding local name is found and returns true,
* If an end tag with name endName is found, return false.
* Otherwise returns false
* Warning: if the element is not found and the end tag is not found, the reader is skipped until the end of the document
*
* @param reader the xml reader
* @param localName local name to look for
* @param endName local name of end tag to find.
* @return boolean return true if localName cannot be found or the endName end tag has been found.
* @throws XMLStreamException
*/
private static boolean goTo(XMLStreamReader reader, Set<String> localNameSet, String endName) throws XMLStreamException {
do {
if (!reader.hasNext()) return false;
if (reader.isEndElement() && TextUtils.equals(reader.getLocalName(), endName)) return false;
reader.next();
} while (!reader.isStartElement() || !localNameSet.contains(reader.getLocalName()) );
return true;
}
/**
* Retrieve the variable, value, unit, time and comment from the forecast data element(s)
* Retrieve comment text from forecast text element
* return true if more forecast periods are available
*
* @throws XMLStreamException;
*/
private void parseForecastData(String locationId, String areaType) throws XMLStreamException {
// not every area has a forecast-period. Try to find the next forecast-period,
// but stop if not found, or an end-tag area has been found.
Set<String> forecastPeriodSubElements = new HashSet<>();
// We're interested in text end element sub elements inside the forecast period.
boolean isRiverBasin = WARNING_AREA_TYPE.equals(areaType);
forecastPeriodSubElements.add("text");
forecastPeriodSubElements.add("element");
List<ForecastData> forecastDataList = new ArrayList<>();
String warningPrediction = null; // There is only one warningPrediction per area.
while (XmlStreamReaderUtils.goTo(reader, "forecast-period", "area")) {
String predictionLevelValue = null;
String commentText = null;
String predictionLowerValueString = null;
String predictionUpperValueString = null;
String time = null;
String targetTimeType = null;
String partDayValueString = null;
Properties.Builder propertiesBuilder = new Properties.Builder();
// inside the forecast-period we need to process the text elements first.
do {
// process all elements until we reach the end tag forecast-period.
if (!goTo(reader, forecastPeriodSubElements, "forecast-period")) {
// break so we can write this forecast as a timeseries.
break;
}
String elementName = reader.getLocalName();
String type = reader.getAttributeValue(null, "type");
if (isRiverBasin && TextUtils.equals("element", elementName) && TextUtils.equals("warning_sequence", type)) warningSequence = reader.getElementText();
if (TextUtils.equals("text", elementName) && TextUtils.equals("warning_prediction", type)) warningPrediction = reader.getElementText();
if (TextUtils.equals("text", elementName) && TextUtils.equals("river_level_prediction", type) ) {
String riverLevelPredictionText = reader.getElementText();
int index = riverLevelPredictionText.indexOf(HYFS_MARKER);
if (index != -1) {
String levelPredictionPart = riverLevelPredictionText.substring(0, index);
//noinspection StringConcatenationMissingWhitespace
String fieldSeparatorRiverLevelPrediction = "\\[FFWDEL" +
"IM]";
String[] splits = levelPredictionPart.split(fieldSeparatorRiverLevelPrediction);
if (splits.length > 0 && splits[0] != null && !splits[0].isEmpty()) {
propertiesBuilder.addString(PREDICTION_LEVEL_TYPE, splits[0].trim());
}
if (splits.length > 2 && splits[2] != null) {
if (!splits[2].isEmpty() && !"##".equals(splits[2])) {
try {
Float.parseFloat(splits[2]);
predictionLowerValueString = splits[2];
} catch (NumberFormatException nfe) {
predictionLowerValueString = null;
}
}
}
if (splits.length > 3 && splits[3] != null) {
if (!splits[3].isEmpty() && !"##".equals(splits[3])) {
try {
Float.parseFloat(splits[3]);
predictionUpperValueString = splits[3];
} catch (NumberFormatException nfe) {
predictionUpperValueString = null;
}
}
}
if (splits.length >4 && splits[4] != null && !splits[4].isEmpty()) {
propertiesBuilder.addString(PREDICTION_LEVEL, splits[4].trim());
}
String valueComment = riverLevelPredictionText.substring(index + HYFS_MARKER.length());
int split = valueComment.indexOf(",");
if (split != -1) {
predictionLevelValue = valueComment.substring(0, split);
commentText = valueComment.substring(split + 1);
try {
Float.parseFloat(predictionLevelValue);
} catch (NumberFormatException nfe) {
log.warn("Import.Warn: Unparsable prediction level value in river_level_prediction: " + predictionLevelValue);
predictionLevelValue = null;
}
if (warningPrediction != null) {
propertiesBuilder.addString(WARNING_PREDICTION_TEXT, warningPrediction.trim());
}
}
if (productIdentifier != null) {
propertiesBuilder.addString(WARNING_ID, productIdentifier);
}
}
} else if (TextUtils.equals("text", elementName) && TextUtils.equals("hydrograph_type", type)) {
String hydroGraphType = reader.getElementText();
propertiesBuilder.addString(HYDROGRAPH_TYPE, hydroGraphType);
} else if (TextUtils.equals("element", elementName) && TextUtils.equals("warning_likelihood", type)) {
String warningLikelihood = reader.getElementText();
propertiesBuilder.addString(WARNING_LIKELIHOOD, warningLikelihood);
} else if (TextUtils.equals("element", elementName) && TextUtils.equals("time", type)) {
String timeString = reader.getElementText();
int index = timeString.indexOf(HYFS_MARKER);
if (index != -1) {
String timeStringSeparator = "\\[FFWDELIM]";
String timeStringPart = timeString.substring(0, index);
String[] splits = timeStringPart.split(timeStringSeparator);
if (splits.length > 0 && splits[0] != null && !splits[0].isEmpty()) {
propertiesBuilder.addString(TARGET_TIME_TYPE, splits[0]);
targetTimeType = splits[0];
}
if (targetTimeType != null && targetTimeType.equals("PartDay") && splits.length > 4 && splits[4] != null && !splits[4].isEmpty()) {
propertiesBuilder.addString(PART_DAY_VALUE, splits[4]);
partDayValueString = splits[4];
}
time = timeString.substring(index + HYFS_MARKER.length());
}
// we found a river_level_prediction.
if (time != null && predictionLevelValue != null) {
if (warningSequence != null) propertiesBuilder.addString(WARNING_SEQUENCE, warningSequence);
if (validWarningNextIssue(warningNextIssue)) propertiesBuilder.addString(WARNING_NEXT_ISSUE, warningNextIssue);
propertiesBuilder.addString(WARNING_TIME_ZONE, time.substring(19));
List<Long> timeRange = getTimeRange(time, targetTimeType, partDayValueString);
ForecastData forecastData = new ForecastData();
forecastData.comment = commentText;
forecastData.localTime = time;
forecastData.timeRangeTime = timeRange == null ? null : timeRange.get(0);
forecastData.timeRangeStart = timeRange == null ? null : timeRange.get(1);
forecastData.timeRangeEnd = timeRange == null ? null: timeRange.get(2);
forecastData.valueRangeStart =predictionLowerValueString == null ? predictionLevelValue : predictionLowerValueString;
forecastData.valueRangeEnd = predictionUpperValueString == null ? predictionLevelValue: predictionUpperValueString;
forecastData.value = predictionLevelValue;
forecastData.parameter = "prediction";
forecastData.propertiesBuilder = propertiesBuilder;
forecastDataList.add(forecastData);
}
}
} while (reader.hasNext());
// All events have been collected
writeTimeSerie(locationId, forecastDataList);
}
}
private boolean validWarningNextIssue(String s) {
if (s == null) return false;
if ("This is a final warning, no further warnings will be issued for this event.".equals(s)) return false;
return true;
}
private static List<Long> determineOvernightRange(String time, int beforeNight, int afterNight) {
long rawTimeZoneOffset = TimeZoneUtils.parseRawTimeZoneOffset(time.substring(19));
long date = parseTime(TimeZoneUtils.getTimeZone(rawTimeZoneOffset), dateFormatString, time);
Calendar cal = GregorianCalendar.getInstance();
cal.setTimeZone(TimeZoneUtils.getTimeZone(rawTimeZoneOffset));
cal.setTimeInMillis(date);
Date currentDate = cal.getTime();
int hour = cal.get(Calendar.HOUR_OF_DAY);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
boolean beforeMidnight = hour <= 24 && hour > 12;
Date beforeDate;
Date afterDate;
if (beforeMidnight) {
cal.set(Calendar.HOUR_OF_DAY, beforeNight);
beforeDate = cal.getTime();
cal.add(Calendar.DATE, 1);
cal.set(Calendar.HOUR_OF_DAY, afterNight);
afterDate = cal.getTime();
} else {
cal.set(Calendar.HOUR_OF_DAY, afterNight);
afterDate = cal.getTime();
cal.add(Calendar.DATE, -1);
cal.set(Calendar.HOUR_OF_DAY, beforeNight);
beforeDate = cal.getTime();
}
if (currentDate.before(beforeDate)) {
log.warn("Import.Warn: Overnight was configured but event time is not inside the overnight range of " + beforeNight + ".00-" + afterNight + ".00. The event time '" + currentDate + "' is before " + beforeDate);
return null;
}
if (currentDate.after(afterDate)) {
log.warn("Import.Warn: Overnight was configured but event time is not inside the overnight range of " + beforeNight + ".00-" + afterNight + ".00. The event time '" + currentDate + "' is after " + afterDate);
return null;
}
List<Long> result = new ArrayList<>();
result.add(currentDate.getTime());
result.add(beforeDate.getTime());
result.add(afterDate.getTime());
return result;
}
public static List<Long> getTimeRange(Date time, String targetTimeType, String partDayValueString) {
DateFormat fmt = FastDateFormat.getInstance(dateFormatString, TimeZoneUtils.GMT, Locale.getDefault(), null);
String timeString = fmt.format(time);
timeString += "+00:00";
return getTimeRange(timeString, targetTimeType, partDayValueString);
}
public static List<Long> getTimeRange(String time, String targetTimeType, String partDayValueString) {
if (TARGET_TIME_TYPE_DATE_TIME.equals(targetTimeType)) return null; // no range
if (TARGET_TIME_TYPE_DAY.equals(targetTimeType)) return timeRangeByHours(time, PART_OF_DAY_MAPPINGS.get(TARGET_TIME_VALUE_DAY));
//if (TARGET_TIME_TYPE_DAY.equals(targetTimeType)) return null; // no range
if (TARGET_TIME_TYPE_PART_DAY.equals(targetTimeType)) {
if (TARGET_TIME_VALUE_OVERNIGHT.equals(partDayValueString)) {
return determineOvernightRange(time, 18, 6);
}
return timeRangeByHours(time, PART_OF_DAY_MAPPINGS.get(partDayValueString));
}
return null;
}
@SuppressWarnings("StringConcatenationMissingWhitespace")
private static List<Long> timeRangeByHours(String time, String[] range) {
if (range == null) return null;
List<Long> ranges = new ArrayList<>();
long rawTimeZoneOffset = TimeZoneUtils.parseRawTimeZoneOffset(time.substring(19));
long date = parseTime(TimeZoneUtils.getTimeZone(rawTimeZoneOffset), dateFormatString, time);
// Replace the hour part of the date: 2016-10-05T12:00:00+11:00
String timeRangeStart = time.substring(0, 11) + range[0] + time.substring(13,19);
String timeRangeEnd = time.substring(0, 11) + range[1] + time.substring(13, 19);
long dateStart = parseTime(TimeZoneUtils.getTimeZone(rawTimeZoneOffset), dateFormatString, timeRangeStart);
long dateEnd = parseTime(TimeZoneUtils.getTimeZone(rawTimeZoneOffset), dateFormatString, timeRangeEnd);
ranges.add(date);
ranges.add(dateStart);
ranges.add(dateEnd);
return ranges;
}
private static long parseTime(TimeZone timeZone, String pattern, String dateTime) {
if (timeZone == null)
throw new IllegalArgumentException("timeZone == null");
if (pattern == null)
throw new IllegalArgumentException("pattern == null");
if (TextUtils.trimToNull(dateTime) == null) {
return Long.MIN_VALUE;
}
dateFormat = FastDateFormat.getInstance(pattern, timeZone, Locale.US, dateFormat);
long res;
try {
res = dateFormat.parseToMillis(dateTime);
} catch (Exception e) {
return Long.MIN_VALUE;
}
return res;
}
/**
* Write a time serie header and it's events based on the forecastData.
* @param locationId location of the timeserie
* @param externalForecastTime forecast time.
* @param forecastCollection collection of forecast events.
*/
private void writeTimeSerie(String locationId, Collection<ForecastData> forecastCollection) {
//noinspection SpellCheckingInspection
header.setForecastTime(TimeZoneUtils.GMT, dateFormatStringWithTimeZone, this.externalForecastTime);
header.setLocationId(locationId);
for (ForecastData forecastData : forecastCollection) {
long rawTimeZoneOffset = TimeZoneUtils.parseRawTimeZoneOffset(forecastData.localTime.substring(19));
String sequenceText = "";
if (warningSequence != null && !warningSequence.isEmpty()) //noinspection StringConcatenationInLoop
sequenceText += " (ws " + warningSequence + ")";
contentHandler.setComment(productIdentifier + sequenceText + ": " + warningTitle + ' '+ '\n' + forecastData.comment);
contentHandler.setProperties(forecastData.propertiesBuilder.build());
header.setParameterId(forecastData.parameter);
contentHandler.setValueAndRange('.', forecastData.value, forecastData.valueRangeStart, forecastData.valueRangeEnd);
if (forecastData.timeRangeTime != null) {
contentHandler.setTimeAndRange(forecastData.timeRangeTime, forecastData.timeRangeStart, forecastData.timeRangeEnd);
} else {
contentHandler.setTime(TimeZoneUtils.getTimeZone(rawTimeZoneOffset), dateFormatString, forecastData.localTime.substring(0, 19));
}
contentHandler.setTimeSeriesHeader(header);
contentHandler.applyCurrentFields();
}
}
}
Overview
Content Tools
