package nl.wldelft.fews.system.plugin.dataImport;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import nl.wldelft.util.Period;
import nl.wldelft.util.PeriodConsumer;
import nl.wldelft.util.Properties;
import nl.wldelft.util.PropertiesConsumer;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.io.ServerParser;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import nl.wldelft.util.timeseries.TimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeSeriesHeadersConsumer;
import org.apache.log4j.Logger;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
/**
* DDSC parser developed by HKV. See https://api.ddsc.nl/api/v1/timeseries
* Upgraded to support v2 by Deltares. See See https://api.ddsc.nl/api/v2/timeseries
* <p>
* Service calls the timeseries service using the following URL, where the uuid defines the unique timeserie. Using properties a mapping has to be made from parameter,location to uuid. :
* <p>
* https://ddsc.lizard.net/api/v2/timeseries/cc0a8831-7758-4dd6-999c-f607c026d176/?start=1421816000000&end=1454595236646&format=json
* <p>
* The response will contain a json file with events in the following format.
* <p>
* events: [
{
timestamp: 1449366444000,
max: 1013.6,
min: 1013.6
},
{
timestamp: 1449367344000,
max: 1013.58,
min: 1013.58
},
The Importer currently uses the max value to store a value.
*/
public class DdscTimeSeriesServerParser implements ServerParser<TimeSeriesContentHandler>, TimeSeriesHeadersConsumer, PeriodConsumer, PropertiesConsumer {
private static final Logger log = Logger.getLogger(DdscTimeSeriesServerParser.class);
public static final String DATETIME = "timestamp";
public static final String VALUE = "max";
public static final int MISSING_VALUE = -999;
public static final String EVENTS = "events";
private TimeSeriesHeader[] headers = null;
private Period period = null;
private final Map uuidMap = new HashMap<>();
@SuppressWarnings({"AssignmentToCollectionOrArrayFieldFromParameter"})
@Override
public void setTimeSeriesHeaders(TimeSeriesHeader[] timeSeriesHeaders) {
this.headers = timeSeriesHeaders;
}
@Override
public void setPeriod(Period period) {
this.period = period;
}
@Override
public void parse(URL url, String username, String password, TimeSeriesContentHandler contentHandler) throws IOException {
if (period == Period.ANY_TIME) throw new IllegalArgumentException("contentHandler.getWantedPeriod() == Period.ANY_TIME");
Date startDate = period.getStartDate();
Date endDate = period.getEndDate();
String periodQuery = "?start=" + startDate.getTime() + "&end=" + endDate.getTime();
for (int i = 0; i < headers.length; i++) {
TimeSeriesHeader header = headers[i];
try {
DefaultTimeSeriesHeader defaultTimeSeriesHeader = new DefaultTimeSeriesHeader();
defaultTimeSeriesHeader.setLocationId(header.getLocationId());
defaultTimeSeriesHeader.setParameterId(header.getParameterId());
contentHandler.addMissingValue(MISSING_VALUE);
contentHandler.createTimeSeriesHeaderAlias(i, defaultTimeSeriesHeader);
contentHandler.setTimeSeriesHeader(i);
String key = header.getParameterId() + "," + header.getLocationId();
String uuid = (String) uuidMap.get(key);
if (uuid == null) {
log.warn("No uuid mapping found for '" + key + "'");
continue;
}
//noinspection StringConcatenationMissingWhitespace
URL timeSeriesUrl = new URL(url.toExternalForm() + uuid + "/" + periodQuery +
"&format=json");
log.info("Processing timeseries url: " + timeSeriesUrl);
try (InputStream timeSeriesInputStream = openHttpInputStream(timeSeriesUrl.toExternalForm(), username, password)) {
if (timeSeriesInputStream != null) {
parseEvents(contentHandler, timeSeriesInputStream);
}
}
} catch (ConnectException e) {
log.warn("Can not connect to " + url + ' ' + e.getMessage(), e);
return;
} catch (Exception e) {
log.warn("Can not parse data for " + header + ' ' + e.getMessage(), e);
}
}
}
private static InputStream openHttpInputStream(String restUrl, String username, String password) throws Exception {
URL url = new URL(restUrl);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
// just want to do an HTTP GET here
connection.setRequestMethod("GET");
// give it 15 seconds to respond
connection.setReadTimeout(15 * 1000);
connection.setRequestProperty("username", username);
connection.setRequestProperty("password", password);
connection.connect();
return connection.getInputStream();
}
static void parseEvents(TimeSeriesContentHandler contentHandler, InputStream inputStream) throws IOException {
if (inputStream != null) {
try (JsonParser jsonParser = new JsonFactory().createParser(inputStream)) {
boolean startParsingEvents = false;
for (JsonToken token; (token = jsonParser.nextToken()) != null; ) {
if (token == JsonToken.FIELD_NAME) {
String fieldName = jsonParser.getCurrentName();
if (TextUtils.equals(EVENTS, fieldName)) {
startParsingEvents = true; // The events object was found.
continue;
}
}
if (startParsingEvents) {
processJsonEvents(contentHandler, jsonParser);
startParsingEvents = false;
}
}
}
}
}
private static void processJsonEvents(TimeSeriesContentHandler contentHandler, JsonParser jsonParser) throws IOException {
float value = MISSING_VALUE;
String date = null;
while (jsonParser.nextToken() != JsonToken.END_ARRAY) { // process all events until the end of the events array.
String fieldName = jsonParser.getCurrentName();
if (VALUE.equals(fieldName)) {
jsonParser.nextToken();
value = jsonParser.getFloatValue();
} else if (DATETIME.equals(fieldName)) {
jsonParser.nextToken();
date = jsonParser.getText();
}
if (date != null && value != MISSING_VALUE) {
contentHandler.setTime(Long.parseLong(date));
contentHandler.setValue(value);
contentHandler.applyCurrentFields();
value = MISSING_VALUE;
date = null;
}
}
}
/**
* Properties are used to map parameter location combinations to uuid.
* <properties>
* <string key="param1,location1" value="uuid1"/>
* <string key="param1,location2" value="uuid2"/>
* <string key="param2,location1" value="uuid3"/>
* </properties>
*/
@Override
public void setProperties(Properties properties) {
for (int i = 0; i < properties.size(); i++) {
String key = properties.getKey(i);
String value = properties.getString(i);
//noinspection ResultOfMethodCallIgnored
uuidMap.put(key, value);
}
}
}
