...
| Code Block |
|---|
package nl.wldelft.fews.system.plugin.dataImport; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import nl.wldelft.util.FastDateFormatPeriod; import nl.wldelft.util.PeriodConsumer; import nl.wldelft.util.Properties; import nl.wldelft.util.PeriodPropertiesConsumer; import nl.wldelft.util.PeriodConsumerTextUtils; import nl.wldelft.util.io.ServerParser; import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader; import nl.wldelft.util.timeseries.TimeSeriesContentHandler; import nl.wldelft.util.timeseries.TimeSeriesHeader; import nl.wldelft.util.timeseries.TimeSeriesHeadersConsumer; import org.apache.log4j.Logger; import java.io.IOException; import java.io.InputStream; import java.net.ConnectException; import java.net.HttpURLConnection; import java.net.URL; import java.netutil.URLEncoderDate; import java.util.DateHashMap; import java.util.LocaleMap; /** * DDSC parser developed by HKV. See https://api.ddsc.nl/api/v1/timeseries * Upgraded to support v2 by Deltares. See See https://api.ddsc.nl/api/v2/timeseries * <p> * Service calls the timeseries service using the following URL, where namethe existuuid ofdefines locationname,parameternamethe unique (intimeserie. thisUsing exampleproperties isa themapping locationhas nameto OMZD_29be andmade thefrom parameter 9_BARO),location to uuid. : * <p> * https://apiddsc.ddsclizard.nlnet/api/v2/timeseries/cc0a8831-7758-4dd6-999c-f607c026d176/?start=2015-12-03T08:00:00.000Z1421816000000&end=2015-12-10T08:00:00.000Z&name=OMZD_29,9_BARO&format==1454595236646&format=json * <p> * The response will contain a json file with events in the following format. * <p> * events: [ { timestamp: 1449366444000, max: 1013.6, min: 1013.6 }, { timestamp: 1449367344000, max: 1013.58, min: 1013.58 }, The Importer currently uses the max value to store a value. */ public class DdscTimeSeriesServerParser implements ServerParser<TimeSeriesContentHandler>, TimeSeriesHeadersConsumer, PeriodConsumer, PropertiesConsumer { private static final Logger log = Logger.getLogger(DdscTimeSeriesServerParser.class); public static final String RESULTS = "results"; public static final String DATETIME = "timestamp"; public static final String VALUE = "max"; public static final int MISSING_VALUE = -999; public static final String EVENTS = "events"; private FastDateFormatTimeSeriesHeader[] dateFormatheaders = null; private TimeSeriesHeader[]Period headersperiod = null; private Periodfinal Map perioduuidMap = new nullHashMap<>(); @SuppressWarnings({"AssignmentToCollectionOrArrayFieldFromParameter"}) @Override public void setTimeSeriesHeaders(TimeSeriesHeader[] timeSeriesHeaders) { this.headers = timeSeriesHeaders; } @Override public void setPeriod(Period period) { this.period = period; } @Override public void parse(URL url, String username, String password, TimeSeriesContentHandler contentHandler) throws IOException { if (period == Period.ANY_TIME) throw new IllegalArgumentException("contentHandler.getWantedPeriod() == Period.ANY_TIME"); Date dateFormatstartDate = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", contentHandler.getDefaultTimeZone(), Locale.US, dateFormatperiod.getStartDate(); Date startDateendDate = period.getStartDategetEndDate(); Date endDate = period.getEndDate(); String periodQuery = "?start=" + dateFormatstartDate.formatgetTime(startDate) + "&end=" + dateFormatendDate.formatgetTime(endDate); for (int i = 0; i < headers.length; i++) { TimeSeriesHeader header = headers[i]; try { DefaultTimeSeriesHeader defaultTimeSeriesHeader = new DefaultTimeSeriesHeader(); defaultTimeSeriesHeader.setLocationId(header.getLocationId()); defaultTimeSeriesHeader.setParameterId(header.getParameterId()); contentHandler.addMissingValue(MISSING_VALUE); contentHandler.createTimeSeriesHeaderAlias(i, defaultTimeSeriesHeader); contentHandler.setTimeSeriesHeader(i); //noinspection StringConcatenationMissingWhitespace String key = header.getParameterId() + "," + header.getLocationId(); URL timeSeriesUrl String uuid = new URL(url.toExternalForm() + periodQuery + (String) uuidMap.get(key); if (uuid == null) { "&name=" + URLEncoder.encode(header.getLocationId(), "utf-8") + "," + URLEncoder.encode(header.getParameterId(), "utf-8") + log.warn("No uuid mapping found for '" + key + "'"); continue; "&format=json"); } log.debug("timeseries url: " + timeSeriesUrl); //noinspection StringConcatenationMissingWhitespace tryURL (InputStreamtimeSeriesUrl timeSeriesInputStream= =new openHttpInputStreamURL(timeSeriesUrlurl.toExternalForm(), username, password)) { + uuid + "/" + periodQuery + if (timeSeriesInputStream != null) { "&format=json"); log.info("Processing timeseries url: " parseEvents(contentHandler, timeSeriesInputStream+ timeSeriesUrl); try (InputStream timeSeriesInputStream } = openHttpInputStream(timeSeriesUrl.toExternalForm(), username, password)) { } if (timeSeriesInputStream } catch (ConnectException e!= null) { log.warn("Can not connect to " + url + ' ' + e.getMessageparseEvents()contentHandler, etimeSeriesInputStream); return; } } catch (Exception e) { } } catch (ConnectException e) { log.warn("Can not parseconnect data forto " + headerurl + ' ' + e.getMessage(), e); } return; } } private} static InputStreamcatch openHttpInputStream(StringException restUrl, String username, String password) throws Exception { e) { URL url = new URL(restUrl); HttpURLConnection connection = (HttpURLConnection) url.openConnection( log.warn("Can not parse data for " + header + ' ' + e.getMessage(), e); // just want to do} an HTTP GET here } connection.setRequestMethod("GET");} private static InputStream openHttpInputStream(String restUrl, //String giveusername, itString 15password) secondsthrows toException respond{ connection.setReadTimeout(15 * 1000URL url = new URL(restUrl); HttpURLConnection connection.setRequestProperty("username", username = (HttpURLConnection) url.openConnection(); connection.setRequestProperty("password", password); connection.connect();// just want to do an HTTP GET here return connection.getInputStreamsetRequestMethod("GET"); } static// voidgive parseEvents(TimeSeriesContentHandler contentHandler, InputStream inputStream) throws IOException {it 15 seconds to respond if (inputStream != null) {connection.setReadTimeout(15 * 1000); connection.setRequestProperty("username", username); try (JsonParser jsonParser = new JsonFactory()connection.createParser(inputStream)) {setRequestProperty("password", password); connection.connect(); whilereturn (jsonParserconnection.nextTokengetInputStream(); != JsonToken.END_OBJECT) { } static void parseEvents(TimeSeriesContentHandler contentHandler, InputStream inputStream) throws IOException { Stringif fieldName(inputStream != jsonParser.getCurrentName(); null) { try (JsonParser jsonParser = new JsonFactory().createParser(inputStream)) { // Lees eventuele next url boolean startParsingEvents = false; if (RESULTS.equals(fieldName)) writeTimeSeriesEvent(contentHandler, jsonParser); for (JsonToken token; (token = jsonParser.nextToken()) != null; ) { } if (token == JsonToken.FIELD_NAME) { } } } private static void writeTimeSeriesEvent(TimeSeriesContentHandler contentHandler, JsonParser jsonParser) throws IOException { String fieldName = jsonParser.getCurrentName(); while (jsonParser.nextToken() != JsonToken.END_ARRAY) { if (EVENTSTextUtils.equals(jsonParser.getCurrentName(EVENTS, fieldName))) { processJsonEvents(contentHandler, jsonParser); startParsingEvents } = true; // The events object was }found. } private static void processJsonEvents(TimeSeriesContentHandler contentHandler, JsonParser jsonParser) throws IOException { float value = MISSING_VALUEcontinue; String date = null; while (jsonParser.nextToken() != JsonToken.END_ARRAY) { // process all events until the end of the events array. } } String fieldName = jsonParser.getCurrentName(); if (VALUE.equals(fieldNamestartParsingEvents)) { jsonParser.nextToken( processJsonEvents(contentHandler, jsonParser); value = jsonParser.getFloatValue(); startParsingEvents = false; } else if (DATETIME.equals(fieldName)) { } jsonParser.nextToken(); } date = jsonParser.getText(); } } } private static void if (date != null && value != MISSING_VALUE) {processJsonEvents(TimeSeriesContentHandler contentHandler, JsonParser jsonParser) throws IOException { float value = MISSING_VALUE; String date = null; while (jsonParser.nextToken() != JsonToken.END_ARRAY) { // process all events until the end of the events array. String fieldName = jsonParser.getCurrentName(); if (VALUE.equals(fieldName)) { jsonParser.nextToken(); value = jsonParser.getFloatValue(); } else if (DATETIME.equals(fieldName)) { jsonParser.nextToken(); date = jsonParser.getText(); } if (date != null && value != MISSING_VALUE) { contentHandler.setTime(Long.parseLong(date)); contentHandler.setValue(value); contentHandler.applyCurrentFields(); value = MISSING_VALUE; date = null; } } } /** * Properties are used to map parameter location combinations to uuid. * <properties> * <string key="param1,location1" value="uuid1"/> * <string key="param1,location2" value="uuid2"/> * <string key="param2,location1" value="uuid3"/> * </properties> */ @Override public void contentHandler.setTime(Long.parseLong(date));setProperties(Properties properties) { for (int i = 0; i < contentHandlerproperties.setValuesize(value); i++) { String key = contentHandlerproperties.applyCurrentFieldsgetKey(i); String value = MISSING_VALUEproperties.getString(i); //noinspection ResultOfMethodCallIgnored date = null; }uuidMap.put(key, value); } } } |