package nl.wldelft.timeseriesparsers; import nl.wldelft.util.BinaryUtils; import nl.wldelft.util.ByteArrayUtils; import nl.wldelft.util.FastGregorianCalendar; import nl.wldelft.util.FloatArrayUtils; import nl.wldelft.util.IOUtils; import nl.wldelft.util.NumberType; import nl.wldelft.util.TextUtils; import nl.wldelft.util.coverage.NonGeoReferencedGridGeometry; import nl.wldelft.util.io.BinaryParser; import nl.wldelft.util.io.LittleEndianDataInputStream; import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader; import nl.wldelft.util.timeseries.TimeSeriesContentHandler; import org.apache.log4j.Logger; import java.io.BufferedInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Locale; /** * Each file consists of one or more records held in sequential format. Each record consists * of a 512 byte header followed by a data array. The data array may be in integer format with 1,2 or 4 bytes per item or in real format with 4 bytes per item. * The default values for each element of the header will be; -32767 for integer elements, -32767.0 for real elements, and a 'null' string for character elements. * It is recommended that all input data files have their data origin at the top left hand corner whenever possible. * However, routines for reading the contents of Nimrod files will contain the option to return a data array with the first element being either the top left or bottom left point of the image/field. */ public class NimrodGridTimeSeriesParser implements BinaryParser<TimeSeriesContentHandler> { private static final Logger log = Logger.getLogger(NimrodGridTimeSeriesParser.class); private TimeSeriesContentHandler contentHandler = null; private DataInput dataInput = null; private NumberType numberType = null; private NonGeoReferencedGridGeometry geometry = null; private DefaultTimeSeriesHeader header = new DefaultTimeSeriesHeader(); private FastGregorianCalendar calendar = null; private byte[] byteBuffer = ByteArrayUtils.EMPTY_ARRAY; private float[] floatBuffer = FloatArrayUtils.EMPTY_ARRAY; private short missingDataValue = Short.MIN_VALUE; @Override public void parse(BufferedInputStream inputStream, String virtualFileName, TimeSeriesContentHandler contentHandler) throws Exception { this.contentHandler = contentHandler; this.calendar = new FastGregorianCalendar(contentHandler.getDefaultTimeZone(), Locale.US); ByteOrder byteOrder = getByteOrder(inputStream); this.dataInput = byteOrder == ByteOrder.BIG_ENDIAN ? new DataInputStream(inputStream) : new LittleEndianDataInputStream(inputStream); for (;;) { long n = IOUtils.trySkipFullyDataInput(dataInput, 4); if (n != 4) return; inputStream.mark(1); n = IOUtils.trySkipFullyDataInput(dataInput, 1); if (n != 1) return; inputStream.reset(); readHeader(); resizeBuffers(); dataInput.readFully(byteBuffer); if (!contentHandler.isCurrentTimeSeriesHeaderForCurrentTimeRejected()) { BinaryUtils.copy(byteBuffer, 0, byteBuffer.length, floatBuffer, 0, floatBuffer.length, numberType, 0f, 1f, missingDataValue, byteOrder); contentHandler.setCoverageValues(floatBuffer); contentHandler.applyCurrentFields(); } n = IOUtils.trySkipFullyDataInput(dataInput, 4); if (n != 4) return; } } private void resizeBuffers() { if (floatBuffer.length == geometry.size()) return; floatBuffer = new float[geometry.size()]; byteBuffer = new byte[geometry.size() * numberType.getSize()]; } private static ByteOrder getByteOrder(BufferedInputStream inputStream) throws IOException { inputStream.mark(6); try { IOUtils.skipFully(inputStream, 4); byte[] buffer = new byte[2]; IOUtils.readFully(inputStream, buffer); int year = BinaryUtils.getUnsignedShort(buffer, 0); return year > 5000 ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN; } finally { inputStream.reset(); } } @SuppressWarnings("OverlyLongMethod") public boolean readHeader() throws IOException { readTimeStamp(true); long time = calendar.getTimeInMillis(); contentHandler.setTime(time); readTimeStamp(false); header.setForecastTime(calendar.getTimeInMillis()); readNumberType(); int experimentNumber = dataInput.readUnsignedShort(); int horizontalGridType = dataInput.readUnsignedShort(); geometry = NonGeoReferencedGridGeometry.create(dataInput.readUnsignedShort(), dataInput.readUnsignedShort()); int headerVersion = dataInput.readUnsignedShort(); header.setParameterId(Integer.toString(dataInput.readUnsignedShort())); int verticalCoordinate = dataInput.readUnsignedShort(); int verticalReferenceLevelCoordinate = dataInput.readUnsignedShort(); int dataSpecificElementsNumber60 = dataInput.readUnsignedShort(); int dataSpecificElementsNumber109 = dataInput.readUnsignedShort(); int locationOfOrigin = dataInput.readUnsignedShort(); missingDataValue = dataInput.readShort(); int accumulationOrAverageMinutes = dataInput.readUnsignedShort(); int numberOfModelLevels = dataInput.readUnsignedShort(); // 0 = Airy 1830 (NG), // 1 = International 1924 (modified UTM-32), // 2 = GRS80 (GUGiK 1992/19) int projection = dataInput.readUnsignedShort(); IOUtils.skipFullyDataInput(dataInput, 6); float verticalCoordinateValue = dataInput.readFloat(); float verticalCoordinateReferenceValue = dataInput.readFloat(); float northing = dataInput.readFloat(); float dy = dataInput.readFloat(); float easting = dataInput.readFloat(); float dx = dataInput.readFloat(); float realMissingValue = dataInput.readFloat(); float mksScalingFactor = dataInput.readFloat(); float dataOffsetValue = dataInput.readFloat(); float xOffsetOfModelDataFromGridPoints = dataInput.readFloat(); float yOffsetOfModelDataFromGridPoints = dataInput.readFloat(); float standardLatitudeOrLatitudeOfTrueOrigin = dataInput.readFloat(); float standardLongitudeOrLatitudeOfTrueOrigin = dataInput.readFloat(); float eastingOfTrueOriginTMProjectionInMetres = dataInput.readFloat(); float northingOfTrueOriginTMProjectionInMetres = dataInput.readFloat(); float scaleFactorOnCentralMeridianForTMProjections = dataInput.readFloat(); // Skip field 48-59 (12*4 bytes) IOUtils.skipFullyDataInput(dataInput, 48); float topLeftCornerLatOfTheImage = dataInput.readFloat(); float topLeftCornerLongOfTheImage = dataInput.readFloat(); float topRightCornerLatOfTheImage = dataInput.readFloat(); float topRightCornerLonOfTheImage = dataInput.readFloat(); float bottomRightCornerLatOfTheImage = dataInput.readFloat(); float bottomRightCornerLonOfTheImage = dataInput.readFloat(); float bottomLeftCornerLatOfTheImage = dataInput.readFloat(); float bottomLeftCornerLonOfTheImage = dataInput.readFloat(); float satelliteCalibrationCoefficient = dataInput.readFloat(); float spaceCountSatelliteData = dataInput.readFloat(); float ductingIndex = dataInput.readFloat(); float elevationAngle = dataInput.readFloat(); // Skip field 72-104 33*4 bytes) IOUtils.skipFullyDataInput(dataInput, 132); //Read field 105-106 from bytes 355-410 : unit (8 bytes), source of data (24 bytes) // and title of field (24 bytes) byte[] byteBuffer = new byte[8]; dataInput.readFully(byteBuffer); String unit = new String(byteBuffer).trim(); byteBuffer = new byte[24]; dataInput.readFully(byteBuffer); String source = new String(byteBuffer).trim(); dataInput.readFully(byteBuffer); String title = new String(byteBuffer).trim(); // Skip the rest of the header and more so we have read 524 bytes from where readHeader started // That is apparently where the data starts. dataInput.readFully(new byte[110]); contentHandler.setTimeSeriesHeader(header); contentHandler.setGeometry(geometry); contentHandler.setValueResolution(numberType.isInteger() ? 1.0f : Float.NaN); if (!log.isDebugEnabled()) return true; List<String> list = new ArrayList<String>(20); list.add(" ------ header ---------------"); list.add("Time: forecast time = " + new Date(this.header.getForecastTime()) + ", time = " + new Date(calendar.getTimeInMillis())); list.add("Number type: " + numberType); list.add("ExperimentNumber: " + experimentNumber); list.add("HorizontalGridType: " + horizontalGridType); list.add("Rows: " + geometry.getRows()); list.add("Columns: " + geometry.getCols()); list.add("HeaderVersion: " + headerVersion); list.add("FieldCode: " + header.getParameterId()); list.add("VerticalCoordinate: " + verticalCoordinate); list.add("VerticalReferenceLevelCoordinate: " + verticalReferenceLevelCoordinate); list.add("DataSpecificElementsNumber60: " + dataSpecificElementsNumber60); list.add("DataSpecificElementsNumber109: " + dataSpecificElementsNumber109); list.add("LocationOfOrigin: " + locationOfOrigin); list.add("MissingDataValue: " + missingDataValue); list.add("AccumulationOrAveragePeriod: " + accumulationOrAverageMinutes); list.add("ModelLevels: " + numberOfModelLevels); list.add("Projection: " + projection); list.add("VerticalCoordinateValue: " + verticalCoordinateValue); list.add("VerticalCoordinateReferenceValue: " + verticalCoordinateReferenceValue); list.add("Northing: " + northing); list.add("Dy: " + dy); list.add("Easting: " + easting); list.add("Dx: " + dx); list.add("RealMissingValue: " + realMissingValue); list.add("MKS scaling factor: " + mksScalingFactor); list.add("Data offset value: " + dataOffsetValue); list.add("X-offset of model data from grid points: " + xOffsetOfModelDataFromGridPoints); list.add("Y-offset of model data from grid points: " + yOffsetOfModelDataFromGridPoints); list.add("Standard latitude or latitude of true origin: " + standardLatitudeOrLatitudeOfTrueOrigin); list.add("Standard longitude or longitude of true origin: " + standardLongitudeOrLatitudeOfTrueOrigin); list.add("Easting of true origin (TM Projection) in metres: " + eastingOfTrueOriginTMProjectionInMetres); list.add("Northing of true origin (TM Projection) in metres: " + northingOfTrueOriginTMProjectionInMetres); list.add("Scale factor on central meridian for TM Projections: " + scaleFactorOnCentralMeridianForTMProjections); list.add("Top left corner of the image lat, long: " + topLeftCornerLatOfTheImage + ", " + topLeftCornerLongOfTheImage); list.add("Top right corner of the image lat, long: " + topRightCornerLatOfTheImage + ", " + topRightCornerLonOfTheImage); list.add("Bottom right corner of the image lat, long: " + bottomRightCornerLatOfTheImage + ", " + bottomRightCornerLonOfTheImage); list.add("Bottom left corner of the image lat, long: " + bottomLeftCornerLatOfTheImage + ", " + bottomLeftCornerLonOfTheImage); list.add("Satellite calibration co-efficient: " + satelliteCalibrationCoefficient); list.add("Space count (satellite data): " + spaceCountSatelliteData); list.add("Ducting Index: " + ductingIndex); list.add("Elevation Angle: " + elevationAngle); list.add("Unit: " + unit); list.add("Source: " + source); list.add("Title: " + title); list.add(" ---------------------"); log.debug(TextUtils.join(list, '\n')); return true; } private void readNumberType() throws IOException { int type = dataInput.readUnsignedShort(); int numberOfBytes = dataInput.readUnsignedShort(); switch (type) { case 0: this.numberType = NumberType.FLOAT; if (numberOfBytes != 4) throw new IOException("numberOfBytes should be 4 for float type"); return; case 1: this.numberType = NumberType.getSignedIntType(numberOfBytes * 8); if (numberType == null) { throw new IOException("unsupported number of byes per data point: " + numberOfBytes); } return; case 2: this.numberType = NumberType.UINT8; if (numberOfBytes != 4) throw new IOException("numberOfBytes should be 1 for byte type"); return; default: throw new IOException("unsupported value type " + type + " found"); } } private void readTimeStamp(boolean hasSeconds) throws IOException { calendar.set(Calendar.YEAR, dataInput.readUnsignedShort()); calendar.set(Calendar.MONTH, dataInput.readUnsignedShort() - 1); // month is zero based calendar.set(Calendar.DAY_OF_MONTH, dataInput.readUnsignedShort()); calendar.set(Calendar.HOUR_OF_DAY, dataInput.readUnsignedShort()); calendar.set(Calendar.MINUTE, dataInput.readUnsignedShort()); calendar.set(Calendar.SECOND, hasSeconds ? dataInput.readUnsignedShort() : 0); calendar.set(Calendar.MILLISECOND, 0); } }