package nl.wldelft.timeseriesparsers;
import nl.wldelft.util.BinaryUtils;
import nl.wldelft.util.ByteArrayUtils;
import nl.wldelft.util.FastGregorianCalendar;
import nl.wldelft.util.FloatArrayUtils;
import nl.wldelft.util.IOUtils;
import nl.wldelft.util.NumberType;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.coverage.NonGeoReferencedGridGeometry;
import nl.wldelft.util.io.BinaryParser;
import nl.wldelft.util.io.LittleEndianDataInputStream;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import org.apache.log4j.Logger;
import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Locale;
/**
* Each file consists of one or more records held in sequential format. Each record consists
* of a 512 byte header followed by a data array. The data array may be in integer format with 1,2 or 4 bytes per item or in real format with 4 bytes per item.
* The default values for each element of the header will be; -32767 for integer elements, -32767.0 for real elements, and a 'null' string for character elements.
* It is recommended that all input data files have their data origin at the top left hand corner whenever possible.
* However, routines for reading the contents of Nimrod files will contain the option to return a data array with the first element being either the top left or bottom left point of the image/field.
*/
public class NimrodGridTimeSeriesParser implements BinaryParser<TimeSeriesContentHandler> {
private static final Logger log = Logger.getLogger(NimrodGridTimeSeriesParser.class);
private TimeSeriesContentHandler contentHandler = null;
private DataInput dataInput = null;
private NumberType numberType = null;
private NonGeoReferencedGridGeometry geometry = null;
private DefaultTimeSeriesHeader header = new DefaultTimeSeriesHeader();
private FastGregorianCalendar calendar = null;
private byte[] byteBuffer = ByteArrayUtils.EMPTY_ARRAY;
private float[] floatBuffer = FloatArrayUtils.EMPTY_ARRAY;
private short missingDataValue = Short.MIN_VALUE;
@Override
public void parse(BufferedInputStream inputStream, String virtualFileName, TimeSeriesContentHandler contentHandler) throws Exception {
this.contentHandler = contentHandler;
this.calendar = new FastGregorianCalendar(contentHandler.getDefaultTimeZone(), Locale.US);
ByteOrder byteOrder = getByteOrder(inputStream);
this.dataInput = byteOrder == ByteOrder.BIG_ENDIAN ? new DataInputStream(inputStream) : new LittleEndianDataInputStream(inputStream);
for (;;) {
long n = IOUtils.trySkipFullyDataInput(dataInput, 4);
if (n != 4) return;
inputStream.mark(1);
n = IOUtils.trySkipFullyDataInput(dataInput, 1);
if (n != 1) return;
inputStream.reset();
readHeader();
resizeBuffers();
dataInput.readFully(byteBuffer);
if (!contentHandler.isCurrentTimeSeriesHeaderForCurrentTimeRejected()) {
BinaryUtils.copy(byteBuffer, 0, byteBuffer.length, floatBuffer, 0, floatBuffer.length, numberType, 0f, 1f, missingDataValue, byteOrder);
contentHandler.setCoverageValues(floatBuffer);
contentHandler.applyCurrentFields();
}
n = IOUtils.trySkipFullyDataInput(dataInput, 4);
if (n != 4) return;
}
}
private void resizeBuffers() {
if (floatBuffer.length == geometry.size()) return;
floatBuffer = new float[geometry.size()];
byteBuffer = new byte[geometry.size() * numberType.getSize()];
}
private static ByteOrder getByteOrder(BufferedInputStream inputStream) throws IOException {
inputStream.mark(6);
try {
IOUtils.skipFully(inputStream, 4);
byte[] buffer = new byte[2];
IOUtils.readFully(inputStream, buffer);
int year = BinaryUtils.getUnsignedShort(buffer, 0);
return year > 5000 ? ByteOrder.LITTLE_ENDIAN : ByteOrder.BIG_ENDIAN;
} finally {
inputStream.reset();
}
}
@SuppressWarnings("OverlyLongMethod")
public void readHeader() throws IOException {
readTimeStamp(true);
long time = calendar.getTimeInMillis();
contentHandler.setTime(time);
readTimeStamp(false);
header.setForecastTime(calendar.getTimeInMillis());
readNumberType();
int experimentNumber = dataInput.readUnsignedShort();
int horizontalGridType = dataInput.readUnsignedShort();
geometry = NonGeoReferencedGridGeometry.create(dataInput.readUnsignedShort(), dataInput.readUnsignedShort());
int headerVersion = dataInput.readUnsignedShort();
header.setParameterId(Integer.toString(dataInput.readUnsignedShort()));
int verticalCoordinate = dataInput.readUnsignedShort();
int verticalReferenceLevelCoordinate = dataInput.readUnsignedShort();
int dataSpecificElementsNumber60 = dataInput.readUnsignedShort();
int dataSpecificElementsNumber109 = dataInput.readUnsignedShort();
int locationOfOrigin = dataInput.readUnsignedShort();
missingDataValue = dataInput.readShort();
int accumulationOrAverageMinutes = dataInput.readUnsignedShort();
int numberOfModelLevels = dataInput.readUnsignedShort();
// 0 = Airy 1830 (NG),
// 1 = International 1924 (modified UTM-32),
// 2 = GRS80 (GUGiK 1992/19)
int projection = dataInput.readUnsignedShort();
IOUtils.skipFullyDataInput(dataInput, 6);
float verticalCoordinateValue = dataInput.readFloat();
float verticalCoordinateReferenceValue = dataInput.readFloat();
float northing = dataInput.readFloat();
float dy = dataInput.readFloat();
float easting = dataInput.readFloat();
float dx = dataInput.readFloat();
float realMissingValue = dataInput.readFloat();
float mksScalingFactor = dataInput.readFloat();
float dataOffsetValue = dataInput.readFloat();
float xOffsetOfModelDataFromGridPoints = dataInput.readFloat();
float yOffsetOfModelDataFromGridPoints = dataInput.readFloat();
float standardLatitudeOrLatitudeOfTrueOrigin = dataInput.readFloat();
float standardLongitudeOrLatitudeOfTrueOrigin = dataInput.readFloat();
float eastingOfTrueOriginTMProjectionInMetres = dataInput.readFloat();
float northingOfTrueOriginTMProjectionInMetres = dataInput.readFloat();
float scaleFactorOnCentralMeridianForTMProjections = dataInput.readFloat();
// Skip field 48-59 (12*4 bytes)
IOUtils.skipFullyDataInput(dataInput, 48);
float topLeftCornerLatOfTheImage = dataInput.readFloat();
float topLeftCornerLongOfTheImage = dataInput.readFloat();
float topRightCornerLatOfTheImage = dataInput.readFloat();
float topRightCornerLonOfTheImage = dataInput.readFloat();
float bottomRightCornerLatOfTheImage = dataInput.readFloat();
float bottomRightCornerLonOfTheImage = dataInput.readFloat();
float bottomLeftCornerLatOfTheImage = dataInput.readFloat();
float bottomLeftCornerLonOfTheImage = dataInput.readFloat();
float satelliteCalibrationCoefficient = dataInput.readFloat();
float spaceCountSatelliteData = dataInput.readFloat();
float ductingIndex = dataInput.readFloat();
float elevationAngle = dataInput.readFloat();
// Skip field 72-104 33*4 bytes)
IOUtils.skipFullyDataInput(dataInput, 132);
//Read field 105-106 from bytes 355-410 : unit (8 bytes), source of data (24 bytes)
// and title of field (24 bytes)
byte[] byteBuffer = new byte[8];
dataInput.readFully(byteBuffer);
String unit = new String(byteBuffer).trim();
byteBuffer = new byte[24];
dataInput.readFully(byteBuffer);
String source = new String(byteBuffer).trim();
dataInput.readFully(byteBuffer);
String title = new String(byteBuffer).trim();
// Skip the rest of the header and more so we have read 524 bytes from where readHeader started
// That is apparently where the data starts.
dataInput.readFully(new byte[110]);
contentHandler.setTimeSeriesHeader(header);
contentHandler.setGeometry(geometry);
contentHandler.setValueResolution(numberType.isInteger() ? 1.0f : Float.NaN);
if (!log.isDebugEnabled()) return;
List<String> list = new ArrayList<String>(20);
list.add(" ------ header ---------------");
list.add("Time: forecast time = " + new Date(this.header.getForecastTime()) + ", time = " + new Date(calendar.getTimeInMillis()));
list.add("Number type: " + numberType);
list.add("ExperimentNumber: " + experimentNumber);
list.add("HorizontalGridType: " + horizontalGridType);
list.add("Rows: " + geometry.getRows());
list.add("Columns: " + geometry.getCols());
list.add("HeaderVersion: " + headerVersion);
list.add("FieldCode: " + header.getParameterId());
list.add("VerticalCoordinate: " + verticalCoordinate);
list.add("VerticalReferenceLevelCoordinate: " + verticalReferenceLevelCoordinate);
list.add("DataSpecificElementsNumber60: " + dataSpecificElementsNumber60);
list.add("DataSpecificElementsNumber109: " + dataSpecificElementsNumber109);
list.add("LocationOfOrigin: " + locationOfOrigin);
list.add("MissingDataValue: " + missingDataValue);
list.add("AccumulationOrAveragePeriod: " + accumulationOrAverageMinutes);
list.add("ModelLevels: " + numberOfModelLevels);
list.add("Projection: " + projection);
list.add("VerticalCoordinateValue: " + verticalCoordinateValue);
list.add("VerticalCoordinateReferenceValue: " + verticalCoordinateReferenceValue);
list.add("Northing: " + northing);
list.add("Dy: " + dy);
list.add("Easting: " + easting);
list.add("Dx: " + dx);
list.add("RealMissingValue: " + realMissingValue);
list.add("MKS scaling factor: " + mksScalingFactor);
list.add("Data offset value: " + dataOffsetValue);
list.add("X-offset of model data from grid points: " + xOffsetOfModelDataFromGridPoints);
list.add("Y-offset of model data from grid points: " + yOffsetOfModelDataFromGridPoints);
list.add("Standard latitude or latitude of true origin: " + standardLatitudeOrLatitudeOfTrueOrigin);
list.add("Standard longitude or longitude of true origin: " + standardLongitudeOrLatitudeOfTrueOrigin);
list.add("Easting of true origin (TM Projection) in metres: " + eastingOfTrueOriginTMProjectionInMetres);
list.add("Northing of true origin (TM Projection) in metres: " + northingOfTrueOriginTMProjectionInMetres);
list.add("Scale factor on central meridian for TM Projections: " + scaleFactorOnCentralMeridianForTMProjections);
list.add("Top left corner of the image lat, long: " + topLeftCornerLatOfTheImage + ", " + topLeftCornerLongOfTheImage);
list.add("Top right corner of the image lat, long: " + topRightCornerLatOfTheImage + ", " + topRightCornerLonOfTheImage);
list.add("Bottom right corner of the image lat, long: " + bottomRightCornerLatOfTheImage + ", " + bottomRightCornerLonOfTheImage);
list.add("Bottom left corner of the image lat, long: " + bottomLeftCornerLatOfTheImage + ", " + bottomLeftCornerLonOfTheImage);
list.add("Satellite calibration co-efficient: " + satelliteCalibrationCoefficient);
list.add("Space count (satellite data): " + spaceCountSatelliteData);
list.add("Ducting Index: " + ductingIndex);
list.add("Elevation Angle: " + elevationAngle);
list.add("Unit: " + unit);
list.add("Source: " + source);
list.add("Title: " + title);
list.add(" ---------------------");
log.debug(TextUtils.join(list, '\n'));
}
private void readNumberType() throws IOException {
int type = dataInput.readUnsignedShort();
int numberOfBytes = dataInput.readUnsignedShort();
switch (type) {
case 0:
this.numberType = NumberType.FLOAT;
if (numberOfBytes != 4)
throw new IOException("numberOfBytes should be 4 for float type");
return;
case 1:
this.numberType = NumberType.getSignedIntType(numberOfBytes * 8);
if (numberType == null) {
throw new IOException("unsupported number of byes per data point: " + numberOfBytes);
}
return;
case 2:
this.numberType = NumberType.UINT8;
if (numberOfBytes != 4)
throw new IOException("numberOfBytes should be 1 for byte type");
return;
default:
throw new IOException("unsupported value type " + type + " found");
}
}
private void readTimeStamp(boolean hasSeconds) throws IOException {
calendar.set(Calendar.YEAR, dataInput.readUnsignedShort());
calendar.set(Calendar.MONTH, dataInput.readUnsignedShort() - 1); // month is zero based
calendar.set(Calendar.DAY_OF_MONTH, dataInput.readUnsignedShort());
calendar.set(Calendar.HOUR_OF_DAY, dataInput.readUnsignedShort());
calendar.set(Calendar.MINUTE, dataInput.readUnsignedShort());
calendar.set(Calendar.SECOND, hasSeconds ? dataInput.readUnsignedShort() : 0);
calendar.set(Calendar.MILLISECOND, 0);
}
}
Overview
Content Tools
