source code
/* ================================================================
* Delft FEWS
* ================================================================
*
* Project Info: http://www.wldelft.nl/soft/fews/index.html
* Project Lead: Karel Heynert (karel.heynert@wldelft.nl)
*
* (C) Copyright 2003, by WL | Delft Hydraulics
* P.O. Box 177
* 2600 MH Delft
* The Netherlands
* http://www.wldelft.nl
*
* DELFT-FEWS is a sophisticated collection of modules designed
* for building a FEWS customised to the specific requirements
* of individual agencies. An open modelling approach allows users
* to add their own modules in an efficient way.
*
* ----------------------------------------------------------------
* PiBilParser.java
* ----------------------------------------------------------------
* (C) Copyright 2003, by WL | Delft Hydraulics
*
* Original Author: Onno van de Akker
* Contributor(s): Bart Adriaanse
*
* Changes:
* --------
* Apr 26, 2009 : Version 1 ();
* Jan 2019 : BilBipBsqParser, based on BilParser class
*
*/
package nl.wldelft.timeseriesparsers;
import nl.wldelft.util.BinaryUtils;
import nl.wldelft.util.FileUtils;
import nl.wldelft.util.IOUtils;
import nl.wldelft.util.NumberType;
import nl.wldelft.util.TextUtils;
import nl.wldelft.util.coverage.NonGeoReferencedGridGeometry;
import nl.wldelft.util.coverage.RegularGridGeometry;
import nl.wldelft.util.geodatum.GeoPoint;
import nl.wldelft.util.io.BinaryParser;
import nl.wldelft.util.io.LineReader;
import nl.wldelft.util.io.VirtualInputDir;
import nl.wldelft.util.io.VirtualInputDirConsumer;
import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader;
import nl.wldelft.util.timeseries.TimeSeriesContentHandler;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileFilter;
import java.nio.ByteOrder;
import java.util.HashMap;
import java.util.Map;
/**
* Time series parser for BIP / BIP / BSQ format raster data.
* This raster image format was originally developed by USGS for CCT tape formats for Landsat satellite data from 1972 on.
*
* The format was implemented and documented by ESRI for use in their GIS software systems:
* http://webhelp.esri.com/arcgisdesktop/9.3/index.cfm?topicname=BIL,_BIP,_and_BSQ_raster_files
* http://desktop.arcgis.com/en/arcmap/latest/manage-data/raster-and-images/bil-bip-and-bsq-raster-files.htm
*
* BIL/BIP/BSQ file formats all use a common header file format (*.HDR) but the storage of image bands is organized differently:
* BIL = Band Interleaved by Line
* BIP = Band Interleaved by Pixel
* BSQ = Bands Sequential
* in FEWS, multiple bands are used to store multiple parameters in one file.
*
* NOTE: this class is a work in progress based on PiBilParser, which cannot be modified to improve the the implementation of the BIL / BIP / BSQ format without breaking backward compatibility
* The intention of this parser is to properly implement the ESRI definitions of BIL / BIP / BSQ formats without the specific FEWS extensions that were implemented in PiBilParser.
*
**/
public class BilBipBsqTimeSeriesParser implements BinaryParser<TimeSeriesContentHandler>, VirtualInputDirConsumer, FileFilter {
private VirtualInputDir virtualInputDir = null;
private String virtualFileName = null;
private String headerFileName = null;
private TimeSeriesContentHandler contentHandler = null;
private final DefaultTimeSeriesHeader timeSeriesHeader = new DefaultTimeSeriesHeader();
// header info
private final Map<String, String> headerProperties = new HashMap<>(10);
private boolean headerPreset = false;
private ByteOrder byteOrder = null;
private NumberType numberType = null;
private int nRows = -1;
private int nCols = -1;
private int skipBytes = -1;
private int bandRowBytes = -1;
private int totalRowBytes = -1;
private int bandGapBytes = -1;
private String layout = "";
private int nBands = -1; // nr of parameters in fews
@Override
public void parse(BufferedInputStream inputStream, String virtualFileName, TimeSeriesContentHandler contentHandler) throws Exception {
this.virtualFileName = virtualFileName;
this.contentHandler = contentHandler;
parseHeader();
IOUtils.skipFully(inputStream, skipBytes);
int cellCount = nRows * nCols;
float[] coverageValues = new float[cellCount];
if (layout.equalsIgnoreCase("BSQ") || nBands == 1) {
// BSQ layout can simply be read by band at once, so can the other layouts if only one band is present
byte[] buffer = new byte[cellCount * numberType.getSize()];
for (int iBand = 0; iBand < nBands; iBand++) {
timeSeriesHeader.setParameterId(TextUtils.toString(iBand));
contentHandler.setTimeSeriesHeader(timeSeriesHeader);
IOUtils.readFully(inputStream, buffer);
BinaryUtils.copy(buffer, 0, cellCount * numberType.getSize(), coverageValues, 0, cellCount, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder);
if (iBand < nBands -1 && bandGapBytes > 0) IOUtils.skipFully(inputStream, bandGapBytes);
contentHandler.setCoverageValues(coverageValues);
contentHandler.applyCurrentFields();
}
} else {
// BIL & BIP layouts needs to be transposed by line or pixel
// for each band scan the file line by line to avoid running out of memory on large files
int bufferLength = Math.max(totalRowBytes, nBands * Math.max(bandRowBytes, nCols * numberType.getSize()));
byte[] buffer = new byte[bufferLength];
inputStream.mark(nRows * buffer.length);
for (int iBand = 0; iBand < nBands; iBand++) {
timeSeriesHeader.setParameterId(TextUtils.toString(iBand));
contentHandler.setTimeSeriesHeader(timeSeriesHeader);
inputStream.reset();
if (layout.equalsIgnoreCase("BIL")) {
int bandLength = nCols * numberType.getSize();
int bandBytes = Math.max(bandRowBytes, bandLength);
for (int iRow = 0; iRow < nRows; iRow++) {
IOUtils.readFully(inputStream, buffer);
BinaryUtils.copy(buffer, iBand * bandBytes, bandLength, coverageValues, iRow * nCols, nCols, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder);
}
}
else if (layout.equalsIgnoreCase("BIP")) {
int pixelLength = numberType.getSize();
for (int iRow = 0; iRow < nRows; iRow++) {
IOUtils.readFully(inputStream, buffer);
for (int iCol = 0; iCol < nCols; iCol++) {
BinaryUtils.copy(buffer, pixelLength * ( iCol * nBands + iBand), pixelLength, coverageValues, iRow * nCols + iCol, 1, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder);
}
}
}
contentHandler.setCoverageValues(coverageValues);
contentHandler.applyCurrentFields();
}
}
}
@Override
public boolean accept(File pathname) {
return "bil".equalsIgnoreCase(FileUtils.getFileExt(pathname)) ||
"bsq".equalsIgnoreCase(FileUtils.getFileExt(pathname)) ||
"bip".equalsIgnoreCase(FileUtils.getFileExt(pathname));
}
public void setHeaderProperties(Map<String, String> hashMap) {
this.headerProperties. clear();
this.headerProperties.putAll(hashMap);
headerPreset = !hashMap.isEmpty();
}
private void parseHeader() throws Exception {
if (!headerPreset) {
headerProperties.clear();
headerFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "hdr");
try (LineReader reader = virtualInputDir.getReader(headerFileName)) {
reader.setCommentLinePrefix(';'); // this is not an official standard
for (String[] line = new String[2]; reader.readLine(' ', '\"', line) != -1; ) {
String existing = headerProperties.put(line[0].trim().toUpperCase(), line[1].trim());
if (existing != null) {
throw new Exception("Duplicate header item " + line[0] + " at " + reader.getFileAndLineNumber());
}
}
}
}
parseByteOrder();
nRows = getPositiveInt("NROWS");
nCols = getPositiveInt("NCOLS");
String pixelType = getString("PIXELTYPE", "UNSIGNEDINT"); // parameter PIXELTYPE is introduced in ArcGis 9.3
int nBits = getNonNegativeInt("NBITS", 8);
numberType = null;
if (pixelType.equalsIgnoreCase("SIGNEDINT")) {
numberType = NumberType.getSignedIntType(nBits);
} else if (pixelType.equalsIgnoreCase("UNSIGNEDINT")) {
numberType = NumberType.getUnsignedIntType(nBits);
} else {
throw new Exception("Unsupported PIXELTYPE " + pixelType + " in " + headerFileName);
}
if (numberType == null)
throw new Exception("Unsupported PIXELTYPE " + pixelType);
contentHandler.setValueResolution(numberType.isInteger() ? 1f : Float.NaN);
layout = this.getString("LAYOUT", "");
if (!layout.equalsIgnoreCase("BIL") && !layout.equalsIgnoreCase("BIP") && !layout.equalsIgnoreCase("BSQ")) {
throw new Exception("Unsupported LAYOUT is specified in" + headerFileName);
}
nBands = getPositiveInt("NBANDS", 1); // nr of time series
double ulXMap = getDouble("ULXMAP", Double.NaN);
double ulYMap = getDouble("ULYMAP", Double.NaN);
double xDim = getDouble("XDIM", 0);
double yDim = getDouble("YDIM", 0);
if (Double.isNaN(ulXMap) || Double.isNaN(ulYMap)) {
contentHandler.setGeometry(NonGeoReferencedGridGeometry.create(nRows, nCols));
} else {
GeoPoint firstCellCenter = contentHandler.getDefaultGeoDatum().createXYZ(ulXMap, ulYMap, 0d);
contentHandler.setGeometry(RegularGridGeometry.create(contentHandler.getDefaultGeoDatum(), firstCellCenter, xDim, yDim, nRows, nCols));
}
skipBytes = getNonNegativeInt("SKIPBYTES", 0);
bandRowBytes = getNonNegativeInt("BANDROWBYTES", 0);
totalRowBytes = getNonNegativeInt("TOTALROWBYTES", 0);
bandGapBytes = getNonNegativeInt("BANDGAPBYTES", 0);
}
private void parseByteOrder() throws Exception {
String byteOrderChar = getString("BYTEORDER");
if (byteOrderChar == null || byteOrderChar.isEmpty()) {
byteOrder = ByteOrder.nativeOrder();
} else if (byteOrderChar.equalsIgnoreCase("I")) {
byteOrder = ByteOrder.LITTLE_ENDIAN;
} else if (byteOrderChar.equalsIgnoreCase("M")) {
byteOrder = ByteOrder.BIG_ENDIAN;
} else {
throw new Exception("Unknown byte order " + byteOrderChar);
}
}
private String getString(String name, String defaultValue) {
String res = headerProperties.get(name);
if (res == null) return defaultValue;
return res;
}
private String getString(String name) throws Exception {
String res = headerProperties.get(name);
if (res == null) throw new Exception("Can not found header property " + name + " in " + headerFileName);
return res;
}
private int getPositiveInt(String name) throws Exception {
String text = getString(name);
try {
int res = TextUtils.parseInt(text);
if (res > 0) return res;
} catch (NumberFormatException e) {
throw new Exception("Expected integer instead of " + text + " for header element " + name);
}
throw new Exception("Expected positive integer instead of " + text + " for header element " + name);
}
private int getPositiveInt(String name, int defaultValue) throws Exception {
String text = this.getString(name, null);
if (text == null) return defaultValue;
try {
int res = TextUtils.parseInt(text);
if (res > 0) return res;
} catch (NumberFormatException e) {
throw new Exception("Expected integer instead of " + text + " for header element " + name);
}
throw new Exception("Expected positive integer instead of " + text + " for header element " + name);
}
private int getNonNegativeInt(String name, int defaultValue) throws Exception {
String text = this.getString(name, null);
if (text == null) return defaultValue;
try {
int res = TextUtils.parseInt(text);
if (res >= 0) return res;
} catch (NumberFormatException e) {
throw new Exception("Expected integer instead of " + text + " for header element " + name);
}
throw new Exception("Expected non negative integer instead of " + text + " for header element " + name);
}
private double getDouble(String name, double defaultValue) throws Exception {
String text = this.getString(name, null);
if (text == null) return defaultValue;
try {
return TextUtils.parseDouble(text);
} catch (NumberFormatException e) {
throw new Exception("Expected number instead of " + text + "for header element " + name);
}
}
@Override
public void setVirtualInputDir(VirtualInputDir virtualInputDir) {
this.virtualInputDir = virtualInputDir;
}
}
