source code
/* ================================================================ * Delft FEWS * ================================================================ * * Project Info: http://www.wldelft.nl/soft/fews/index.html * Project Lead: Karel Heynert (karel.heynert@wldelft.nl) * * (C) Copyright 2003, by WL | Delft Hydraulics * P.O. Box 177 * 2600 MH Delft * The Netherlands * http://www.wldelft.nl * * DELFT-FEWS is a sophisticated collection of modules designed * for building a FEWS customised to the specific requirements * of individual agencies. An open modelling approach allows users * to add their own modules in an efficient way. * * ---------------------------------------------------------------- * PiBilParser.java * ---------------------------------------------------------------- * (C) Copyright 2003, by WL | Delft Hydraulics * * Original Author: Onno van de Akker * Contributor(s): Bart Adriaanse * * Changes: * -------- * Apr 26, 2009 : Version 1 (); * Jan 2019 : BilBipBsqParser, based on BilParser class * */ package nl.wldelft.timeseriesparsers; import nl.wldelft.util.BinaryUtils; import nl.wldelft.util.FileUtils; import nl.wldelft.util.IOUtils; import nl.wldelft.util.NumberType; import nl.wldelft.util.TextUtils; import nl.wldelft.util.coverage.NonGeoReferencedGridGeometry; import nl.wldelft.util.coverage.RegularGridGeometry; import nl.wldelft.util.geodatum.GeoPoint; import nl.wldelft.util.io.BinaryParser; import nl.wldelft.util.io.LineReader; import nl.wldelft.util.io.VirtualInputDir; import nl.wldelft.util.io.VirtualInputDirConsumer; import nl.wldelft.util.timeseries.DefaultTimeSeriesHeader; import nl.wldelft.util.timeseries.TimeSeriesContentHandler; import java.io.BufferedInputStream; import java.io.File; import java.io.FileFilter; import java.nio.ByteOrder; import java.util.HashMap; import java.util.Map; /** * Time series parser for BIP / BIP / BSQ format raster data. * This raster image format was originally developed by USGS for CCT tape formats for Landsat satellite data from 1972 on. * * The format was implemented and documented by ESRI for use in their GIS software systems: * http://webhelp.esri.com/arcgisdesktop/9.3/index.cfm?topicname=BIL,_BIP,_and_BSQ_raster_files * http://desktop.arcgis.com/en/arcmap/latest/manage-data/raster-and-images/bil-bip-and-bsq-raster-files.htm * * BIL/BIP/BSQ file formats all use a common header file format (*.HDR) but the storage of image bands is organized differently: * BIL = Band Interleaved by Line * BIP = Band Interleaved by Pixel * BSQ = Bands Sequential * in FEWS, multiple bands are used to store multiple parameters in one file. * * NOTE: this class is a work in progress based on PiBilParser, which cannot be modified to improve the the implementation of the BIL / BIP / BSQ format without breaking backward compatibility * The intention of this parser is to properly implement the ESRI definitions of BIL / BIP / BSQ formats without the specific FEWS extensions that were implemented in PiBilParser. * **/ public class BilBipBsqTimeSeriesParser implements BinaryParser<TimeSeriesContentHandler>, VirtualInputDirConsumer, FileFilter { private VirtualInputDir virtualInputDir = null; private String virtualFileName = null; private String headerFileName = null; private TimeSeriesContentHandler contentHandler = null; private final DefaultTimeSeriesHeader timeSeriesHeader = new DefaultTimeSeriesHeader(); // header info private final Map<String, String> headerProperties = new HashMap<>(10); private boolean headerPreset = false; private ByteOrder byteOrder = null; private NumberType numberType = null; private int nRows = -1; private int nCols = -1; private int skipBytes = -1; private int bandRowBytes = -1; private int totalRowBytes = -1; private int bandGapBytes = -1; private String layout = ""; private int nBands = -1; // nr of parameters in fews @Override public void parse(BufferedInputStream inputStream, String virtualFileName, TimeSeriesContentHandler contentHandler) throws Exception { this.virtualFileName = virtualFileName; this.contentHandler = contentHandler; parseHeader(); IOUtils.skipFully(inputStream, skipBytes); int cellCount = nRows * nCols; float[] coverageValues = new float[cellCount]; if (layout.equalsIgnoreCase("BSQ") || nBands == 1) { // BSQ layout can simply be read by band at once, so can the other layouts if only one band is present byte[] buffer = new byte[cellCount * numberType.getSize()]; for (int iBand = 0; iBand < nBands; iBand++) { timeSeriesHeader.setParameterId(TextUtils.toString(iBand)); contentHandler.setTimeSeriesHeader(timeSeriesHeader); IOUtils.readFully(inputStream, buffer); BinaryUtils.copy(buffer, 0, cellCount * numberType.getSize(), coverageValues, 0, cellCount, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder); if (iBand < nBands -1 && bandGapBytes > 0) IOUtils.skipFully(inputStream, bandGapBytes); contentHandler.setCoverageValues(coverageValues); contentHandler.applyCurrentFields(); } } else { // BIL & BIP layouts needs to be transposed by line or pixel // for each band scan the file line by line to avoid running out of memory on large files int bufferLength = Math.max(totalRowBytes, nBands * Math.max(bandRowBytes, nCols * numberType.getSize())); byte[] buffer = new byte[bufferLength]; inputStream.mark(nRows * buffer.length); for (int iBand = 0; iBand < nBands; iBand++) { timeSeriesHeader.setParameterId(TextUtils.toString(iBand)); contentHandler.setTimeSeriesHeader(timeSeriesHeader); inputStream.reset(); if (layout.equalsIgnoreCase("BIL")) { int bandLength = nCols * numberType.getSize(); int bandBytes = Math.max(bandRowBytes, bandLength); for (int iRow = 0; iRow < nRows; iRow++) { IOUtils.readFully(inputStream, buffer); BinaryUtils.copy(buffer, iBand * bandBytes, bandLength, coverageValues, iRow * nCols, nCols, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder); } } else if (layout.equalsIgnoreCase("BIP")) { int pixelLength = numberType.getSize(); for (int iRow = 0; iRow < nRows; iRow++) { IOUtils.readFully(inputStream, buffer); for (int iCol = 0; iCol < nCols; iCol++) { BinaryUtils.copy(buffer, pixelLength * ( iCol * nBands + iBand), pixelLength, coverageValues, iRow * nCols + iCol, 1, numberType, 0f, 1f, Integer.MIN_VALUE, byteOrder); } } } contentHandler.setCoverageValues(coverageValues); contentHandler.applyCurrentFields(); } } } @Override public boolean accept(File pathname) { return "bil".equalsIgnoreCase(FileUtils.getFileExt(pathname)) || "bsq".equalsIgnoreCase(FileUtils.getFileExt(pathname)) || "bip".equalsIgnoreCase(FileUtils.getFileExt(pathname)); } public void setHeaderProperties(Map<String, String> hashMap) { this.headerProperties. clear(); this.headerProperties.putAll(hashMap); headerPreset = !hashMap.isEmpty(); } private void parseHeader() throws Exception { if (!headerPreset) { headerProperties.clear(); headerFileName = FileUtils.getPathWithOtherExtension(virtualFileName, "hdr"); try (LineReader reader = virtualInputDir.getReader(headerFileName)) { reader.setCommentLinePrefix(';'); // this is not an official standard for (String[] line = new String[2]; reader.readLine(' ', '\"', line) != -1; ) { String existing = headerProperties.put(line[0].trim().toUpperCase(), line[1].trim()); if (existing != null) { throw new Exception("Duplicate header item " + line[0] + " at " + reader.getFileAndLineNumber()); } } } } parseByteOrder(); nRows = getPositiveInt("NROWS"); nCols = getPositiveInt("NCOLS"); String pixelType = getString("PIXELTYPE", "UNSIGNEDINT"); // parameter PIXELTYPE is introduced in ArcGis 9.3 int nBits = getNonNegativeInt("NBITS", 8); numberType = null; if (pixelType.equalsIgnoreCase("SIGNEDINT")) { numberType = NumberType.getSignedIntType(nBits); } else if (pixelType.equalsIgnoreCase("UNSIGNEDINT")) { numberType = NumberType.getUnsignedIntType(nBits); } else { throw new Exception("Unsupported PIXELTYPE " + pixelType + " in " + headerFileName); } if (numberType == null) throw new Exception("Unsupported PIXELTYPE " + pixelType); contentHandler.setValueResolution(numberType.isInteger() ? 1f : Float.NaN); layout = this.getString("LAYOUT", ""); if (!layout.equalsIgnoreCase("BIL") && !layout.equalsIgnoreCase("BIP") && !layout.equalsIgnoreCase("BSQ")) { throw new Exception("Unsupported LAYOUT is specified in" + headerFileName); } nBands = getPositiveInt("NBANDS", 1); // nr of time series double ulXMap = getDouble("ULXMAP", Double.NaN); double ulYMap = getDouble("ULYMAP", Double.NaN); double xDim = getDouble("XDIM", 0); double yDim = getDouble("YDIM", 0); if (Double.isNaN(ulXMap) || Double.isNaN(ulYMap)) { contentHandler.setGeometry(NonGeoReferencedGridGeometry.create(nRows, nCols)); } else { GeoPoint firstCellCenter = contentHandler.getDefaultGeoDatum().createXYZ(ulXMap, ulYMap, 0d); contentHandler.setGeometry(RegularGridGeometry.create(contentHandler.getDefaultGeoDatum(), firstCellCenter, xDim, yDim, nRows, nCols)); } skipBytes = getNonNegativeInt("SKIPBYTES", 0); bandRowBytes = getNonNegativeInt("BANDROWBYTES", 0); totalRowBytes = getNonNegativeInt("TOTALROWBYTES", 0); bandGapBytes = getNonNegativeInt("BANDGAPBYTES", 0); } private void parseByteOrder() throws Exception { String byteOrderChar = getString("BYTEORDER"); if (byteOrderChar == null || byteOrderChar.isEmpty()) { byteOrder = ByteOrder.nativeOrder(); } else if (byteOrderChar.equalsIgnoreCase("I")) { byteOrder = ByteOrder.LITTLE_ENDIAN; } else if (byteOrderChar.equalsIgnoreCase("M")) { byteOrder = ByteOrder.BIG_ENDIAN; } else { throw new Exception("Unknown byte order " + byteOrderChar); } } private String getString(String name, String defaultValue) { String res = headerProperties.get(name); if (res == null) return defaultValue; return res; } private String getString(String name) throws Exception { String res = headerProperties.get(name); if (res == null) throw new Exception("Can not found header property " + name + " in " + headerFileName); return res; } private int getPositiveInt(String name) throws Exception { String text = getString(name); try { int res = TextUtils.parseInt(text); if (res > 0) return res; } catch (NumberFormatException e) { throw new Exception("Expected integer instead of " + text + " for header element " + name); } throw new Exception("Expected positive integer instead of " + text + " for header element " + name); } private int getPositiveInt(String name, int defaultValue) throws Exception { String text = this.getString(name, null); if (text == null) return defaultValue; try { int res = TextUtils.parseInt(text); if (res > 0) return res; } catch (NumberFormatException e) { throw new Exception("Expected integer instead of " + text + " for header element " + name); } throw new Exception("Expected positive integer instead of " + text + " for header element " + name); } private int getNonNegativeInt(String name, int defaultValue) throws Exception { String text = this.getString(name, null); if (text == null) return defaultValue; try { int res = TextUtils.parseInt(text); if (res >= 0) return res; } catch (NumberFormatException e) { throw new Exception("Expected integer instead of " + text + " for header element " + name); } throw new Exception("Expected non negative integer instead of " + text + " for header element " + name); } private double getDouble(String name, double defaultValue) throws Exception { String text = this.getString(name, null); if (text == null) return defaultValue; try { return TextUtils.parseDouble(text); } catch (NumberFormatException e) { throw new Exception("Expected number instead of " + text + "for header element " + name); } } @Override public void setVirtualInputDir(VirtualInputDir virtualInputDir) { this.virtualInputDir = virtualInputDir; } }