package nl.deltares.hydrotel; import nl.wldelft.netcdf.NetcdfUtils; import nl.wldelft.util.Box; import nl.wldelft.util.FastDateFormat; import nl.wldelft.util.FileUtils; import nl.wldelft.util.TextUtils; import nl.wldelft.util.io.LineReader; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.log4j.PropertyConfigurator; import ucar.ma2.Array; import ucar.ma2.ArrayDouble; import ucar.ma2.ArrayFloat; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFileWriter; import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.constants.CF; import ucar.nc2.units.DateUnit; import ucar.nc2.units.TimeUnit; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.Path; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; import static nl.wldelft.netcdf.NetcdfUtils.T_AXIS; public class HydrotelPostAdapter { private static final Logger log = Logger.getLogger(HydrotelPostAdapter.class); private static final String PROPERTIES = "properties"; private static final String SORTIES_HYDROTEL = "sortiesHydrotel"; private static final String SAUVEGARDE_ETATS = "sauvegardeEtats"; private static final String RESULTS_OFFSET = "resultsOffset"; private static final String TIME_VARIABLE_NAME = CF.TIME; private static final String STANDARD_NAME_ATTRIBUTE = CF.STANDARD_NAME; private static final String LONG_NAME_ATTRIBUTE = CDM.LONG_NAME; private static final String UNITS_ATTRIBUTE = CDM.UNITS; private static final String AXIS_ATTRIBUTE = CF.AXIS; private final File runFile; private Path runPath = null; private NetcdfFile netcdfRunFile = null; private String logLevel = "INFO"; private static final String LOG_LEVEL = "log_level"; private static final String WORK_DIR = "work_dir"; private static final String START_TIME = "start_time"; private String workDirString = null; private static final String OUTPUT_NETCDF_FILES_VARIABLE_NAME = "output_netcdf_files"; private File workDirFile = null; private String[] outputTimeSeriesFiles = null; private String[] outputValuesHydrotel = null; private double startDateTime = 0.0; private static final TimeZone EST = TimeZone.getTimeZone("EST"); private static final FastDateFormat YYYY_MM_DDHHMM = FastDateFormat.getInstance("yyyy-MM-dd HH:mm", EST, Locale.CANADA, null); private static final FastDateFormat YYYYMMDDHH = FastDateFormat.getInstance("yyyyMMddHH", EST, Locale.CANADA, null); private boolean sauvegardeEtatsBool = false; private int outputOffsetMinutes = 0; private HydrotelPostAdapter(File runFile) { this.runFile = runFile; } public static void main(String[] args) throws Exception { if (args.length != 1) throw new IllegalArgumentException("Specify either one argument that is the path to the run file, or 4 arguments that are 'work dir', input file, output file and 'log HZ reeks'."); File runFile = new File(args[0]); if (!runFile.exists()) throw new Exception("Can not find run file specified as argument " + runFile); HydrotelPostAdapter adapter = new HydrotelPostAdapter(runFile); try { adapter.run(); if (log.isInfoEnabled()) log.info("HydrotelAdapter run finished without exception"); } catch (Exception e) { log.error(e.getMessage(), e); throw e; } finally { LogManager.shutdown(); } } private void run() throws Exception { try { runPath = runFile.getParentFile().toPath(); String absolutePath = runFile.getAbsolutePath(); netcdfRunFile = NetcdfFile.open(absolutePath); readRunInfoFile(); } finally { netcdfRunFile.close(); } convertTimeSeries(); if (sauvegardeEtatsBool) copyOutputStateFiles(); } private void copyOutputStateFiles() throws IOException { File etatDir = new File(workDirFile, "etat"); File outputStatesDir = new File(workDirFile, "output/states"); File[] stateFiles = etatDir.listFiles((dir, name) -> { try { int length = name.length(); if (length < 14) return false; String substring = name.substring(length - 14, length - 4); long stateTime = YYYYMMDDHH.parseToMillis(substring); return stateTime > startDateTime; } catch (ParseException e) { return false; } }); if (stateFiles == null) return; for (File stateFile : stateFiles) { String name = stateFile.getName(); String substring = name.substring(0, name.length() - 15); File targetFile = new File(outputStatesDir, substring + ".csv"); FileUtils.copy(stateFile, targetFile); } } private void readRunInfoFile() throws Exception { logLevel = netcdfRunFile.findVariable(LOG_LEVEL).readScalarString(); String relativeWorkDirString = String.valueOf((char[]) netcdfRunFile.findVariable(WORK_DIR).read().copyToNDJavaArray()).trim(); if (TextUtils.equals(relativeWorkDirString, ".")) { workDirString = runPath.toString(); } else { workDirString = runPath.resolve(new File(relativeWorkDirString).toPath()).toString(); } workDirFile = new File(workDirString); if (!workDirFile.exists()) { throw new Exception("Work dir not found: " + workDirString); } configureLogging(); if (log.isDebugEnabled()) log.debug("HydrotelAdapter started"); getOutputFiles(); getProperties(); readStartEndTime(); if (log.isInfoEnabled()) log.info("Work dir: " + workDirString); } private void readStartEndTime() throws Exception { Variable startTimeVar = netcdfRunFile.findVariable(START_TIME); double relativeStartDateTime = startTimeVar.readScalarDouble(); Attribute units = startTimeVar.findAttribute("units"); String tunitsString = units.getStringValue(); if (log.isInfoEnabled()) log.info("INFO: Start time: " + relativeStartDateTime + ' ' + tunitsString); DateUnit referenceUnit = new DateUnit(units.getStringValue()); long currentReferenceTime = referenceUnit.getDateOrigin().getTime(); TimeUnit timeUnit = referenceUnit.getTimeUnit(); startDateTime = timeUnit.getValueInSeconds(relativeStartDateTime) * 1000 + currentReferenceTime; } private void getProperties() throws Exception { Variable var = netcdfRunFile.findVariable(PROPERTIES); String sortiesHydrotel = getPropertyStringValue(var, SORTIES_HYDROTEL); outputValuesHydrotel = sortiesHydrotel.split(";"); String sauvegardeEtats = getPropertyStringValue(var, SAUVEGARDE_ETATS); sauvegardeEtatsBool = sauvegardeEtats != null ? Boolean.valueOf(sauvegardeEtats) : false; Integer resultsOffsetPropertyIntValue = getPropertyIntValue(var, RESULTS_OFFSET); outputOffsetMinutes = resultsOffsetPropertyIntValue != null ? resultsOffsetPropertyIntValue : 0; } private void getOutputFiles() throws IOException { Variable outputNetcdfVar = netcdfRunFile.findVariable(OUTPUT_NETCDF_FILES_VARIABLE_NAME); Array outputNetcdfArray = outputNetcdfVar.read(); Object outputTimeSeriesFilesNDArrayObject = outputNetcdfArray.copyToNDJavaArray(); char[][] outputTimeSeriesFilesChar = (char[][]) outputTimeSeriesFilesNDArrayObject; List<String> outputTimeSeriesFilesList = new ArrayList<>(); for (char[] chArray : outputTimeSeriesFilesChar) { File outputFileRelativePath = new File(String.valueOf(chArray).trim()); String absoluteFilePath = runPath.resolve(outputFileRelativePath.toPath()).toString(); outputTimeSeriesFilesList.add(absoluteFilePath); } outputTimeSeriesFiles = new String[outputTimeSeriesFilesList.size()]; outputTimeSeriesFilesList.toArray(outputTimeSeriesFiles); } private void configureLogging() { File logFile = new File(workDirFile, "hydrotel_post_adapter.log"); if (logFile.exists()) { //noinspection ResultOfMethodCallIgnored logFile.delete(); } Properties props = new Properties(); props.setProperty("log4j.rootLogger", logLevel + ", hydrotel"); props.setProperty("log4j.appender.hydrotel.File", logFile.getAbsolutePath()); props.setProperty("log4j.appender.hydrotel", "org.apache.log4j.RollingFileAppender"); props.setProperty("log4j.appender.hydrotel.layout", "org.apache.log4j.PatternLayout"); props.setProperty("log4j.appender.hydrotel.layout.ConversionPattern", "%p - %m \n"); LogManager.resetConfiguration(); PropertyConfigurator.configure(props); if (log.isDebugEnabled()) log.debug("Debug log level enabled"); if (log.isInfoEnabled()) log.info("Info log level enabled"); } private void convertTimeSeries() throws Exception { for (String outputTimeSeriesFile : outputTimeSeriesFiles) { if (outputTimeSeriesFile.endsWith("resultats.nc")) { writeResults(outputTimeSeriesFile); } } } private void writeResults(String outputTimeSeriesFile) throws IOException, InvalidRangeException, ParseException { File file = new File(outputTimeSeriesFile); //noinspection ResultOfMethodCallIgnored file.getParentFile().mkdirs(); NetcdfFileWriter netcdfFileWriter = null; Map<String, Float> map = getWeights(); try { netcdfFileWriter = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outputTimeSeriesFile); File resultDir = new File(workDirString, "simulation/simulation/resultat"); Set<Long> times = new TreeSet<>(); List<Box<String, List<Float>>> values = new ArrayList<>(); for (int j = 0; j < outputValuesHydrotel.length; j++) { String outputValue = outputValuesHydrotel[j]; if (outputValue.equals("DEBITS AVAL")) outputValue = "DEBIT_AVAL"; String outputFileName = outputValue.toLowerCase().replaceAll(" ", "_"); File resultFile = new File(resultDir, outputFileName + ".csv"); if (!resultFile.exists()) { log.error("Result File: " + resultFile + " not found."); continue; } readValuesFromCsvFile(map, times, values, resultFile, outputValue); } String timeUnitString = NetcdfUtils.createTimeUnitString(EST); //create time dimension. Dimension timeDimension = netcdfFileWriter.addDimension(null, TIME_VARIABLE_NAME, times.size()); //create time variable. ArrayList<Dimension> timeDimensionList = new ArrayList<>(); timeDimensionList.add(timeDimension); Variable timeVariable = createTimeVariable(netcdfFileWriter, timeUnitString, timeDimensionList); Variable[] variables = createVariables(netcdfFileWriter, timeDimension, values); netcdfFileWriter.create(); writeValues(netcdfFileWriter, times, values, variables, timeVariable); } finally { if (netcdfFileWriter != null) netcdfFileWriter.close(); } } private static Variable createTimeVariable(NetcdfFileWriter netcdfFileWriter, String timeUnitString, ArrayList<Dimension> timeDimensionList) { Variable timeVariable = netcdfFileWriter.addVariable(null, TIME_VARIABLE_NAME, DataType.DOUBLE, timeDimensionList); netcdfFileWriter.addVariableAttribute(timeVariable, new Attribute(STANDARD_NAME_ATTRIBUTE, TIME_VARIABLE_NAME)); netcdfFileWriter.addVariableAttribute(timeVariable, new Attribute(LONG_NAME_ATTRIBUTE, TIME_VARIABLE_NAME)); netcdfFileWriter.addVariableAttribute(timeVariable, new Attribute(UNITS_ATTRIBUTE, timeUnitString)); netcdfFileWriter.addVariableAttribute(timeVariable, new Attribute(AXIS_ATTRIBUTE, T_AXIS)); return timeVariable; } private void writeValues(NetcdfFileWriter netcdfFileWriter, Set<Long> times, List<Box<String, List<Float>>> values, Variable[] variables, Variable timeVariable) throws IOException, InvalidRangeException { writeTimeValues(netcdfFileWriter, times, timeVariable); for (int j = 0, size = values.size(); j < size; j++) { List<Float> floats = values.get(j).getObject1(); ArrayFloat.D1 array = new ArrayFloat.D1(times.size()); for (int i = 0; i < times.size(); i++) { array.set(i, floats.get(i)); } netcdfFileWriter.write(variables[j], array); } } private void writeTimeValues(NetcdfFileWriter netcdfFileWriter, Set<Long> times, Variable timeVariable) throws IOException, InvalidRangeException { ArrayDouble.D1 timeArray = new ArrayDouble.D1(times.size()); int i = 0; for (Long time : times) { long minutes = time / nl.wldelft.util.TimeUnit.MINUTE_MILLIS + outputOffsetMinutes; timeArray.set(i++, minutes); } netcdfFileWriter.write(timeVariable, timeArray); } private static Variable[] createVariables(NetcdfFileWriter netcdfFileWriter, Dimension timeDimension, List<Box<String, List<Float>>> values) { List<Dimension> dims = new ArrayList<>(); dims.add(timeDimension); int size = values.size(); Variable[] variables = new Variable[size]; for (int i = 0; i < size; i++) { String outputVarName = values.get(i).getObject0(); variables[i] = netcdfFileWriter.addVariable(null, outputVarName.toLowerCase(), DataType.FLOAT, dims); } return variables; } private static void readValuesFromCsvFile(Map<String, Float> map, Set<Long> times, List<Box<String, List<Float>>> values, File resultFile, String outputValue) throws IOException, ParseException { try (LineReader lineReader = new LineReader(resultFile, Charset.defaultCharset())) { lineReader.skipLines(1); String line = lineReader.readLine(); String[] ids = line.split(";"); line = lineReader.readLine(); List<Float> floats = new ArrayList<>(); while (line != null) { String[] columns = line.split(";"); long parse = YYYY_MM_DDHHMM.parseToMillis(columns[0]) + 3 * nl.wldelft.util.TimeUnit.HOUR_MILLIS; times.add(parse); float value = resultFile.getName().equals("debit_aval.csv") ? Float.valueOf(columns[1]) : getWeightedValue(map, ids, columns); floats.add(value); line = lineReader.readLine(); } values.add(new Box(outputValue, floats)); } } private static float getWeightedValue(Map<String, Float> map, String[] ids, String[] columns) { float sum = 0.0f; float totalWeight = 0.0f; for (int i = 1; i < ids.length; i++) { String id = ids[i]; Float weight = map.get(id); if (weight == null || weight == 0.0) continue; totalWeight += weight; sum += weight * Float.valueOf(columns[i]); } return sum / totalWeight; } private Map<String, Float> getWeights() throws IOException { File weightFile = new File(workDirFile, "info/uhrh_amont.txt"); Map<String, Float> map = new HashMap<>(); try (LineReader lineReader = new LineReader(weightFile, Charset.defaultCharset())) { String line = lineReader.readLine(); while (line != null) { String[] columns = line.split(";"); String id = columns[0]; float weight = Float.valueOf(columns[1]); map.put(id, weight); line = lineReader.readLine(); } } return map; } private static String getPropertyStringValue(Variable propertiesVariable, String propertyName) throws Exception { if (propertiesVariable == null) return null; Attribute attribute = propertiesVariable.findAttribute(propertyName); if (attribute == null) return null; String value = attribute.getStringValue(); if (value == null || value.trim().isEmpty()) { throw new Exception("Property '" + propertyName + "' in netcdf run file is empty or is not of type String."); } String trimmedValue = value.trim(); if (log.isDebugEnabled()) log.debug("Read property " + propertyName + " with value '" + trimmedValue + "' from netcdf run file."); return trimmedValue; } private static Integer getPropertyIntValue(Variable propertiesVariable, String propertyName) throws Exception { if (propertiesVariable == null) return null; Attribute attribute = propertiesVariable.findAttribute(propertyName); if (attribute == null) return null; Number value = attribute.getNumericValue(); if (value == null) { throw new Exception("Property '" + propertyName + "' in netcdf run file is empty or is not of type String."); } return value.intValue(); } }