From 587c3f106fd87b209073a33668999ce646626fa8 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 22 Jul 2026 10:24:47 +0200 Subject: [PATCH 01/10] =?UTF-8?q?Added=20a=20streaming=20writer=20producin?= =?UTF-8?q?g=20time=20=C3=97=20bin=5Findex=20variables=20and=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../processing/l3/SeaGridNetcdfFormatter.java | 286 ++++++++++++++++++ .../l3/SeaGridNetcdfFormatterTest.java | 157 ++++++++++ 2 files changed, 443 insertions(+) create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java new file mode 100644 index 000000000..939452d99 --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java @@ -0,0 +1,286 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.ProductData; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Writes a fully populated, flattened sinusoidal NetCDF file. + * + * The structural model follows the OC-CCI reference products: science + * variables use {@code (time, bin_index)}, while latitude and longitude use + * {@code (bin_index)}. Product-specific metadata is intentionally left to a + * later post-processing step. + */ +final class SeaGridNetcdfFormatter { + + private static final int BUFFER_SIZE = 8192; + private static final long MILLIS_PER_DAY = 24L * 60L * 60L * 1000L; + private static final Set RESERVED_VARIABLE_NAMES = new HashSet( + Arrays.asList("time", "bin_index", "lat", "lon", "crs")); + + private SeaGridNetcdfFormatter() { + } + + static void write(File outputFile, + SEAGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames, + ProductData.UTC startTime) throws IOException { + write(outputFile, planetaryGrid, temporalBinSource, featureNames, startTime, + NetcdfFileWriter.Version.netcdf4_classic); + } + + /** + * Package-private format selection keeps the structural test independent + * of the native NetCDF-4 library. Production always uses NetCDF-4 classic. + */ + static void write(File outputFile, + SEAGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames, + ProductData.UTC startTime, + NetcdfFileWriter.Version version) throws IOException { + validateArguments(outputFile, planetaryGrid, temporalBinSource, featureNames); + + final long numBinsLong = planetaryGrid.getNumBins(); + if (numBinsLong > Integer.MAX_VALUE) { + throw new IOException("The sinusoidal bin count exceeds the NetCDF dimension limit: " + numBinsLong); + } + final int numBins = (int) numBinsLong; + + NetcdfFileWriter writer = NetcdfFileWriter.createNew(version, outputFile.getAbsolutePath()); + writer.setFill(true); + writer.setLargeFile(true); + + final Dimension timeDimension = writer.addDimension("time", 1); + final Dimension binIndexDimension = writer.addDimension("bin_index", numBins); + + final Variable timeVariable = writer.addVariable("time", DataType.INT, + Arrays.asList(timeDimension)); + timeVariable.addAttribute(new Attribute("axis", "T")); + timeVariable.addAttribute(new Attribute("standard_name", "time")); + timeVariable.addAttribute(new Attribute("units", "days since 1970-01-01")); + + final Variable crsVariable = writer.addVariable("crs", DataType.INT, + Arrays.asList(timeDimension)); + crsVariable.addAttribute(new Attribute("grid_mapping_name", "1D binned sinusoidal")); + crsVariable.addAttribute(new Attribute("number_of_latitude_rows", planetaryGrid.getNumRows())); + crsVariable.addAttribute(new Attribute("total_number_of_bins", numBins)); + + final Variable latitudeVariable = writer.addVariable("lat", DataType.FLOAT, + Arrays.asList(binIndexDimension)); + latitudeVariable.addAttribute(new Attribute("standard_name", "latitude")); + latitudeVariable.addAttribute(new Attribute("units", "degrees_north")); + latitudeVariable.addAttribute(new Attribute("axis", "Y")); + + final Variable longitudeVariable = writer.addVariable("lon", DataType.FLOAT, + Arrays.asList(binIndexDimension)); + longitudeVariable.addAttribute(new Attribute("standard_name", "longitude")); + longitudeVariable.addAttribute(new Attribute("units", "degrees_east")); + longitudeVariable.addAttribute(new Attribute("axis", "X")); + + final List featureVariables = new ArrayList(featureNames.length); + for (String featureName : featureNames) { + Variable featureVariable = writer.addVariable(featureName, DataType.FLOAT, + Arrays.asList(timeDimension, binIndexDimension)); + featureVariable.addAttribute(new Attribute("_FillValue", Float.NaN)); + featureVariable.addAttribute(new Attribute("coordinates", "lat lon")); + featureVariable.addAttribute(new Attribute("grid_mapping", "crs")); + featureVariables.add(featureVariable); + } + + writer.addGlobalAttribute("Conventions", "CF-1.7"); + + boolean sourceOpened = false; + try { + writer.create(); + writeScalarVariables(writer, timeVariable, crsVariable, startTime); + writeCoordinates(writer, planetaryGrid, latitudeVariable, longitudeVariable, numBins); + + final FeatureBuffer featureBuffer = new FeatureBuffer(writer, featureVariables, numBins); + final int partCount = temporalBinSource.open(); + sourceOpened = true; + for (int partIndex = 0; partIndex < partCount; partIndex++) { + Iterator part = temporalBinSource.getPart(partIndex); + while (part.hasNext()) { + featureBuffer.add(part.next()); + } + temporalBinSource.partProcessed(partIndex, part); + } + featureBuffer.flush(); + } catch (InvalidRangeException e) { + throw new IOException("Failed to write sinusoidal NetCDF data.", e); + } finally { + IOException closeFailure = null; + if (sourceOpened) { + try { + temporalBinSource.close(); + } catch (IOException e) { + closeFailure = e; + } + } + try { + writer.close(); + } catch (IOException e) { + if (closeFailure == null) { + closeFailure = e; + } else { + closeFailure.addSuppressed(e); + } + } + if (closeFailure != null) { + throw closeFailure; + } + } + } + + private static void validateArguments(File outputFile, + SEAGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames) { + if (outputFile == null || planetaryGrid == null || temporalBinSource == null || featureNames == null) { + throw new NullPointerException("Output file, grid, bin source, and feature names are required."); + } + if (featureNames.length == 0) { + throw new IllegalArgumentException("At least one science variable is required."); + } + Set uniqueNames = new HashSet(); + for (String featureName : featureNames) { + if (featureName == null || featureName.trim().isEmpty()) { + throw new IllegalArgumentException("Science-variable names must not be empty."); + } + if (RESERVED_VARIABLE_NAMES.contains(featureName)) { + throw new IllegalArgumentException("Reserved NetCDF variable name: " + featureName); + } + if (!uniqueNames.add(featureName)) { + throw new IllegalArgumentException("Duplicate science-variable name: " + featureName); + } + } + } + + private static void writeScalarVariables(NetcdfFileWriter writer, + Variable timeVariable, + Variable crsVariable, + ProductData.UTC startTime) + throws IOException, InvalidRangeException { + int epochDay = startTime != null ? (int) (startTime.getAsDate().getTime() / MILLIS_PER_DAY) : 0; + writer.write(timeVariable, Array.factory(DataType.INT, new int[]{1}, new int[]{epochDay})); + writer.write(crsVariable, Array.factory(DataType.INT, new int[]{1}, new int[]{0})); + } + + private static void writeCoordinates(NetcdfFileWriter writer, + SEAGrid planetaryGrid, + Variable latitudeVariable, + Variable longitudeVariable, + int numBins) + throws IOException, InvalidRangeException { + for (int origin = 0; origin < numBins; origin += BUFFER_SIZE) { + int length = Math.min(BUFFER_SIZE, numBins - origin); + float[] latitudes = new float[length]; + float[] longitudes = new float[length]; + for (int offset = 0; offset < length; offset++) { + double[] center = planetaryGrid.getCenterLatLon((long) origin + offset); + latitudes[offset] = (float) center[0]; + longitudes[offset] = (float) center[1]; + } + writer.write(latitudeVariable, new int[]{origin}, + Array.factory(DataType.FLOAT, new int[]{length}, latitudes)); + writer.write(longitudeVariable, new int[]{origin}, + Array.factory(DataType.FLOAT, new int[]{length}, longitudes)); + } + } + + private static final class FeatureBuffer { + + private final NetcdfFileWriter writer; + private final List variables; + private final int numBins; + private final float[][] values; + + private long startIndex = -1; + private long lastIndex = -1; + private int length; + + private FeatureBuffer(NetcdfFileWriter writer, List variables, int numBins) { + this.writer = writer; + this.variables = variables; + this.numBins = numBins; + values = new float[variables.size()][BUFFER_SIZE]; + } + + private void add(TemporalBin temporalBin) throws IOException, InvalidRangeException { + long binIndex = temporalBin.getIndex(); + if (binIndex < 0 || binIndex >= numBins) { + throw new IOException("Temporal bin index outside the sinusoidal grid: " + binIndex); + } + if (binIndex <= lastIndex) { + throw new IOException("Temporal bins must be ordered by increasing global bin index: " + binIndex); + } + if (temporalBin.getFeatureValues().length != variables.size()) { + throw new IOException("Temporal bin " + binIndex + " has " + + temporalBin.getFeatureValues().length + " features; expected " + + variables.size() + '.'); + } + if (startIndex < 0 || binIndex >= startIndex + BUFFER_SIZE) { + flush(); + reset(binIndex); + } + + int offset = (int) (binIndex - startIndex); + for (int featureIndex = 0; featureIndex < values.length; featureIndex++) { + values[featureIndex][offset] = temporalBin.getFeatureValues()[featureIndex]; + } + length = Math.max(length, offset + 1); + lastIndex = binIndex; + } + + private void reset(long newStartIndex) { + startIndex = newStartIndex; + length = 0; + for (float[] featureValues : values) { + Arrays.fill(featureValues, Float.NaN); + } + } + + private void flush() throws IOException, InvalidRangeException { + if (length == 0) { + return; + } + int[] origin = new int[]{0, (int) startIndex}; + for (int featureIndex = 0; featureIndex < variables.size(); featureIndex++) { + float[] data = Arrays.copyOf(values[featureIndex], length); + writer.write(variables.get(featureIndex), origin, + Array.factory(DataType.FLOAT, new int[]{1, length}, data)); + } + startIndex = -1; + length = 0; + } + } +} diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java new file mode 100644 index 000000000..9020441ec --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class SeaGridNetcdfFormatterTest { + + private File outputFile; + + @Before + public void setUp() throws IOException { + outputFile = File.createTempFile("calvalus-isin-structure-", ".nc"); + } + + @After + public void tearDown() { + if (outputFile != null && outputFile.exists() && !outputFile.delete()) { + outputFile.deleteOnExit(); + } + } + + @Test + public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { + SEAGrid grid = new SEAGrid(4); + TemporalBin firstBin = createBin(0, 1.25f, 3.5f); + TemporalBin secondBin = createBin(10, 2.5f, 7.0f); + TemporalBinSource source = new SinglePartSource(firstBin, secondBin); + + SeaGridNetcdfFormatter.write(outputFile, + grid, + source, + new String[]{"chlor_a", "total_nobs"}, + ProductData.UTC.create(new Date(1659312000000L), 0), + NetcdfFileWriter.Version.netcdf3); + + NetcdfFile netcdfFile = NetcdfFile.open(outputFile.getAbsolutePath()); + try { + Dimension time = netcdfFile.findDimension("time"); + Dimension binIndex = netcdfFile.findDimension("bin_index"); + assertNotNull(time); + assertNotNull(binIndex); + assertEquals(1, time.getLength()); + assertEquals(grid.getNumBins(), binIndex.getLength()); + + assertDimensions(netcdfFile.findVariable("chlor_a"), "time", "bin_index"); + assertDimensions(netcdfFile.findVariable("total_nobs"), "time", "bin_index"); + assertDimensions(netcdfFile.findVariable("lat"), "bin_index"); + assertDimensions(netcdfFile.findVariable("lon"), "bin_index"); + assertDimensions(netcdfFile.findVariable("time"), "time"); + assertDimensions(netcdfFile.findVariable("crs"), "time"); + + Variable crs = netcdfFile.findVariable("crs"); + assertEquals("1D binned sinusoidal", + crs.findAttribute("grid_mapping_name").getStringValue()); + assertEquals(4, crs.findAttribute("number_of_latitude_rows").getNumericValue().intValue()); + assertEquals(grid.getNumBins(), + crs.findAttribute("total_number_of_bins").getNumericValue().longValue()); + + Array chlorA = netcdfFile.findVariable("chlor_a").read(); + Array totalNobs = netcdfFile.findVariable("total_nobs").read(); + assertEquals(1.25f, chlorA.getFloat(0), 0.0f); + assertEquals(3.5f, totalNobs.getFloat(0), 0.0f); + assertTrue(Float.isNaN(chlorA.getFloat(1))); + assertEquals(2.5f, chlorA.getFloat(10), 0.0f); + assertEquals(7.0f, totalNobs.getFloat(10), 0.0f); + + Attribute conventions = netcdfFile.findGlobalAttribute("Conventions"); + assertNotNull(conventions); + assertEquals("CF-1.7", conventions.getStringValue()); + } finally { + netcdfFile.close(); + } + } + + @Test + public void requestedResolutionsHaveExpectedGlobalBinCounts() { + assertEquals(16501208L, new SEAGrid(3600).getNumBins()); + assertEquals(2640174L, new SEAGrid(1440).getNumBins()); + } + + private static TemporalBin createBin(long index, float... values) { + TemporalBin bin = new TemporalBin(index, values.length); + bin.setNumObs(1); + bin.setNumPasses(1); + System.arraycopy(values, 0, bin.getFeatureValues(), 0, values.length); + return bin; + } + + private static void assertDimensions(Variable variable, String... names) { + assertNotNull(variable); + assertEquals(names.length, variable.getDimensions().size()); + for (int index = 0; index < names.length; index++) { + assertEquals(names[index], variable.getDimension(index).getShortName()); + } + } + + private static final class SinglePartSource implements TemporalBinSource { + + private final Iterable bins; + + private SinglePartSource(TemporalBin... bins) { + this.bins = Collections.unmodifiableList(Arrays.asList(bins)); + } + + @Override + public int open() { + return 1; + } + + @Override + public Iterator getPart(int index) { + if (index != 0) { + throw new IndexOutOfBoundsException(String.valueOf(index)); + } + return bins.iterator(); + } + + @Override + public void partProcessed(int index, Iterator part) { + } + + @Override + public void close() { + } + } +} From ffa3f91213eacd42966b27cc32b0027c08ad1455 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 22 Jul 2026 10:26:53 +0200 Subject: [PATCH 02/10] Added validator + tests and NetCDF-4 smoke test --- .../processing/l3/SeaGridNetcdfValidator.java | 203 ++++++++++++++++++ .../l3/SeaGridNetcdf4SmokeTest.java | 93 ++++++++ .../l3/SeaGridNetcdfValidatorTest.java | 143 ++++++++++++ 3 files changed, 439 insertions(+) create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidatorTest.java diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java new file mode 100644 index 000000000..d78cdadf9 --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.support.SEAGrid; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Validates the structural parts of a flattened sinusoidal NetCDF product. + * Product-specific names, types, and metadata remain outside this contract. + */ +public final class SeaGridNetcdfValidator { + + private static final Set STRUCTURAL_VARIABLE_NAMES = new HashSet( + Arrays.asList("time", "bin_index", "lat", "lon", "crs")); + + private SeaGridNetcdfValidator() { + } + + /** + * Returns all structural problems found in {@code inputFile}. + */ + public static List validate(File inputFile, int numRows) throws IOException { + if (inputFile == null) { + throw new NullPointerException("Input file is required."); + } + if (!inputFile.isFile()) { + throw new IOException("NetCDF file does not exist: " + inputFile.getAbsolutePath()); + } + + SEAGrid grid = new SEAGrid(numRows); + NetcdfFile netcdfFile = NetcdfFile.open(inputFile.getAbsolutePath()); + try { + return validate(netcdfFile, grid); + } finally { + netcdfFile.close(); + } + } + + static List validate(NetcdfFile netcdfFile, SEAGrid grid) { + List problems = new ArrayList(); + long expectedBinCount = grid.getNumBins(); + + expectDimension(netcdfFile, "time", 1, problems); + expectDimension(netcdfFile, "bin_index", expectedBinCount, problems); + + expectVariableDimensions(netcdfFile, "time", problems, "time"); + expectVariableDimensions(netcdfFile, "crs", problems, "time"); + expectVariableDimensions(netcdfFile, "lat", problems, "bin_index"); + expectVariableDimensions(netcdfFile, "lon", problems, "bin_index"); + + Variable crs = netcdfFile.findVariable("crs"); + if (crs != null) { + expectStringAttribute(crs, "grid_mapping_name", "1D binned sinusoidal", problems); + expectNumericAttribute(crs, "number_of_latitude_rows", grid.getNumRows(), problems); + expectNumericAttribute(crs, "total_number_of_bins", expectedBinCount, problems); + } + + int scienceVariableCount = 0; + for (Variable variable : netcdfFile.getVariables()) { + if (!STRUCTURAL_VARIABLE_NAMES.contains(variable.getShortName())) { + scienceVariableCount++; + expectDimensions(variable, problems, "time", "bin_index"); + } + } + if (scienceVariableCount == 0) { + problems.add("No science variables were found."); + } + + return Collections.unmodifiableList(problems); + } + + /** + * Command-line entry point for validating local or downloaded products. + */ + public static void main(String[] args) throws IOException { + if (args.length != 2) { + System.err.println("Usage: SeaGridNetcdfValidator "); + System.exit(2); + } + + final int numRows; + try { + numRows = Integer.parseInt(args[1]); + } catch (NumberFormatException e) { + System.err.println("Invalid number of latitude rows: " + args[1]); + System.exit(2); + return; + } + + File inputFile = new File(args[0]); + List problems = validate(inputFile, numRows); + if (problems.isEmpty()) { + System.out.println("Valid flattened sinusoidal structure: " + inputFile.getAbsolutePath()); + return; + } + + System.err.println("Invalid flattened sinusoidal structure: " + inputFile.getAbsolutePath()); + for (String problem : problems) { + System.err.println("- " + problem); + } + System.exit(1); + } + + private static void expectDimension(NetcdfFile netcdfFile, + String name, + long expectedLength, + List problems) { + Dimension dimension = netcdfFile.findDimension(name); + if (dimension == null) { + problems.add("Missing dimension '" + name + "'."); + } else if (dimension.getLength() != expectedLength) { + problems.add("Dimension '" + name + "' has length " + dimension.getLength() + + "; expected " + expectedLength + '.'); + } + } + + private static void expectVariableDimensions(NetcdfFile netcdfFile, + String variableName, + List problems, + String... expectedDimensions) { + Variable variable = netcdfFile.findVariable(variableName); + if (variable == null) { + problems.add("Missing variable '" + variableName + "'."); + } else { + expectDimensions(variable, problems, expectedDimensions); + } + } + + private static void expectDimensions(Variable variable, + List problems, + String... expectedDimensions) { + List dimensions = variable.getDimensions(); + if (dimensions.size() != expectedDimensions.length) { + problems.add("Variable '" + variable.getShortName() + "' has dimensions " + + dimensionNames(dimensions) + "; expected " + Arrays.toString(expectedDimensions) + '.'); + return; + } + for (int index = 0; index < expectedDimensions.length; index++) { + if (!expectedDimensions[index].equals(dimensions.get(index).getShortName())) { + problems.add("Variable '" + variable.getShortName() + "' has dimensions " + + dimensionNames(dimensions) + "; expected " + Arrays.toString(expectedDimensions) + '.'); + return; + } + } + } + + private static String dimensionNames(List dimensions) { + List names = new ArrayList(dimensions.size()); + for (Dimension dimension : dimensions) { + names.add(dimension.getShortName()); + } + return names.toString(); + } + + private static void expectStringAttribute(Variable variable, + String attributeName, + String expectedValue, + List problems) { + Attribute attribute = variable.findAttribute(attributeName); + if (attribute == null) { + problems.add("Variable '" + variable.getShortName() + "' is missing attribute '" + + attributeName + "'."); + } else if (!expectedValue.equals(attribute.getStringValue())) { + problems.add("Variable '" + variable.getShortName() + "' attribute '" + attributeName + + "' is '" + attribute.getStringValue() + "'; expected '" + expectedValue + "'."); + } + } + + private static void expectNumericAttribute(Variable variable, + String attributeName, + long expectedValue, + List problems) { + Attribute attribute = variable.findAttribute(attributeName); + if (attribute == null || attribute.getNumericValue() == null) { + problems.add("Variable '" + variable.getShortName() + "' is missing numeric attribute '" + + attributeName + "'."); + } else if (attribute.getNumericValue().longValue() != expectedValue) { + problems.add("Variable '" + variable.getShortName() + "' attribute '" + attributeName + + "' is " + attribute.getNumericValue() + "; expected " + expectedValue + '.'); + } + } +} + diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java new file mode 100644 index 000000000..4bd523fb8 --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; + +import static org.junit.Assert.assertTrue; + +/** + * Opt-in smoke test for the native NetCDF-4-classic writer. + */ +public class SeaGridNetcdf4SmokeTest { + + private File outputFile; + + @Before + public void setUp() throws IOException { + Assume.assumeTrue("Enable with -Dcalvalus.test.netcdf4=true.", + Boolean.getBoolean("calvalus.test.netcdf4")); + outputFile = File.createTempFile("calvalus-seagrid-netcdf4-", ".nc"); + } + + @After + public void tearDown() { + if (outputFile != null && outputFile.exists() && !outputFile.delete()) { + outputFile.deleteOnExit(); + } + } + + @Test + public void writesAndReadsNetcdf4Classic() throws Exception { + SEAGrid grid = new SEAGrid(4); + TemporalBin bin = new TemporalBin(2, 1); + bin.getFeatureValues()[0] = 42.0f; + + SeaGridNetcdfFormatter.write(outputFile, + grid, + new SinglePartSource(bin), + new String[]{"science_value"}, + ProductData.UTC.create(new Date(1659312000000L), 0)); + + assertTrue(SeaGridNetcdfValidator.validate(outputFile, 4).isEmpty()); + } + + private static final class SinglePartSource implements TemporalBinSource { + + private final Iterable bins; + + private SinglePartSource(TemporalBin... bins) { + this.bins = Collections.unmodifiableList(Arrays.asList(bins)); + } + + @Override + public int open() { + return 1; + } + + @Override + public Iterator getPart(int index) { + return bins.iterator(); + } + + @Override + public void partProcessed(int index, Iterator part) { + } + + @Override + public void close() { + } + } +} + diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidatorTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidatorTest.java new file mode 100644 index 000000000..8f19e7704 --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidatorTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.NetcdfFileWriter; +import ucar.nc2.Variable; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class SeaGridNetcdfValidatorTest { + + private File outputFile; + + @Before + public void setUp() throws IOException { + outputFile = File.createTempFile("calvalus-seagrid-validator-", ".nc"); + } + + @After + public void tearDown() { + if (outputFile != null && outputFile.exists() && !outputFile.delete()) { + outputFile.deleteOnExit(); + } + } + + @Test + public void acceptsWriterOutput() throws Exception { + SEAGrid grid = new SEAGrid(4); + TemporalBin bin = new TemporalBin(2, 1); + bin.getFeatureValues()[0] = 42.0f; + + SeaGridNetcdfFormatter.write(outputFile, + grid, + new SinglePartSource(bin), + new String[]{"science_value"}, + ProductData.UTC.create(new Date(1659312000000L), 0), + NetcdfFileWriter.Version.netcdf3); + + assertTrue(SeaGridNetcdfValidator.validate(outputFile, 4).isEmpty()); + } + + @Test + public void reportsScienceVariableWithoutTimeDimension() throws Exception { + writeInvalidStructure(outputFile, new SEAGrid(4)); + + List problems = SeaGridNetcdfValidator.validate(outputFile, 4); + + assertEquals(1, problems.size()); + assertEquals("Variable 'science_value' has dimensions [bin_index]; expected [time, bin_index].", + problems.get(0)); + } + + @Test + public void acceptsConfiguredReferenceProduct() throws Exception { + String referencePath = System.getProperty("calvalus.test.referenceNetcdf"); + Assume.assumeTrue("Set -Dcalvalus.test.referenceNetcdf= to validate the reference product.", + referencePath != null && !referencePath.trim().isEmpty()); + + File referenceFile = new File(referencePath); + assertTrue("Reference product does not exist: " + referenceFile, referenceFile.isFile()); + List problems = SeaGridNetcdfValidator.validate(referenceFile, 4320); + assertTrue(problems.toString(), problems.isEmpty()); + } + + private static void writeInvalidStructure(File file, SEAGrid grid) throws Exception { + NetcdfFileWriter writer = NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, + file.getAbsolutePath()); + Dimension time = writer.addDimension("time", 1); + Dimension binIndex = writer.addDimension("bin_index", (int) grid.getNumBins()); + writer.addVariable("time", DataType.INT, Arrays.asList(time)); + Variable crs = writer.addVariable("crs", DataType.INT, Arrays.asList(time)); + crs.addAttribute(new Attribute("grid_mapping_name", "1D binned sinusoidal")); + crs.addAttribute(new Attribute("number_of_latitude_rows", grid.getNumRows())); + crs.addAttribute(new Attribute("total_number_of_bins", (int) grid.getNumBins())); + writer.addVariable("lat", DataType.FLOAT, Arrays.asList(binIndex)); + writer.addVariable("lon", DataType.FLOAT, Arrays.asList(binIndex)); + Variable science = writer.addVariable("science_value", DataType.FLOAT, Arrays.asList(binIndex)); + science.addAttribute(new Attribute("_FillValue", Float.NaN)); + try { + writer.create(); + writer.write(science, Array.factory(DataType.FLOAT, + new int[]{(int) grid.getNumBins()}, + new float[(int) grid.getNumBins()])); + } finally { + writer.close(); + } + } + + private static final class SinglePartSource implements TemporalBinSource { + + private final Iterable bins; + + private SinglePartSource(TemporalBin... bins) { + this.bins = Collections.unmodifiableList(Arrays.asList(bins)); + } + + @Override + public int open() { + return 1; + } + + @Override + public Iterator getPart(int index) { + return bins.iterator(); + } + + @Override + public void partProcessed(int index, Iterator part) { + } + + @Override + public void close() { + } + } +} From b320f3084459bdc94739f3e20afbe7f0da268b37 Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 22 Jul 2026 12:12:20 +0200 Subject: [PATCH 03/10] more edage case and benchmark tests --- .../SeaGridNetcdfFormatterBenchmarkTest.java | 164 ++++++++++++++++ .../SeaGridNetcdfFormatterEdgeCasesTest.java | 176 ++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterBenchmarkTest.java create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterBenchmarkTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterBenchmarkTest.java new file mode 100644 index 000000000..c6ed03415 --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterBenchmarkTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.junit.Assume; +import org.junit.Test; + +import java.io.File; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.NoSuchElementException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Opt-in end-to-end writer benchmark. The source is generated lazily so the + * measurement also verifies that a global grid does not require all bins in memory. + */ +public class SeaGridNetcdfFormatterBenchmarkTest { + + @Test + public void benchmarksConfiguredGlobalGrid() throws Exception { + Assume.assumeTrue("Enable with -Dcalvalus.test.seagridBenchmark=true.", + Boolean.getBoolean("calvalus.test.seagridBenchmark")); + + int numRows = positiveIntProperty("calvalus.benchmark.numRows", 1440); + int featureCount = positiveIntProperty("calvalus.benchmark.featureCount", 4); + int binStride = positiveIntProperty("calvalus.benchmark.binStride", 32); + SEAGrid grid = new SEAGrid(numRows); + long totalBins = grid.getNumBins(); + long populatedBins = (totalBins + binStride - 1) / binStride; + String configuredOutput = System.getProperty("calvalus.benchmark.output"); + boolean retainOutput = configuredOutput != null && !configuredOutput.trim().isEmpty(); + File outputFile = retainOutput + ? new File(configuredOutput) + : File.createTempFile("calvalus-seagrid-benchmark-" + numRows + "-", ".nc"); + GeneratedSource source = new GeneratedSource(totalBins, featureCount, binStride); + + try { + long startedNanos = System.nanoTime(); + SeaGridNetcdfFormatter.write(outputFile, + grid, + source, + featureNames(featureCount), + null); + double elapsedSeconds = (System.nanoTime() - startedNanos) / 1.0e9; + long outputBytes = outputFile.length(); + double totalBinsPerSecond = totalBins / elapsedSeconds; + + List problems = SeaGridNetcdfValidator.validate(outputFile, numRows); + assertTrue(problems.toString(), problems.isEmpty()); + assertEquals(1, source.processedPartCount); + assertTrue(source.closed); + assertEquals(populatedBins, source.generatedBinCount); + assertTrue("Benchmark output must not be empty.", outputBytes > 0); + + System.out.printf(Locale.ENGLISH, + "SEAGRID_BENCHMARK numRows=%d totalBins=%d populatedBins=%d " + + "featureCount=%d stride=%d elapsedSeconds=%.3f outputBytes=%d " + + "totalBinsPerSecond=%.0f%n", + numRows, totalBins, populatedBins, featureCount, binStride, + elapsedSeconds, outputBytes, totalBinsPerSecond); + } finally { + if (!retainOutput && outputFile.exists() && !outputFile.delete()) { + outputFile.deleteOnExit(); + } + } + } + + private static int positiveIntProperty(String name, int defaultValue) { + String text = System.getProperty(name); + int value = text == null ? defaultValue : Integer.parseInt(text); + if (value <= 0) { + throw new IllegalArgumentException(name + " must be positive: " + value); + } + return value; + } + + private static String[] featureNames(int featureCount) { + String[] names = new String[featureCount]; + for (int index = 0; index < names.length; index++) { + names[index] = "science_" + index; + } + return names; + } + + private static final class GeneratedSource implements TemporalBinSource { + + private final long totalBins; + private final int featureCount; + private final int stride; + private long generatedBinCount; + private int processedPartCount; + private boolean closed; + + private GeneratedSource(long totalBins, int featureCount, int stride) { + this.totalBins = totalBins; + this.featureCount = featureCount; + this.stride = stride; + } + + @Override + public int open() { + return 1; + } + + @Override + public Iterator getPart(int index) { + if (index != 0) { + throw new IndexOutOfBoundsException(String.valueOf(index)); + } + return new Iterator() { + private long binIndex; + + @Override + public boolean hasNext() { + return binIndex < totalBins; + } + + @Override + public TemporalBin next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + TemporalBin bin = new TemporalBin(binIndex, featureCount); + float[] values = bin.getFeatureValues(); + for (int featureIndex = 0; featureIndex < values.length; featureIndex++) { + values[featureIndex] = (float) (binIndex % 1000) + featureIndex; + } + generatedBinCount++; + binIndex += stride; + return bin; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public void partProcessed(int index, Iterator part) { + processedPartCount++; + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java new file mode 100644 index 000000000..de0734f70 --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.TemporalBin; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.support.SEAGrid; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFileWriter; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class SeaGridNetcdfFormatterEdgeCasesTest { + + private File outputFile; + + @Before + public void setUp() throws IOException { + outputFile = File.createTempFile("calvalus-seagrid-edge-", ".nc"); + } + + @After + public void tearDown() { + if (outputFile != null && outputFile.exists() && !outputFile.delete()) { + outputFile.deleteOnExit(); + } + } + + @Test + public void streamsMultiplePartsAcrossBufferSizedGap() throws Exception { + SEAGrid grid = new SEAGrid(360); + TrackingSource source = new TrackingSource(Arrays.asList(createBin(0, 1.0f, 2.0f)), + Arrays.asList(createBin(10000, 3.0f, 4.0f))); + + write(grid, source, new String[]{"first", "second"}); + + assertEquals(2, source.processedPartCount); + assertTrue(source.closed); + NetcdfFile netcdfFile = NetcdfFile.open(outputFile.getAbsolutePath()); + try { + Array first = netcdfFile.findVariable("first").read(); + Array second = netcdfFile.findVariable("second").read(); + assertEquals(1.0f, first.getFloat(0), 0.0f); + assertTrue(Float.isNaN(first.getFloat(9999))); + assertEquals(3.0f, first.getFloat(10000), 0.0f); + assertEquals(4.0f, second.getFloat(10000), 0.0f); + } finally { + netcdfFile.close(); + } + } + + @Test + public void rejectsUnorderedAndDuplicateBinsAndClosesSource() throws Exception { + assertBadBins(new TemporalBin[]{createBin(5, 1.0f), createBin(4, 2.0f)}, + "ordered by increasing global bin index"); + assertBadBins(new TemporalBin[]{createBin(5, 1.0f), createBin(5, 2.0f)}, + "ordered by increasing global bin index"); + } + + @Test + public void rejectsOutOfRangeIndexAndWrongFeatureCount() throws Exception { + SEAGrid grid = new SEAGrid(4); + assertBadBins(grid, + new TemporalBin[]{createBin(grid.getNumBins(), 1.0f)}, + new String[]{"value"}, + "outside the sinusoidal grid"); + assertBadBins(grid, + new TemporalBin[]{createBin(0, 1.0f)}, + new String[]{"first", "second"}, + "has 1 features; expected 2"); + } + + @Test + public void rejectsReservedDuplicateAndEmptyFeatureNamesBeforeOpeningSource() throws Exception { + assertBadNames(new String[]{"time"}, "Reserved NetCDF variable name"); + assertBadNames(new String[]{"value", "value"}, "Duplicate science-variable name"); + assertBadNames(new String[]{" "}, "must not be empty"); + } + + private void assertBadBins(TemporalBin[] bins, String expectedMessage) throws Exception { + assertBadBins(new SEAGrid(4), bins, new String[]{"value"}, expectedMessage); + } + + private void assertBadBins(SEAGrid grid, + TemporalBin[] bins, + String[] featureNames, + String expectedMessage) throws Exception { + TrackingSource source = new TrackingSource(Arrays.asList(bins)); + try { + write(grid, source, featureNames); + fail("Expected IOException containing: " + expectedMessage); + } catch (IOException expected) { + assertTrue(expected.getMessage(), expected.getMessage().contains(expectedMessage)); + } + assertTrue("Source must close when bin validation fails.", source.closed); + } + + private void assertBadNames(String[] featureNames, String expectedMessage) throws Exception { + TrackingSource source = new TrackingSource(Arrays.asList(createBin(0, 1.0f))); + try { + write(new SEAGrid(4), source, featureNames); + fail("Expected IllegalArgumentException containing: " + expectedMessage); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage(), expected.getMessage().contains(expectedMessage)); + } + assertTrue("Argument validation must happen before opening the source.", !source.opened); + } + + private void write(SEAGrid grid, TrackingSource source, String[] featureNames) throws IOException { + SeaGridNetcdfFormatter.write(outputFile, + grid, + source, + featureNames, + null, + NetcdfFileWriter.Version.netcdf3); + } + + private static TemporalBin createBin(long index, float... values) { + TemporalBin bin = new TemporalBin(index, values.length); + System.arraycopy(values, 0, bin.getFeatureValues(), 0, values.length); + return bin; + } + + private static final class TrackingSource implements TemporalBinSource { + + private final List[] parts; + private boolean opened; + private boolean closed; + private int processedPartCount; + + @SafeVarargs + private TrackingSource(List... parts) { + this.parts = parts; + } + + @Override + public int open() { + opened = true; + return parts.length; + } + + @Override + public Iterator getPart(int index) { + return parts[index].iterator(); + } + + @Override + public void partProcessed(int index, Iterator part) { + processedPartCount++; + } + + @Override + public void close() { + closed = true; + } + } +} From 926a2feb87b6845db018599ade2b5a728dc5c48f Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 22 Jul 2026 15:50:05 +0200 Subject: [PATCH 04/10] use https instead of http --- calvalus-snap/pom.xml | 4 ++-- pom.xml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/calvalus-snap/pom.xml b/calvalus-snap/pom.xml index ead2a3264..535f85b5a 100644 --- a/calvalus-snap/pom.xml +++ b/calvalus-snap/pom.xml @@ -576,7 +576,7 @@ bc-nexus-repo Public Nexus Repository for BC - http://nexus.senbox.net/nexus/content/repositories/public/ + https://nexus.senbox.net/nexus/content/repositories/public/ true warn @@ -602,7 +602,7 @@ osgeo Open Source Geospatial Foundation Repository - http://download.osgeo.org/webdav/geotools/ + https://download.osgeo.org/webdav/geotools/ diff --git a/pom.xml b/pom.xml index 1df7e2755..a7e41ec47 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ bc-nexus-repo Public Nexus Repository for BC - http://nexus.senbox.net/nexus/content/repositories/public/ + https://nexus.senbox.net/nexus/content/repositories/public/ true warn @@ -74,7 +74,7 @@ osgeo Open Source Geospatial Foundation Repository - http://download.osgeo.org/webdav/geotools/ + https://download.osgeo.org/webdav/geotools/ @@ -90,13 +90,13 @@ bc-nexus-repo Public Nexus Repository for BC - http://nexus.senbox.net/nexus/content/repositories/releases/ + https://nexus.senbox.net/nexus/content/repositories/releases/ true bc-nexus-repo Public Nexus Repository for BC - http://nexus.senbox.net/nexus/content/repositories/snapshots/ + https://nexus.senbox.net/nexus/content/repositories/snapshots/ false @@ -134,11 +134,11 @@ 2.9.1 - http://download.oracle.com/javase/8/docs/api/ - http://hadoop.apache.org/docs/r2.6.0/api/ - http://step.esa.int/docs/v2.0/apidoc/engine/ + https://download.oracle.com/javase/8/docs/api/ + https://hadoop.apache.org/docs/r2.6.0/api/ + https://step.esa.int/docs/v2.0/apidoc/engine/ - http://google-web-toolkit.googlecode.com/svn/javadoc/1.5/ + https://google-web-toolkit.googlecode.com/svn/javadoc/1.5/ true true From cbbf83d566e6c1b07fa560352ed59a70823f8b10 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 30 Jul 2026 10:55:25 +0200 Subject: [PATCH 05/10] Add SEAGrid NetCDF formatter with south-to-north mirroring and tests --- .../processing/l2/ProductFormatter.java | 8 ++ .../calvalus/processing/l3/L3Formatter.java | 34 ++++- .../processing/l3/SeaGridNetcdfFormatter.java | 124 ++++++++++++------ .../processing/l2/ProductFormatterTest.java | 30 +++++ .../SeaGridNetcdfFormatterEdgeCasesTest.java | 17 ++- .../l3/SeaGridNetcdfFormatterTest.java | 34 ++++- 6 files changed, 195 insertions(+), 52 deletions(-) create mode 100644 calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java index eefb6fe37..862b1244e 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java @@ -46,6 +46,8 @@ */ public class ProductFormatter { + public static final String FORMAT_NETCDF4_SEAGRID = "NetCDF4-SEAGrid"; + private static final Logger LOG = CalvalusLogger.getLogger(); private final String outputFormat; @@ -91,6 +93,12 @@ public ProductFormatter(String productName, String outputFormat, String desiredO outputExtension = ".nc"; outputCompression = ""; // no further compression required outputFormat = "NetCDF4-BEAM"; // use NetCDF with BEAM extensions + } else if (outputFormat.equalsIgnoreCase(FORMAT_NETCDF4_SEAGRID)) { + outputExtension = ".nc"; + outputCompression = ""; // already written as NetCDF-4 + // ProductFormatter provides the temporary file and HDFS copy only. + // L3Formatter selects the dedicated writer from the requested format. + outputFormat = "NetCDF4-BEAM"; } else if (outputFormat.equals("GeoTIFF")) { outputExtension = ".tif"; outputCompression = desiredOutputCompression; diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java index fb308f99c..b2e1904e0 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java @@ -29,6 +29,7 @@ import com.bc.ceres.binding.ConverterRegistry; import org.esa.snap.binning.operator.formatter.FormatterFactory; import org.esa.snap.binning.support.IsinPlanetaryGrid; +import org.esa.snap.binning.support.SEAGrid; import org.locationtech.jts.geom.Geometry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -36,7 +37,6 @@ import org.esa.snap.binning.PlanetaryGrid; import org.esa.snap.binning.TemporalBinSource; import org.esa.snap.binning.operator.BinningConfig; -import org.esa.snap.binning.operator.formatter.Formatter; import org.esa.snap.binning.operator.formatter.FormatterConfig; import org.esa.snap.core.datamodel.MetadataElement; import org.esa.snap.core.datamodel.Product; @@ -63,6 +63,7 @@ public class L3Formatter { private final Configuration configuration; private final PlanetaryGrid planetaryGrid; private final String[] featureNames; + private final File outputFile; private final MetadataSerializer metadataSerializer; private final BinningConfig binningConfig; private FormatterConfig formatterConfig; @@ -74,6 +75,7 @@ private L3Formatter(String dateStart, String dateStop, String outputFile, String this.startTime = parseTime(dateStart); this.endTime = parseTime(dateStop); this.configuration = conf; + this.outputFile = new File(outputFile); featureNames = conf.getStrings(JobConfigNames.CALVALUS_L3_FEATURE_NAMES); String formatterXML = conf.get(JobConfigNames.CALVALUS_L3_FORMAT_PARAMETERS); @@ -89,7 +91,27 @@ private L3Formatter(String dateStart, String dateStop, String outputFile, String metadataSerializer = new MetadataSerializer(); } - private void format(TemporalBinSource temporalBinSource, String regionName, String regionWKT) throws Exception { + private void format(TemporalBinSource temporalBinSource, + String regionName, + String regionWKT, + boolean useSeaGridNetcdfFormatter) throws Exception { + if (useSeaGridNetcdfFormatter) { + if (!(planetaryGrid instanceof SEAGrid)) { + throw new IllegalArgumentException( + ProductFormatter.FORMAT_NETCDF4_SEAGRID + + " requires org.esa.snap.binning.support.SEAGrid, but the request uses " + + planetaryGrid.getClass().getName()); + } + LOG.info("Using flattened SEAGrid NetCDF formatter for output format " + + ProductFormatter.FORMAT_NETCDF4_SEAGRID + '.'); + SeaGridNetcdfFormatter.write(outputFile, + (SEAGrid) planetaryGrid, + temporalBinSource, + featureNames, + startTime); + return; + } + Geometry regionGeometry = GeometryUtils.createGeometry(regionWKT); final String processingHistoryXml = configuration.get(JobConfigNames.PROCESSING_HISTORY); final MetadataElement processingGraphMetadata = metadataSerializer.fromXml(processingHistoryXml); @@ -138,6 +160,7 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo String format = conf.get(JobConfigNames.CALVALUS_OUTPUT_FORMAT, null); String compression = conf.get(JobConfigNames.CALVALUS_OUTPUT_COMPRESSION, null); BinningConfig binningConfig = HadoopBinManager.getBinningConfig(conf); + boolean useSeaGridNetcdfFormatter = usesSeaGridNetcdfFormatter(format); ProductFormatter productFormatter; if ("org.esa.snap.binning.support.IsinPlanetaryGrid".equals(binningConfig.getPlanetaryGrid())){ productFormatter = new ProductFormatter(productName, "dir",null); @@ -153,7 +176,7 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo conf); LOG.info("Start formatting product to file: " + productFile.getName()); context.setStatus("formatting"); - formatter.format(temporalBinSource, regionName, regionWKT); + formatter.format(temporalBinSource, regionName, regionWKT, useSeaGridNetcdfFormatter); LOG.info("Finished formatting product."); context.setStatus("copying"); @@ -174,6 +197,11 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo } } + static boolean usesSeaGridNetcdfFormatter(String outputFormat) { + return outputFormat != null && + ProductFormatter.FORMAT_NETCDF4_SEAGRID.equalsIgnoreCase(outputFormat); + } + private static class ProductConverter implements Converter { private final Configuration conf; diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java index 939452d99..2ccd22afa 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java @@ -35,8 +35,10 @@ * * The structural model follows the OC-CCI reference products: science * variables use {@code (time, bin_index)}, while latitude and longitude use - * {@code (bin_index)}. Product-specific metadata is intentionally left to a - * later post-processing step. + * {@code (bin_index)}. The bin-row order is south to north, matching the + * reference; longitude order within each row is preserved. + * Product-specific metadata is intentionally left to a later post-processing + * step. */ final class SeaGridNetcdfFormatter { @@ -68,6 +70,7 @@ static void write(File outputFile, ProductData.UTC startTime, NetcdfFileWriter.Version version) throws IOException { validateArguments(outputFile, planetaryGrid, temporalBinSource, featureNames); + validateMirroredRows(planetaryGrid); final long numBinsLong = planetaryGrid.getNumBins(); if (numBinsLong > Integer.MAX_VALUE) { @@ -122,9 +125,10 @@ static void write(File outputFile, try { writer.create(); writeScalarVariables(writer, timeVariable, crsVariable, startTime); - writeCoordinates(writer, planetaryGrid, latitudeVariable, longitudeVariable, numBins); + writeCoordinates(writer, planetaryGrid, latitudeVariable, longitudeVariable); - final FeatureBuffer featureBuffer = new FeatureBuffer(writer, featureVariables, numBins); + final FeatureBuffer featureBuffer = + new FeatureBuffer(writer, featureVariables, planetaryGrid); final int partCount = temporalBinSource.open(); sourceOpened = true; for (int partIndex = 0; partIndex < partCount; partIndex++) { @@ -165,8 +169,10 @@ private static void validateArguments(File outputFile, SEAGrid planetaryGrid, TemporalBinSource temporalBinSource, String[] featureNames) { - if (outputFile == null || planetaryGrid == null || temporalBinSource == null || featureNames == null) { - throw new NullPointerException("Output file, grid, bin source, and feature names are required."); + if (outputFile == null || planetaryGrid == null || temporalBinSource == null || + featureNames == null) { + throw new NullPointerException( + "Output file, grid, bin source, and feature names are required."); } if (featureNames.length == 0) { throw new IllegalArgumentException("At least one science variable is required."); @@ -185,6 +191,17 @@ private static void validateArguments(File outputFile, } } + private static void validateMirroredRows(SEAGrid planetaryGrid) throws IOException { + int numRows = planetaryGrid.getNumRows(); + for (int row = 0; row < numRows / 2; row++) { + int mirroredRow = numRows - 1 - row; + if (planetaryGrid.getNumCols(row) != planetaryGrid.getNumCols(mirroredRow)) { + throw new IOException("Cannot reverse SEAGrid rows " + row + " and " + mirroredRow + + " because their column counts differ."); + } + } + } + private static void writeScalarVariables(NetcdfFileWriter writer, Variable timeVariable, Variable crsVariable, @@ -198,22 +215,32 @@ private static void writeScalarVariables(NetcdfFileWriter writer, private static void writeCoordinates(NetcdfFileWriter writer, SEAGrid planetaryGrid, Variable latitudeVariable, - Variable longitudeVariable, - int numBins) + Variable longitudeVariable) throws IOException, InvalidRangeException { - for (int origin = 0; origin < numBins; origin += BUFFER_SIZE) { - int length = Math.min(BUFFER_SIZE, numBins - origin); - float[] latitudes = new float[length]; - float[] longitudes = new float[length]; - for (int offset = 0; offset < length; offset++) { - double[] center = planetaryGrid.getCenterLatLon((long) origin + offset); - latitudes[offset] = (float) center[0]; - longitudes[offset] = (float) center[1]; + int numRows = planetaryGrid.getNumRows(); + for (int outputRow = 0; outputRow < numRows; outputRow++) { + // SNAP's SEAGrid enumerates rows north-to-south. Mirror them on output to work around + // this limitation and produce the south-to-north bin order used by standard L3 products. + int sourceRow = numRows - 1 - outputRow; + int numCols = planetaryGrid.getNumCols(outputRow); + long outputRowStart = planetaryGrid.getFirstBinIndex(outputRow); + long sourceRowStart = planetaryGrid.getFirstBinIndex(sourceRow); + for (int columnOrigin = 0; columnOrigin < numCols; columnOrigin += BUFFER_SIZE) { + int length = Math.min(BUFFER_SIZE, numCols - columnOrigin); + float[] latitudes = new float[length]; + float[] longitudes = new float[length]; + for (int offset = 0; offset < length; offset++) { + double[] center = planetaryGrid.getCenterLatLon( + sourceRowStart + columnOrigin + offset); + latitudes[offset] = (float) center[0]; + longitudes[offset] = (float) center[1]; + } + int origin = (int) (outputRowStart + columnOrigin); + writer.write(latitudeVariable, new int[]{origin}, + Array.factory(DataType.FLOAT, new int[]{length}, latitudes)); + writer.write(longitudeVariable, new int[]{origin}, + Array.factory(DataType.FLOAT, new int[]{length}, longitudes)); } - writer.write(latitudeVariable, new int[]{origin}, - Array.factory(DataType.FLOAT, new int[]{length}, latitudes)); - writer.write(longitudeVariable, new int[]{origin}, - Array.factory(DataType.FLOAT, new int[]{length}, longitudes)); } } @@ -221,18 +248,27 @@ private static final class FeatureBuffer { private final NetcdfFileWriter writer; private final List variables; - private final int numBins; + private final SEAGrid planetaryGrid; + private final long numBins; private final float[][] values; - private long startIndex = -1; private long lastIndex = -1; - private int length; + private int sourceRow = -1; + private int minColumn; + private int maxColumn; - private FeatureBuffer(NetcdfFileWriter writer, List variables, int numBins) { + private FeatureBuffer(NetcdfFileWriter writer, + List variables, + SEAGrid planetaryGrid) { this.writer = writer; this.variables = variables; - this.numBins = numBins; - values = new float[variables.size()][BUFFER_SIZE]; + this.planetaryGrid = planetaryGrid; + this.numBins = planetaryGrid.getNumBins(); + int maxNumCols = 0; + for (int row = 0; row < planetaryGrid.getNumRows(); row++) { + maxNumCols = Math.max(maxNumCols, planetaryGrid.getNumCols(row)); + } + values = new float[variables.size()][maxNumCols]; } private void add(TemporalBin temporalBin) throws IOException, InvalidRangeException { @@ -248,39 +284,47 @@ private void add(TemporalBin temporalBin) throws IOException, InvalidRangeExcept temporalBin.getFeatureValues().length + " features; expected " + variables.size() + '.'); } - if (startIndex < 0 || binIndex >= startIndex + BUFFER_SIZE) { + int binRow = planetaryGrid.getRowIndex(binIndex); + if (sourceRow != binRow) { flush(); - reset(binIndex); + reset(binRow); } - int offset = (int) (binIndex - startIndex); + int column = (int) (binIndex - planetaryGrid.getFirstBinIndex(sourceRow)); + float[] featureValues = temporalBin.getFeatureValues(); for (int featureIndex = 0; featureIndex < values.length; featureIndex++) { - values[featureIndex][offset] = temporalBin.getFeatureValues()[featureIndex]; + values[featureIndex][column] = featureValues[featureIndex]; } - length = Math.max(length, offset + 1); + minColumn = Math.min(minColumn, column); + maxColumn = Math.max(maxColumn, column); lastIndex = binIndex; } - private void reset(long newStartIndex) { - startIndex = newStartIndex; - length = 0; + private void reset(int newSourceRow) { + sourceRow = newSourceRow; + int numCols = planetaryGrid.getNumCols(sourceRow); for (float[] featureValues : values) { - Arrays.fill(featureValues, Float.NaN); + Arrays.fill(featureValues, 0, numCols, Float.NaN); } + minColumn = numCols; + maxColumn = -1; } private void flush() throws IOException, InvalidRangeException { - if (length == 0) { + if (sourceRow < 0 || maxColumn < minColumn) { return; } - int[] origin = new int[]{0, (int) startIndex}; + // Apply the same SNAP-to-standard row-order conversion as for the coordinate variables. + int outputRow = planetaryGrid.getNumRows() - 1 - sourceRow; + long outputRowStart = planetaryGrid.getFirstBinIndex(outputRow); + int length = maxColumn - minColumn + 1; + int[] origin = new int[]{0, (int) (outputRowStart + minColumn)}; for (int featureIndex = 0; featureIndex < variables.size(); featureIndex++) { - float[] data = Arrays.copyOf(values[featureIndex], length); + float[] data = Arrays.copyOfRange(values[featureIndex], minColumn, maxColumn + 1); writer.write(variables.get(featureIndex), origin, Array.factory(DataType.FLOAT, new int[]{1, length}, data)); } - startIndex = -1; - length = 0; + sourceRow = -1; } } } diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java new file mode 100644 index 000000000..cef789b84 --- /dev/null +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2026 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + */ + +package com.bc.calvalus.processing.l2; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class ProductFormatterTest { + + @Test + public void preparesUncompressedNetcdfFileForSeaGridWriter() { + ProductFormatter formatter = new ProductFormatter( + "example", + ProductFormatter.FORMAT_NETCDF4_SEAGRID, + "none"); + + assertEquals("NetCDF4-BEAM", formatter.getOutputFormat()); + assertEquals("example.nc", formatter.getProductFilename()); + assertEquals("example.nc", formatter.getOutputFilename()); + assertEquals("", formatter.getOutputCompression()); + } +} diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java index de0734f70..f8568821f 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java @@ -59,10 +59,12 @@ public void streamsMultiplePartsAcrossBufferSizedGap() throws Exception { try { Array first = netcdfFile.findVariable("first").read(); Array second = netcdfFile.findVariable("second").read(); - assertEquals(1.0f, first.getFloat(0), 0.0f); - assertTrue(Float.isNaN(first.getFloat(9999))); - assertEquals(3.0f, first.getFloat(10000), 0.0f); - assertEquals(4.0f, second.getFloat(10000), 0.0f); + int firstOutputIndex = outputIndex(grid, 0); + int secondOutputIndex = outputIndex(grid, 10000); + assertEquals(1.0f, first.getFloat(firstOutputIndex), 0.0f); + assertTrue(Float.isNaN(first.getFloat(secondOutputIndex - 1))); + assertEquals(3.0f, first.getFloat(secondOutputIndex), 0.0f); + assertEquals(4.0f, second.getFloat(secondOutputIndex), 0.0f); } finally { netcdfFile.close(); } @@ -140,6 +142,13 @@ private static TemporalBin createBin(long index, float... values) { return bin; } + private static int outputIndex(SEAGrid grid, long sourceIndex) { + int sourceRow = grid.getRowIndex(sourceIndex); + int outputRow = grid.getNumRows() - 1 - sourceRow; + long column = sourceIndex - grid.getFirstBinIndex(sourceRow); + return (int) (grid.getFirstBinIndex(outputRow) + column); + } + private static final class TrackingSource implements TemporalBinSource { private final List[] parts; diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java index 9020441ec..6a649e31c 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java @@ -89,11 +89,20 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { Array chlorA = netcdfFile.findVariable("chlor_a").read(); Array totalNobs = netcdfFile.findVariable("total_nobs").read(); - assertEquals(1.25f, chlorA.getFloat(0), 0.0f); - assertEquals(3.5f, totalNobs.getFloat(0), 0.0f); - assertTrue(Float.isNaN(chlorA.getFloat(1))); - assertEquals(2.5f, chlorA.getFloat(10), 0.0f); - assertEquals(7.0f, totalNobs.getFloat(10), 0.0f); + int firstOutputIndex = outputIndex(grid, 0); + int secondOutputIndex = outputIndex(grid, 10); + assertEquals(1.25f, chlorA.getFloat(firstOutputIndex), 0.0f); + assertEquals(3.5f, totalNobs.getFloat(firstOutputIndex), 0.0f); + assertTrue(Float.isNaN(chlorA.getFloat(firstOutputIndex + 1))); + assertEquals(2.5f, chlorA.getFloat(secondOutputIndex), 0.0f); + assertEquals(7.0f, totalNobs.getFloat(secondOutputIndex), 0.0f); + + Array latitudes = netcdfFile.findVariable("lat").read(); + Array longitudes = netcdfFile.findVariable("lon").read(); + assertTrue(latitudes.getFloat(0) < 0.0f); + assertTrue(latitudes.getFloat((int) grid.getNumBins() - 1) > 0.0f); + assertTrue(longitudes.getFloat(0) < + longitudes.getFloat(grid.getNumCols(0) - 1)); Attribute conventions = netcdfFile.findGlobalAttribute("Conventions"); assertNotNull(conventions); @@ -109,6 +118,14 @@ public void requestedResolutionsHaveExpectedGlobalBinCounts() { assertEquals(2640174L, new SEAGrid(1440).getNumBins()); } + @Test + public void dedicatedOutputFormatSelectsSeaGridFormatter() { + assertTrue(L3Formatter.usesSeaGridNetcdfFormatter("NetCDF4-SEAGrid")); + assertTrue(L3Formatter.usesSeaGridNetcdfFormatter("netcdf4-seagrid")); + assertTrue(!L3Formatter.usesSeaGridNetcdfFormatter("NetCDF4-BEAM")); + assertTrue(!L3Formatter.usesSeaGridNetcdfFormatter(null)); + } + private static TemporalBin createBin(long index, float... values) { TemporalBin bin = new TemporalBin(index, values.length); bin.setNumObs(1); @@ -117,6 +134,13 @@ private static TemporalBin createBin(long index, float... values) { return bin; } + private static int outputIndex(SEAGrid grid, long sourceIndex) { + int sourceRow = grid.getRowIndex(sourceIndex); + int outputRow = grid.getNumRows() - 1 - sourceRow; + long column = sourceIndex - grid.getFirstBinIndex(sourceRow); + return (int) (grid.getFirstBinIndex(outputRow) + column); + } + private static void assertDimensions(Variable variable, String... names) { assertNotNull(variable); assertEquals(names.length, variable.getDimensions().size()); From f75e0500fcd0c6b2d03cd91dae964c4e36474724 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 30 Jul 2026 11:59:59 +0200 Subject: [PATCH 06/10] Add NetCDF4-SEAGrid format to available output types --- .../com/bc/calvalus/portal/client/OrderL3ProductionView.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL3ProductionView.java b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL3ProductionView.java index 663faa62b..7c9181db5 100644 --- a/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL3ProductionView.java +++ b/calvalus-portal/src/main/java/com/bc/calvalus/portal/client/OrderL3ProductionView.java @@ -113,7 +113,8 @@ public void onChange(ChangeEvent event) { l3ConfigForm.compositingPeriodLength.setValue(30); outputParametersForm = new OutputParametersForm(portalContext); - outputParametersForm.setAvailableOutputFormats("BEAM-DIMAP", "NetCDF", "NetCDF4", "GeoTIFF", "BigGeoTiff"); + outputParametersForm.setAvailableOutputFormats("BEAM-DIMAP", "NetCDF", "NetCDF4", "NetCDF4-SEAGrid", + "GeoTIFF", "BigGeoTiff"); l2ConfigForm.setProductSet(productSetSelectionForm.getSelectedProductSet()); updateTemporalParameters(productSetFilterForm.getValueMap()); @@ -279,4 +280,4 @@ public void setProductionParameters(Map parameters) { l3ConfigForm.setValues(parameters); outputParametersForm.setValues(parameters); } -} \ No newline at end of file +} From e37ec35164263a347296b7ce64237689a4565a1c Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 30 Jul 2026 12:11:17 +0200 Subject: [PATCH 07/10] Simplify `usesSeaGridNetcdfFormatter` null check --- .../main/java/com/bc/calvalus/processing/l3/L3Formatter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java index b2e1904e0..fe8912f71 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java @@ -198,8 +198,7 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo } static boolean usesSeaGridNetcdfFormatter(String outputFormat) { - return outputFormat != null && - ProductFormatter.FORMAT_NETCDF4_SEAGRID.equalsIgnoreCase(outputFormat); + return ProductFormatter.FORMAT_NETCDF4_SEAGRID.equalsIgnoreCase(outputFormat); } private static class ProductConverter implements Converter { From af38f0fbcbffe5fa251701b9e2efbeaedabc13b8 Mon Sep 17 00:00:00 2001 From: Marco Date: Thu, 30 Jul 2026 12:43:16 +0200 Subject: [PATCH 08/10] Add SeaGrid NetCDF formatter plugin and integration with SNAP factory --- .../processing/l2/ProductFormatter.java | 32 ++++----- .../calvalus/processing/l3/L3Formatter.java | 52 +++++++-------- .../processing/l3/SeaGridFormatter.java | 65 +++++++++++++++++++ .../processing/l3/SeaGridFormatterPlugin.java | 39 +++++++++++ ...binning.operator.formatter.FormatterPlugin | 1 + .../processing/l2/ProductFormatterTest.java | 2 +- .../l3/SeaGridNetcdfFormatterTest.java | 8 ++- 7 files changed, 152 insertions(+), 47 deletions(-) create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java create mode 100644 calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatterPlugin.java create mode 100644 calvalus-processing/src/main/resources/META-INF/services/org.esa.snap.binning.operator.formatter.FormatterPlugin diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java index 862b1244e..d979e4676 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l2/ProductFormatter.java @@ -96,9 +96,9 @@ public ProductFormatter(String productName, String outputFormat, String desiredO } else if (outputFormat.equalsIgnoreCase(FORMAT_NETCDF4_SEAGRID)) { outputExtension = ".nc"; outputCompression = ""; // already written as NetCDF-4 - // ProductFormatter provides the temporary file and HDFS copy only. - // L3Formatter selects the dedicated writer from the requested format. - outputFormat = "NetCDF4-BEAM"; + // L3Formatter uses the dedicated binning FormatterPlugin; ProductFormatter + // only provides the temporary file and HDFS copy, so no ProductWriterPlugIn + // is required. } else if (outputFormat.equals("GeoTIFF")) { outputExtension = ".tif"; outputCompression = desiredOutputCompression; @@ -115,20 +115,22 @@ public ProductFormatter(String productName, String outputFormat, String desiredO } else { outputCompression = desiredOutputCompression; } - // test if writer for output format exists - ProductIOPlugInManager registry = ProductIOPlugInManager.getInstance(); - Iterator it = registry.getWriterPlugIns(outputFormat); - if(it.hasNext()) { - ProductWriterPlugIn plugIn = (ProductWriterPlugIn) it.next(); - if (outputExtension.isEmpty()) { - // get output extension from writer - String[] defaultFileExtensions = plugIn.getDefaultFileExtensions(); - if (defaultFileExtensions != null && defaultFileExtensions.length > 0) { - outputExtension = defaultFileExtensions[0]; + if (!FORMAT_NETCDF4_SEAGRID.equalsIgnoreCase(outputFormat)) { + // test if writer for output format exists + ProductIOPlugInManager registry = ProductIOPlugInManager.getInstance(); + Iterator it = registry.getWriterPlugIns(outputFormat); + if(it.hasNext()) { + ProductWriterPlugIn plugIn = (ProductWriterPlugIn) it.next(); + if (outputExtension.isEmpty()) { + // get output extension from writer + String[] defaultFileExtensions = plugIn.getDefaultFileExtensions(); + if (defaultFileExtensions != null && defaultFileExtensions.length > 0) { + outputExtension = defaultFileExtensions[0]; + } } + } else { + throw new IllegalArgumentException("Unsupported output format: " + outputFormat); } - } else { - throw new IllegalArgumentException("Unsupported output format: " + outputFormat); } if ("zip".equals(outputCompression)) { diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java index fe8912f71..33caf39c0 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java @@ -27,9 +27,9 @@ import com.bc.ceres.binding.ConversionException; import com.bc.ceres.binding.Converter; import com.bc.ceres.binding.ConverterRegistry; +import org.esa.snap.binning.operator.formatter.Formatter; import org.esa.snap.binning.operator.formatter.FormatterFactory; import org.esa.snap.binning.support.IsinPlanetaryGrid; -import org.esa.snap.binning.support.SEAGrid; import org.locationtech.jts.geom.Geometry; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -63,7 +63,6 @@ public class L3Formatter { private final Configuration configuration; private final PlanetaryGrid planetaryGrid; private final String[] featureNames; - private final File outputFile; private final MetadataSerializer metadataSerializer; private final BinningConfig binningConfig; private FormatterConfig formatterConfig; @@ -75,7 +74,6 @@ private L3Formatter(String dateStart, String dateStop, String outputFile, String this.startTime = parseTime(dateStart); this.endTime = parseTime(dateStop); this.configuration = conf; - this.outputFile = new File(outputFile); featureNames = conf.getStrings(JobConfigNames.CALVALUS_L3_FEATURE_NAMES); String formatterXML = conf.get(JobConfigNames.CALVALUS_L3_FORMAT_PARAMETERS); @@ -93,35 +91,30 @@ private L3Formatter(String dateStart, String dateStop, String outputFile, String private void format(TemporalBinSource temporalBinSource, String regionName, - String regionWKT, - boolean useSeaGridNetcdfFormatter) throws Exception { + String regionWKT) throws Exception { + boolean useSeaGridNetcdfFormatter = + usesSeaGridNetcdfFormatter(formatterConfig.getOutputFormat()); + Formatter formatter; if (useSeaGridNetcdfFormatter) { - if (!(planetaryGrid instanceof SEAGrid)) { - throw new IllegalArgumentException( - ProductFormatter.FORMAT_NETCDF4_SEAGRID + - " requires org.esa.snap.binning.support.SEAGrid, but the request uses " + - planetaryGrid.getClass().getName()); - } - LOG.info("Using flattened SEAGrid NetCDF formatter for output format " + - ProductFormatter.FORMAT_NETCDF4_SEAGRID + '.'); - SeaGridNetcdfFormatter.write(outputFile, - (SEAGrid) planetaryGrid, - temporalBinSource, - featureNames, - startTime); - return; - } - - Geometry regionGeometry = GeometryUtils.createGeometry(regionWKT); - final String processingHistoryXml = configuration.get(JobConfigNames.PROCESSING_HISTORY); - final MetadataElement processingGraphMetadata = metadataSerializer.fromXml(processingHistoryXml); - // TODO maybe replace region information in metadata if overwritten in formatting request - org.esa.snap.binning.operator.formatter.Formatter formatter; - if (planetaryGrid instanceof IsinPlanetaryGrid) { + // SNAP's default formatter maps planetary grids to rectangular raster products. + // Select the Calvalus plugin by output format, not just by grid type, because + // existing SEA-grid requests may still require a standard raster product. + formatter = FormatterFactory.get(SeaGridFormatterPlugin.NAME); + } else if (planetaryGrid instanceof IsinPlanetaryGrid) { formatter = FormatterFactory.get("isin"); } else { formatter = FormatterFactory.get("default"); } + + Geometry regionGeometry = null; + MetadataElement[] metadataElements = new MetadataElement[0]; + if (!useSeaGridNetcdfFormatter) { + regionGeometry = GeometryUtils.createGeometry(regionWKT); + final String processingHistoryXml = configuration.get(JobConfigNames.PROCESSING_HISTORY); + final MetadataElement processingGraphMetadata = metadataSerializer.fromXml(processingHistoryXml); + // TODO maybe replace region information in metadata if overwritten in formatting request + metadataElements = new MetadataElement[]{processingGraphMetadata}; + } formatter.format(planetaryGrid, temporalBinSource, featureNames, @@ -129,7 +122,7 @@ private void format(TemporalBinSource temporalBinSource, regionGeometry, startTime, endTime, - processingGraphMetadata); + metadataElements); } private static ProductData.UTC parseTime(String timeString) { @@ -160,7 +153,6 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo String format = conf.get(JobConfigNames.CALVALUS_OUTPUT_FORMAT, null); String compression = conf.get(JobConfigNames.CALVALUS_OUTPUT_COMPRESSION, null); BinningConfig binningConfig = HadoopBinManager.getBinningConfig(conf); - boolean useSeaGridNetcdfFormatter = usesSeaGridNetcdfFormatter(format); ProductFormatter productFormatter; if ("org.esa.snap.binning.support.IsinPlanetaryGrid".equals(binningConfig.getPlanetaryGrid())){ productFormatter = new ProductFormatter(productName, "dir",null); @@ -176,7 +168,7 @@ public static void write(TaskInputOutputContext context, TemporalBinSource tempo conf); LOG.info("Start formatting product to file: " + productFile.getName()); context.setStatus("formatting"); - formatter.format(temporalBinSource, regionName, regionWKT, useSeaGridNetcdfFormatter); + formatter.format(temporalBinSource, regionName, regionWKT); LOG.info("Finished formatting product."); context.setStatus("copying"); diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java new file mode 100644 index 000000000..44e490669 --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2010 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.processing.l3; + +import com.bc.calvalus.commons.CalvalusLogger; +import com.bc.calvalus.processing.l2.ProductFormatter; +import org.esa.snap.binning.PlanetaryGrid; +import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.operator.formatter.Formatter; +import org.esa.snap.binning.operator.formatter.FormatterConfig; +import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.MetadataElement; +import org.esa.snap.core.datamodel.ProductData; +import org.locationtech.jts.geom.Geometry; + +import java.io.File; +import java.util.logging.Logger; + +/** + * Formats a SNAP {@link SEAGrid} as the flattened NetCDF layout required by + * {@link ProductFormatter#FORMAT_NETCDF4_SEAGRID}. + */ +final class SeaGridFormatter implements Formatter { + + private static final Logger LOG = CalvalusLogger.getLogger(); + + @Override + public void format(PlanetaryGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames, + FormatterConfig formatterConfig, + Geometry regionGeometry, + ProductData.UTC startTime, + ProductData.UTC endTime, + MetadataElement... metadataElements) throws Exception { + if (!(planetaryGrid instanceof SEAGrid)) { + throw new IllegalArgumentException( + ProductFormatter.FORMAT_NETCDF4_SEAGRID + + " requires org.esa.snap.binning.support.SEAGrid, but the request uses " + + planetaryGrid.getClass().getName()); + } + + LOG.info("Using flattened SEAGrid NetCDF formatter for output format " + + ProductFormatter.FORMAT_NETCDF4_SEAGRID + '.'); + SeaGridNetcdfFormatter.write(new File(formatterConfig.getOutputFile()), + (SEAGrid) planetaryGrid, + temporalBinSource, + featureNames, + startTime); + } +} diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatterPlugin.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatterPlugin.java new file mode 100644 index 000000000..8d5658d3f --- /dev/null +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatterPlugin.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2010 Brockmann Consult GmbH (info@brockmann-consult.de) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 3 of the License, or (at your option) + * any later version. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, see http://www.gnu.org/licenses/ + */ + +package com.bc.calvalus.processing.l3; + +import org.esa.snap.binning.operator.formatter.Formatter; +import org.esa.snap.binning.operator.formatter.FormatterPlugin; + +/** + * Makes the Calvalus flattened SEA-grid formatter available to SNAP's + * {@code FormatterFactory}. + */ +public final class SeaGridFormatterPlugin implements FormatterPlugin { + + static final String NAME = "seagrid"; + + @Override + public String getName() { + return NAME; + } + + @Override + public Formatter create() { + return new SeaGridFormatter(); + } +} diff --git a/calvalus-processing/src/main/resources/META-INF/services/org.esa.snap.binning.operator.formatter.FormatterPlugin b/calvalus-processing/src/main/resources/META-INF/services/org.esa.snap.binning.operator.formatter.FormatterPlugin new file mode 100644 index 000000000..f21d1931c --- /dev/null +++ b/calvalus-processing/src/main/resources/META-INF/services/org.esa.snap.binning.operator.formatter.FormatterPlugin @@ -0,0 +1 @@ +com.bc.calvalus.processing.l3.SeaGridFormatterPlugin diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java index cef789b84..0bf1de9ae 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l2/ProductFormatterTest.java @@ -22,7 +22,7 @@ public void preparesUncompressedNetcdfFileForSeaGridWriter() { ProductFormatter.FORMAT_NETCDF4_SEAGRID, "none"); - assertEquals("NetCDF4-BEAM", formatter.getOutputFormat()); + assertEquals(ProductFormatter.FORMAT_NETCDF4_SEAGRID, formatter.getOutputFormat()); assertEquals("example.nc", formatter.getProductFilename()); assertEquals("example.nc", formatter.getOutputFilename()); assertEquals("", formatter.getOutputCompression()); diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java index 6a649e31c..96997ef57 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java @@ -11,6 +11,7 @@ import org.esa.snap.binning.TemporalBin; import org.esa.snap.binning.TemporalBinSource; +import org.esa.snap.binning.operator.formatter.FormatterFactory; import org.esa.snap.binning.support.SEAGrid; import org.esa.snap.core.datamodel.ProductData; import org.junit.After; @@ -40,7 +41,7 @@ public class SeaGridNetcdfFormatterTest { @Before public void setUp() throws IOException { - outputFile = File.createTempFile("calvalus-isin-structure-", ".nc"); + outputFile = File.createTempFile("calvalus-seagrid-structure-", ".nc"); } @After @@ -126,6 +127,11 @@ public void dedicatedOutputFormatSelectsSeaGridFormatter() { assertTrue(!L3Formatter.usesSeaGridNetcdfFormatter(null)); } + @Test + public void seaGridFormatterIsAvailableThroughSnapFactory() { + assertTrue(FormatterFactory.get(SeaGridFormatterPlugin.NAME) instanceof SeaGridFormatter); + } + private static TemporalBin createBin(long index, float... values) { TemporalBin bin = new TemporalBin(index, values.length); bin.setNumObs(1); From 7ec54bb117c40524e63ddc296ff9280667f65d52 Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 31 Jul 2026 07:45:45 +0200 Subject: [PATCH 09/10] Added `num_obs` and `num_passes` variables in SeaGrid NetCDF formatter with tests --- .../processing/l3/SeaGridNetcdfFormatter.java | 44 +++++++++++++++++-- .../SeaGridNetcdfFormatterEdgeCasesTest.java | 2 + .../l3/SeaGridNetcdfFormatterTest.java | 21 +++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java index 2ccd22afa..623b3cf27 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java @@ -45,7 +45,7 @@ final class SeaGridNetcdfFormatter { private static final int BUFFER_SIZE = 8192; private static final long MILLIS_PER_DAY = 24L * 60L * 60L * 1000L; private static final Set RESERVED_VARIABLE_NAMES = new HashSet( - Arrays.asList("time", "bin_index", "lat", "lon", "crs")); + Arrays.asList("time", "bin_index", "lat", "lon", "crs", "num_obs", "num_passes")); private SeaGridNetcdfFormatter() { } @@ -109,10 +109,20 @@ static void write(File outputFile, longitudeVariable.addAttribute(new Attribute("units", "degrees_east")); longitudeVariable.addAttribute(new Attribute("axis", "X")); + final List dataDimensions = Arrays.asList(timeDimension, binIndexDimension); + final Variable numObsVariable = writer.addVariable("num_obs", DataType.INT, dataDimensions); + numObsVariable.addAttribute(new Attribute("_FillValue", -1)); + numObsVariable.addAttribute(new Attribute("coordinates", "lat lon")); + numObsVariable.addAttribute(new Attribute("grid_mapping", "crs")); + + final Variable numPassesVariable = writer.addVariable("num_passes", DataType.SHORT, dataDimensions); + numPassesVariable.addAttribute(new Attribute("_FillValue", (short) -1)); + numPassesVariable.addAttribute(new Attribute("coordinates", "lat lon")); + numPassesVariable.addAttribute(new Attribute("grid_mapping", "crs")); + final List featureVariables = new ArrayList(featureNames.length); for (String featureName : featureNames) { - Variable featureVariable = writer.addVariable(featureName, DataType.FLOAT, - Arrays.asList(timeDimension, binIndexDimension)); + Variable featureVariable = writer.addVariable(featureName, DataType.FLOAT, dataDimensions); featureVariable.addAttribute(new Attribute("_FillValue", Float.NaN)); featureVariable.addAttribute(new Attribute("coordinates", "lat lon")); featureVariable.addAttribute(new Attribute("grid_mapping", "crs")); @@ -128,7 +138,8 @@ static void write(File outputFile, writeCoordinates(writer, planetaryGrid, latitudeVariable, longitudeVariable); final FeatureBuffer featureBuffer = - new FeatureBuffer(writer, featureVariables, planetaryGrid); + new FeatureBuffer(writer, numObsVariable, numPassesVariable, + featureVariables, planetaryGrid); final int partCount = temporalBinSource.open(); sourceOpened = true; for (int partIndex = 0; partIndex < partCount; partIndex++) { @@ -247,9 +258,13 @@ private static void writeCoordinates(NetcdfFileWriter writer, private static final class FeatureBuffer { private final NetcdfFileWriter writer; + private final Variable numObsVariable; + private final Variable numPassesVariable; private final List variables; private final SEAGrid planetaryGrid; private final long numBins; + private final int[] numObsValues; + private final short[] numPassesValues; private final float[][] values; private long lastIndex = -1; @@ -258,9 +273,13 @@ private static final class FeatureBuffer { private int maxColumn; private FeatureBuffer(NetcdfFileWriter writer, + Variable numObsVariable, + Variable numPassesVariable, List variables, SEAGrid planetaryGrid) { this.writer = writer; + this.numObsVariable = numObsVariable; + this.numPassesVariable = numPassesVariable; this.variables = variables; this.planetaryGrid = planetaryGrid; this.numBins = planetaryGrid.getNumBins(); @@ -268,6 +287,8 @@ private FeatureBuffer(NetcdfFileWriter writer, for (int row = 0; row < planetaryGrid.getNumRows(); row++) { maxNumCols = Math.max(maxNumCols, planetaryGrid.getNumCols(row)); } + numObsValues = new int[maxNumCols]; + numPassesValues = new short[maxNumCols]; values = new float[variables.size()][maxNumCols]; } @@ -284,6 +305,11 @@ private void add(TemporalBin temporalBin) throws IOException, InvalidRangeExcept temporalBin.getFeatureValues().length + " features; expected " + variables.size() + '.'); } + int numPasses = temporalBin.getNumPasses(); + if (numPasses < 0 || numPasses > Short.MAX_VALUE) { + throw new IOException("Temporal bin " + binIndex + " has num_passes " + numPasses + + "; expected a value between 0 and " + Short.MAX_VALUE + '.'); + } int binRow = planetaryGrid.getRowIndex(binIndex); if (sourceRow != binRow) { flush(); @@ -291,6 +317,8 @@ private void add(TemporalBin temporalBin) throws IOException, InvalidRangeExcept } int column = (int) (binIndex - planetaryGrid.getFirstBinIndex(sourceRow)); + numObsValues[column] = temporalBin.getNumObs(); + numPassesValues[column] = (short) numPasses; float[] featureValues = temporalBin.getFeatureValues(); for (int featureIndex = 0; featureIndex < values.length; featureIndex++) { values[featureIndex][column] = featureValues[featureIndex]; @@ -303,6 +331,8 @@ private void add(TemporalBin temporalBin) throws IOException, InvalidRangeExcept private void reset(int newSourceRow) { sourceRow = newSourceRow; int numCols = planetaryGrid.getNumCols(sourceRow); + Arrays.fill(numObsValues, 0, numCols, -1); + Arrays.fill(numPassesValues, 0, numCols, (short) -1); for (float[] featureValues : values) { Arrays.fill(featureValues, 0, numCols, Float.NaN); } @@ -319,6 +349,12 @@ private void flush() throws IOException, InvalidRangeException { long outputRowStart = planetaryGrid.getFirstBinIndex(outputRow); int length = maxColumn - minColumn + 1; int[] origin = new int[]{0, (int) (outputRowStart + minColumn)}; + int[] numObsData = Arrays.copyOfRange(numObsValues, minColumn, maxColumn + 1); + writer.write(numObsVariable, origin, + Array.factory(DataType.INT, new int[]{1, length}, numObsData)); + short[] numPassesData = Arrays.copyOfRange(numPassesValues, minColumn, maxColumn + 1); + writer.write(numPassesVariable, origin, + Array.factory(DataType.SHORT, new int[]{1, length}, numPassesData)); for (int featureIndex = 0; featureIndex < variables.size(); featureIndex++) { float[] data = Arrays.copyOfRange(values[featureIndex], minColumn, maxColumn + 1); writer.write(variables.get(featureIndex), origin, diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java index f8568821f..c0679a2ef 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java @@ -94,6 +94,8 @@ public void rejectsOutOfRangeIndexAndWrongFeatureCount() throws Exception { @Test public void rejectsReservedDuplicateAndEmptyFeatureNamesBeforeOpeningSource() throws Exception { assertBadNames(new String[]{"time"}, "Reserved NetCDF variable name"); + assertBadNames(new String[]{"num_obs"}, "Reserved NetCDF variable name"); + assertBadNames(new String[]{"num_passes"}, "Reserved NetCDF variable name"); assertBadNames(new String[]{"value", "value"}, "Duplicate science-variable name"); assertBadNames(new String[]{" "}, "must not be empty"); } diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java index 96997ef57..13574252f 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java @@ -18,6 +18,7 @@ import org.junit.Before; import org.junit.Test; import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Attribute; import ucar.nc2.Dimension; import ucar.nc2.NetcdfFile; @@ -55,7 +56,11 @@ public void tearDown() { public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { SEAGrid grid = new SEAGrid(4); TemporalBin firstBin = createBin(0, 1.25f, 3.5f); + firstBin.setNumObs(17); + firstBin.setNumPasses(2); TemporalBin secondBin = createBin(10, 2.5f, 7.0f); + secondBin.setNumObs(29); + secondBin.setNumPasses(3); TemporalBinSource source = new SinglePartSource(firstBin, secondBin); SeaGridNetcdfFormatter.write(outputFile, @@ -76,6 +81,8 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { assertDimensions(netcdfFile.findVariable("chlor_a"), "time", "bin_index"); assertDimensions(netcdfFile.findVariable("total_nobs"), "time", "bin_index"); + assertDimensions(netcdfFile.findVariable("num_obs"), "time", "bin_index"); + assertDimensions(netcdfFile.findVariable("num_passes"), "time", "bin_index"); assertDimensions(netcdfFile.findVariable("lat"), "bin_index"); assertDimensions(netcdfFile.findVariable("lon"), "bin_index"); assertDimensions(netcdfFile.findVariable("time"), "time"); @@ -90,13 +97,27 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { Array chlorA = netcdfFile.findVariable("chlor_a").read(); Array totalNobs = netcdfFile.findVariable("total_nobs").read(); + Variable numObsVariable = netcdfFile.findVariable("num_obs"); + Variable numPassesVariable = netcdfFile.findVariable("num_passes"); + assertEquals(DataType.INT, numObsVariable.getDataType()); + assertEquals(DataType.SHORT, numPassesVariable.getDataType()); + assertEquals(-1, numObsVariable.findAttribute("_FillValue").getNumericValue().intValue()); + assertEquals(-1, numPassesVariable.findAttribute("_FillValue").getNumericValue().shortValue()); + Array numObs = numObsVariable.read(); + Array numPasses = numPassesVariable.read(); int firstOutputIndex = outputIndex(grid, 0); int secondOutputIndex = outputIndex(grid, 10); assertEquals(1.25f, chlorA.getFloat(firstOutputIndex), 0.0f); assertEquals(3.5f, totalNobs.getFloat(firstOutputIndex), 0.0f); + assertEquals(17, numObs.getInt(firstOutputIndex)); + assertEquals(2, numPasses.getShort(firstOutputIndex)); assertTrue(Float.isNaN(chlorA.getFloat(firstOutputIndex + 1))); + assertEquals(-1, numObs.getInt(firstOutputIndex + 1)); + assertEquals(-1, numPasses.getShort(firstOutputIndex + 1)); assertEquals(2.5f, chlorA.getFloat(secondOutputIndex), 0.0f); assertEquals(7.0f, totalNobs.getFloat(secondOutputIndex), 0.0f); + assertEquals(29, numObs.getInt(secondOutputIndex)); + assertEquals(3, numPasses.getShort(secondOutputIndex)); Array latitudes = netcdfFile.findVariable("lat").read(); Array longitudes = netcdfFile.findVariable("lon").read(); From b1c25c093a9423d15c271ccdea207634154de92c Mon Sep 17 00:00:00 2001 From: Marco Date: Fri, 31 Jul 2026 16:08:01 +0200 Subject: [PATCH 10/10] Add metadata handling in SeaGrid NetCDF formatter with updates to tests --- .../calvalus/processing/l3/L3Formatter.java | 9 +- .../processing/l3/SeaGridFormatter.java | 4 +- .../processing/l3/SeaGridNetcdfFormatter.java | 126 +++++++++++++++++- .../processing/l3/SeaGridNetcdfValidator.java | 3 +- .../l3/SeaGridNetcdf4SmokeTest.java | 33 ++++- .../SeaGridNetcdfFormatterEdgeCasesTest.java | 1 + .../l3/SeaGridNetcdfFormatterTest.java | 31 ++++- 7 files changed, 190 insertions(+), 17 deletions(-) diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java index 33caf39c0..29d577b03 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/L3Formatter.java @@ -107,14 +107,13 @@ private void format(TemporalBinSource temporalBinSource, } Geometry regionGeometry = null; - MetadataElement[] metadataElements = new MetadataElement[0]; if (!useSeaGridNetcdfFormatter) { regionGeometry = GeometryUtils.createGeometry(regionWKT); - final String processingHistoryXml = configuration.get(JobConfigNames.PROCESSING_HISTORY); - final MetadataElement processingGraphMetadata = metadataSerializer.fromXml(processingHistoryXml); - // TODO maybe replace region information in metadata if overwritten in formatting request - metadataElements = new MetadataElement[]{processingGraphMetadata}; } + final String processingHistoryXml = configuration.get(JobConfigNames.PROCESSING_HISTORY); + final MetadataElement processingGraphMetadata = metadataSerializer.fromXml(processingHistoryXml); + // TODO maybe replace region information in metadata if overwritten in formatting request + MetadataElement[] metadataElements = new MetadataElement[]{processingGraphMetadata}; formatter.format(planetaryGrid, temporalBinSource, featureNames, diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java index 44e490669..e0c9aee9a 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridFormatter.java @@ -60,6 +60,8 @@ public void format(PlanetaryGrid planetaryGrid, (SEAGrid) planetaryGrid, temporalBinSource, featureNames, - startTime); + startTime, + endTime, + metadataElements); } } diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java index 623b3cf27..30813e872 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatter.java @@ -12,7 +12,22 @@ import org.esa.snap.binning.TemporalBin; import org.esa.snap.binning.TemporalBinSource; import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.MetadataElement; +import org.esa.snap.core.datamodel.Product; import org.esa.snap.core.datamodel.ProductData; +import org.esa.snap.dataio.netcdf.AbstractNetCdfWriterPlugIn; +import org.esa.snap.dataio.netcdf.DefaultNetCdfWriter; +import org.esa.snap.dataio.netcdf.ProfileWriteContext; +import org.esa.snap.dataio.netcdf.ProfileWriteContextImpl; +import org.esa.snap.dataio.netcdf.metadata.profiles.beam.BeamMetadataPart; +import org.esa.snap.dataio.netcdf.metadata.profiles.beam.BeamNetCdf4WriterPlugIn; +import org.esa.snap.dataio.netcdf.metadata.profiles.beam.BeamNetCdfWriterPlugIn; +import org.esa.snap.dataio.netcdf.metadata.profiles.cf.CfTimePart; +import org.esa.snap.dataio.netcdf.nc.N3Variable; +import org.esa.snap.dataio.netcdf.nc.N4Variable; +import org.esa.snap.dataio.netcdf.nc.NFileWriteable; +import org.esa.snap.dataio.netcdf.nc.NVariable; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; @@ -37,15 +52,17 @@ * variables use {@code (time, bin_index)}, while latitude and longitude use * {@code (bin_index)}. The bin-row order is south to north, matching the * reference; longitude order within each row is preserved. - * Product-specific metadata is intentionally left to a later post-processing - * step. + * The Calvalus processing graph and core BEAM product attributes are retained; + * remaining product-specific metadata is left to a later post-processing step. */ final class SeaGridNetcdfFormatter { private static final int BUFFER_SIZE = 8192; private static final long MILLIS_PER_DAY = 24L * 60L * 60L * 1000L; + private static final String PRODUCT_TYPE = "BINNED-L3"; private static final Set RESERVED_VARIABLE_NAMES = new HashSet( - Arrays.asList("time", "bin_index", "lat", "lon", "crs", "num_obs", "num_passes")); + Arrays.asList("metadata", "time", "bin_index", "lat", "lon", "crs", + "num_obs", "num_passes")); private SeaGridNetcdfFormatter() { } @@ -55,10 +72,22 @@ static void write(File outputFile, TemporalBinSource temporalBinSource, String[] featureNames, ProductData.UTC startTime) throws IOException { - write(outputFile, planetaryGrid, temporalBinSource, featureNames, startTime, + write(outputFile, planetaryGrid, temporalBinSource, featureNames, startTime, startTime, + new MetadataElement[0], NetcdfFileWriter.Version.netcdf4_classic); } + static void write(File outputFile, + SEAGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames, + ProductData.UTC startTime, + ProductData.UTC endTime, + MetadataElement... metadataElements) throws IOException { + write(outputFile, planetaryGrid, temporalBinSource, featureNames, startTime, endTime, + metadataElements, NetcdfFileWriter.Version.netcdf4_classic); + } + /** * Package-private format selection keeps the structural test independent * of the native NetCDF-4 library. Production always uses NetCDF-4 classic. @@ -69,6 +98,18 @@ static void write(File outputFile, String[] featureNames, ProductData.UTC startTime, NetcdfFileWriter.Version version) throws IOException { + write(outputFile, planetaryGrid, temporalBinSource, featureNames, startTime, startTime, + new MetadataElement[0], version); + } + + static void write(File outputFile, + SEAGrid planetaryGrid, + TemporalBinSource temporalBinSource, + String[] featureNames, + ProductData.UTC startTime, + ProductData.UTC endTime, + MetadataElement[] metadataElements, + NetcdfFileWriter.Version version) throws IOException { validateArguments(outputFile, planetaryGrid, temporalBinSource, featureNames); validateMirroredRows(planetaryGrid); @@ -82,6 +123,10 @@ static void write(File outputFile, writer.setFill(true); writer.setLargeFile(true); + writer.addGlobalAttribute("Conventions", "CF-1.7"); + writer.addGlobalAttribute("product_type", PRODUCT_TYPE); + addMetadataAndTimeAttributes(writer, startTime, endTime, metadataElements, version); + final Dimension timeDimension = writer.addDimension("time", 1); final Dimension binIndexDimension = writer.addDimension("bin_index", numBins); @@ -129,8 +174,6 @@ static void write(File outputFile, featureVariables.add(featureVariable); } - writer.addGlobalAttribute("Conventions", "CF-1.7"); - boolean sourceOpened = false; try { writer.create(); @@ -176,6 +219,34 @@ static void write(File outputFile, } } + private static void addMetadataAndTimeAttributes(NetcdfFileWriter writer, + ProductData.UTC startTime, + ProductData.UTC endTime, + MetadataElement[] metadataElements, + NetcdfFileWriter.Version version) throws IOException { + final boolean netcdf4 = version == NetcdfFileWriter.Version.netcdf4 || + version == NetcdfFileWriter.Version.netcdf4_classic; + final ExistingFileWriteable writeable = new ExistingFileWriteable(writer, netcdf4); + final ProfileWriteContext context = new ProfileWriteContextImpl(writeable); + + final Product metadataProduct = new Product("metadata", PRODUCT_TYPE, 1, 1); + final AbstractNetCdfWriterPlugIn writerPlugIn = + netcdf4 ? new BeamNetCdf4WriterPlugIn() : new BeamNetCdfWriterPlugIn(); + metadataProduct.setProductWriter(new DefaultNetCdfWriter(writerPlugIn)); + metadataProduct.setStartTime(startTime); + metadataProduct.setEndTime(endTime); + if (metadataElements != null) { + for (MetadataElement metadataElement : metadataElements) { + if (metadataElement != null) { + metadataProduct.getMetadataRoot().addElement(metadataElement); + } + } + } + + new BeamMetadataPart().preEncode(context, metadataProduct); + new CfTimePart().preEncode(context, metadataProduct); + } + private static void validateArguments(File outputFile, SEAGrid planetaryGrid, TemporalBinSource temporalBinSource, @@ -363,4 +434,47 @@ private void flush() throws IOException, InvalidRangeException { sourceRow = -1; } } + + /** + * Adapts SNAP's metadata profile writer to the already-open definition + * phase of this formatter's NetCDF writer. + */ + private static final class ExistingFileWriteable extends NFileWriteable { + + private final boolean netcdf4; + + private ExistingFileWriteable(NetcdfFileWriter writer, boolean netcdf4) { + this.netcdfFileWriter = writer; + this.netcdf4 = netcdf4; + } + + @Override + public NVariable addScalarVariable(String name, DataType dataType) { + Variable variable = netcdfFileWriter.addVariable( + null, name, dataType, new ArrayList()); + NVariable nVariable = netcdf4 + ? new N4Variable(variable, null, netcdfFileWriter) + : new N3Variable(variable, netcdfFileWriter); + variables.put(name, nVariable); + return nVariable; + } + + @Override + public NVariable addVariable(String name, + DataType dataType, + boolean unsigned, + java.awt.Dimension tileSize, + String dimensions, + int compressionLevel) { + throw new UnsupportedOperationException( + "The metadata adapter only supports scalar variables."); + } + + @Override + public DataType getNetcdfDataType(int dataType) { + return netcdf4 + ? DataTypeUtils.getNetcdf4DataType(dataType) + : DataTypeUtils.getNetcdfDataType(dataType); + } + } } diff --git a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java index d78cdadf9..a3d8b10f9 100644 --- a/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java +++ b/calvalus-processing/src/main/java/com/bc/calvalus/processing/l3/SeaGridNetcdfValidator.java @@ -31,7 +31,7 @@ public final class SeaGridNetcdfValidator { private static final Set STRUCTURAL_VARIABLE_NAMES = new HashSet( - Arrays.asList("time", "bin_index", "lat", "lon", "crs")); + Arrays.asList("metadata", "time", "bin_index", "lat", "lon", "crs")); private SeaGridNetcdfValidator() { } @@ -200,4 +200,3 @@ private static void expectNumericAttribute(Variable variable, } } } - diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java index 4bd523fb8..cec46884b 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdf4SmokeTest.java @@ -12,11 +12,15 @@ import org.esa.snap.binning.TemporalBin; import org.esa.snap.binning.TemporalBinSource; import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.MetadataAttribute; +import org.esa.snap.core.datamodel.MetadataElement; import org.esa.snap.core.datamodel.ProductData; import org.junit.After; import org.junit.Assume; import org.junit.Before; import org.junit.Test; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; import java.io.File; import java.io.IOException; @@ -25,6 +29,8 @@ import java.util.Date; import java.util.Iterator; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; /** @@ -53,14 +59,38 @@ public void writesAndReadsNetcdf4Classic() throws Exception { SEAGrid grid = new SEAGrid(4); TemporalBin bin = new TemporalBin(2, 1); bin.getFeatureValues()[0] = 42.0f; + ProductData.UTC startTime = ProductData.UTC.create(new Date(1659312000000L), 0); + ProductData.UTC endTime = ProductData.UTC.create(new Date(1661904000000L), 0); + MetadataElement processingGraph = new MetadataElement("Processing_Graph"); + MetadataElement node = new MetadataElement("node_0"); + node.addAttribute(new MetadataAttribute( + "operator", ProductData.createInstance("l3-agg"), true)); + processingGraph.addElement(node); SeaGridNetcdfFormatter.write(outputFile, grid, new SinglePartSource(bin), new String[]{"science_value"}, - ProductData.UTC.create(new Date(1659312000000L), 0)); + startTime, + endTime, + processingGraph); assertTrue(SeaGridNetcdfValidator.validate(outputFile, 4).isEmpty()); + NetcdfFile netcdfFile = NetcdfFile.open(outputFile.getAbsolutePath()); + try { + Variable metadata = netcdfFile.findVariable("metadata"); + assertNotNull(metadata); + assertEquals("l3-agg", + metadata.findAttribute("Processing_Graph:node_0:operator").getStringValue()); + assertEquals("BINNED-L3", + netcdfFile.findGlobalAttribute("product_type").getStringValue()); + assertEquals(startTime.format(), + netcdfFile.findGlobalAttribute("start_date").getStringValue()); + assertEquals(endTime.format(), + netcdfFile.findGlobalAttribute("stop_date").getStringValue()); + } finally { + netcdfFile.close(); + } } private static final class SinglePartSource implements TemporalBinSource { @@ -90,4 +120,3 @@ public void close() { } } } - diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java index c0679a2ef..90e79d156 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterEdgeCasesTest.java @@ -94,6 +94,7 @@ public void rejectsOutOfRangeIndexAndWrongFeatureCount() throws Exception { @Test public void rejectsReservedDuplicateAndEmptyFeatureNamesBeforeOpeningSource() throws Exception { assertBadNames(new String[]{"time"}, "Reserved NetCDF variable name"); + assertBadNames(new String[]{"metadata"}, "Reserved NetCDF variable name"); assertBadNames(new String[]{"num_obs"}, "Reserved NetCDF variable name"); assertBadNames(new String[]{"num_passes"}, "Reserved NetCDF variable name"); assertBadNames(new String[]{"value", "value"}, "Duplicate science-variable name"); diff --git a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java index 13574252f..7e5923f96 100644 --- a/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java +++ b/calvalus-processing/src/test/java/com/bc/calvalus/processing/l3/SeaGridNetcdfFormatterTest.java @@ -13,6 +13,8 @@ import org.esa.snap.binning.TemporalBinSource; import org.esa.snap.binning.operator.formatter.FormatterFactory; import org.esa.snap.binning.support.SEAGrid; +import org.esa.snap.core.datamodel.MetadataAttribute; +import org.esa.snap.core.datamodel.MetadataElement; import org.esa.snap.core.datamodel.ProductData; import org.junit.After; import org.junit.Before; @@ -62,12 +64,21 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { secondBin.setNumObs(29); secondBin.setNumPasses(3); TemporalBinSource source = new SinglePartSource(firstBin, secondBin); + ProductData.UTC startTime = ProductData.UTC.create(new Date(1659312000000L), 0); + ProductData.UTC endTime = ProductData.UTC.create(new Date(1661904000000L), 0); + MetadataElement processingGraph = new MetadataElement("Processing_Graph"); + MetadataElement node = new MetadataElement("node_0"); + node.addAttribute(new MetadataAttribute( + "operator", ProductData.createInstance("l3-agg"), true)); + processingGraph.addElement(node); SeaGridNetcdfFormatter.write(outputFile, grid, source, new String[]{"chlor_a", "total_nobs"}, - ProductData.UTC.create(new Date(1659312000000L), 0), + startTime, + endTime, + new MetadataElement[]{processingGraph}, NetcdfFileWriter.Version.netcdf3); NetcdfFile netcdfFile = NetcdfFile.open(outputFile.getAbsolutePath()); @@ -88,6 +99,13 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { assertDimensions(netcdfFile.findVariable("time"), "time"); assertDimensions(netcdfFile.findVariable("crs"), "time"); + Variable metadata = netcdfFile.findVariable("metadata"); + assertNotNull(metadata); + assertEquals(DataType.BYTE, metadata.getDataType()); + assertEquals(0, metadata.getDimensions().size()); + assertEquals("l3-agg", + metadata.findAttribute("Processing_Graph:node_0:operator").getStringValue()); + Variable crs = netcdfFile.findVariable("crs"); assertEquals("1D binned sinusoidal", crs.findAttribute("grid_mapping_name").getStringValue()); @@ -129,6 +147,9 @@ public void writesReferenceDimensionModelFromSyntheticBins() throws Exception { Attribute conventions = netcdfFile.findGlobalAttribute("Conventions"); assertNotNull(conventions); assertEquals("CF-1.7", conventions.getStringValue()); + assertGlobalStringAttribute(netcdfFile, "product_type", "BINNED-L3"); + assertGlobalStringAttribute(netcdfFile, "start_date", startTime.format()); + assertGlobalStringAttribute(netcdfFile, "stop_date", endTime.format()); } finally { netcdfFile.close(); } @@ -176,6 +197,14 @@ private static void assertDimensions(Variable variable, String... names) { } } + private static void assertGlobalStringAttribute(NetcdfFile netcdfFile, + String name, + String expectedValue) { + Attribute attribute = netcdfFile.findGlobalAttribute(name); + assertNotNull(attribute); + assertEquals(expectedValue, attribute.getStringValue()); + } + private static final class SinglePartSource implements TemporalBinSource { private final Iterable bins;