Fix newly introduced bug when opening Zarr arrays (#51)

* Fix newly introduced bug when opening Zarr arrays

* Fix tests for zarr data and add test on to_dataset for Zgroup

* Test rational writing only for zarr

---------

Co-authored-by: Felix Cremer <fcremer@bgc-jena.mpg.de>

Author: meggart

Project.toml

@@ -1,7 +1,7 @@
 name = "YAXArrayBase"
 uuid = "90b8fcef-0c2d-428d-9c56-5f86629e9d14"
 authors = ["Fabian Gans <fgans@bgc-jena.mpg.de>"]
-version = "0.7.9"
+version = "0.7.10"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -14,7 +14,7 @@ julia = "1.9"
 ArchGDAL = "0.10"
 AxisArrays = "0.4"
 AxisKeys = "0.2"
-DimensionalData = "0.27, 0.28, 0.29"
+DimensionalData = "0.27, 0.28, 0.29, 0.30"
 NetCDF = "0.11, 0.12"
 Zarr = "0.8, 0.9"
 

ext/ZarrExt.jl

@@ -15,7 +15,10 @@ end
 struct ZarrDataset
   g::ZGroup
 end
-function ZarrDataset(g::String; mode="r", path="", kwargs...)
+function ZarrDataset(g::Union{String,ZGroup}; mode="r", path="", kwargs...)
+  if g isa ZGroup
+    return ZarrDataset(g)
+  end
   store = if endswith(g, "zip")
     ZipStore(ZipReader(SimpleFileDiskArray(g)))
   else

test/datasets.jl

@@ -64,19 +64,19 @@ end
 end
 
 @testset "Reading Zarr" begin
-p = "gs://cmip6/CMIP6/HighResMIP/CMCC/CMCC-CM2-HR4/highresSST-present/r1i1p1f1/6hrPlev/psl/gn/v20170706/"
-ds_zarr = to_dataset(p,driver=:zarr)
-vn = get_varnames(ds_zarr)
-@test sort(vn) == ["lat", "lat_bnds", "lon", "lon_bnds", "psl", "time", "time_bnds"]
-@test get_var_dims(ds_zarr, "psl") == ["lon", "lat", "time"]
-@test get_var_dims(ds_zarr, "time") == ["time"]
-@test get_var_dims(ds_zarr, "time_bnds") == ["bnds", "time"]
-@test get_var_attrs(ds_zarr,"psl")["long_name"] == "Sea Level Pressure"
-h = get_var_handle(ds_zarr, "psl")
-@test YAXArrayBase.iscompressed(h)
-@test all(isapprox.(h[1:2,1:2,1], [99360.8  99334.9; 99360.8  99335.4]))
-@test allow_parallel_write(ds_zarr) == true
-@test allow_missings(ds_zarr) == false
+  p = "https://s3.bgc-jena.mpg.de:9000/esdl-esdc-v3.0.2/esdc-16d-2.5deg-46x72x1440-3.0.2.zarr"
+  for ds_zarr in [to_dataset(p,driver=:zarr), to_dataset(zopen(p))]
+    vn = get_varnames(ds_zarr)
+    @test sort(vn) == ["aerosol_optical_thickness_550", "air_temperature_2m", "bare_soil_evaporation", "burnt_area", "cot", "cth", "ctt", "evaporation", "evaporation_era5", "evaporative_stress", "gross_primary_productivity", "interception_loss", "kndvi", "lat", "latent_energy", "lon", "max_air_temperature_2m", "min_air_temperature_2m", "nbar_blue", "nbar_green", "nbar_nir", "nbar_red", "nbar_swir1", "nbar_swir2", "nbar_swir3", "ndvi", "net_ecosystem_exchange", "net_radiation", "nirv", "open_water_evaporation", "potential_evaporation", "precipitation_era5", "radiation_era5", "root_moisture", "sensible_heat", "sif_gome2_jj", "sif_gome2_pk", "sif_gosif", "sif_rtsif", "sm", "snow_sublimation", "surface_moisture", "terrestrial_ecosystem_respiration", "time", "transpiration"]
+    @test get_var_dims(ds_zarr, "sm") == ["lon", "lat", "time"]
+    @test get_var_dims(ds_zarr, "time") == ["time"]
+    @test get_var_attrs(ds_zarr,"sm")["long_name"] == "Volumetric Soil Moisture"
+    h = get_var_handle(ds_zarr, "air_temperature_2m")
+    @test YAXArrayBase.iscompressed(h)
+    @test all(isapprox.(h[1:2,1:2,1], [-28.8662  -23.4534; -28.8041  -23.1681]))
+    @test allow_parallel_write(ds_zarr) == true
+    @test allow_missings(ds_zarr) == false
+  end
 end
 @testset "Reading ArchGDAL" begin
   using ArchGDAL
@@ -99,19 +99,27 @@ function test_write(T)
   add_var(ds, 0.5:1:9.5, "lon", ("lon",), Dict("units"=>"degrees_east"))
   add_var(ds, 20:-1.0:1, "lat", ("lat",), Dict("units"=>"degrees_north"))
   v = add_var(ds, Float32, "tas", (10,20), ("lon", "lat"), Dict{String,Any}("units"=>"Celsius"))
-  w = add_var(ds, Rational{Int}, "tas2", (10,20), ("lon", "lat"), Dict{String,Any}("units"=>"Celsius")) 
+  # This is needed, since netcdf can not write rationals.
+  if T == YAXArrayBase.backendlist[:zarr]
+    w = add_var(ds, Rational{Int}, "tas2", (10,20), ("lon", "lat"), Dict{String,Any}("units"=>"Celsius")) 
+    w[:,:] = collect(reshape(1:200, 10, 20)) .// 2
+    w = get_var_handle(ds, "tas2")
+    @test w[1:2,1:2] == [1 11; 2 12] .// 2
+    w = get_var_handle(ds, "tas2")
+    @test w[1:2,1:2] == [1 11; 2 12] .// 2
+    @test sort(get_varnames(ds)) == ["lat","lon","tas", "tas2"]
+  else
+    @test sort(get_varnames(ds)) == ["lat","lon","tas"]
+  end
   v[:,:] = collect(reshape(1:200, 10, 20))
-  w[:,:] = collect(reshape(1:200, 10, 20)) .// 2
-  @test sort(get_varnames(ds)) == ["lat","lon","tas", "tas2"]
+
   @test get_var_dims(ds, "tas") == ["lon", "lat"]
   @test get_var_dims(ds, "lon") == ["lon"]
   @test get_var_attrs(ds,"tas")["units"] == "Celsius"
   h = get_var_handle(ds, "lon")
   @test h[:] == 0.5:1:9.5
   v = get_var_handle(ds, "tas")
   @test v[1:2,1:2] == [1 11; 2 12]
-  w = get_var_handle(ds, "tas2")
-  @test w[1:2,1:2] == [1 11; 2 12] .// 2
 end
 
 @testset "Writing NetCDF" begin