Skip to content

Commit c49daa1

Browse files
adoroszlaiadoroszlai
committed
Merge remote-tracking branch 'origin/master' into dependabot-maven-com.fasterxml.jackson-jackson-bom-2.21.0
2 parents 2196781 + 593d7ea commit c49daa1

69 files changed

Lines changed: 1750 additions & 673 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,18 @@ public Container updateContainer(Container<?> container) throws
254254
} else {
255255
LOG.debug("Container with container Id {} is updated to containerMap",
256256
containerId);
257-
return containerMap.put(containerId, container);
257+
Container<?> oldContainer = containerMap.put(containerId, container);
258+
HddsVolume volume = container.getContainerData().getVolume();
259+
if (volume != null) {
260+
volume.addContainer(container.getContainerData().getContainerID());
261+
}
262+
if (oldContainer != null) {
263+
volume = oldContainer.getContainerData().getVolume();
264+
if (volume != null) {
265+
volume.removeContainer(oldContainer.getContainerData().getContainerID());
266+
}
267+
}
268+
return oldContainer;
258269
}
259270
}
260271

@@ -446,7 +457,7 @@ public Iterator<Container<?>> getContainerIterator(HddsVolume volume) {
446457
while (containerIdIterator.hasNext()) {
447458
Long containerId = containerIdIterator.next();
448459
Container<?> container = containerMap.get(containerId);
449-
if (container != null) {
460+
if (container != null && container.getContainerData().getVolume() == volume) {
450461
containers.add(container);
451462
}
452463
}

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ public final class StorageLocationReport implements StorageLocationReportMXBean
4343
private final StorageType storageType;
4444
private final String storageLocation;
4545
private final long reserved;
46+
private final long fsCapacity;
47+
private final long fsAvailable;
4648

4749
private StorageLocationReport(Builder builder) {
4850
this.id = builder.id;
@@ -55,6 +57,8 @@ private StorageLocationReport(Builder builder) {
5557
this.storageType = builder.storageType;
5658
this.storageLocation = builder.storageLocation;
5759
this.reserved = builder.reserved;
60+
this.fsCapacity = builder.fsCapacity;
61+
this.fsAvailable = builder.fsAvailable;
5862
}
5963

6064
public long getUsableSpace() {
@@ -143,6 +147,14 @@ public long getReserved() {
143147
return reserved;
144148
}
145149

150+
public long getFsCapacity() {
151+
return fsCapacity;
152+
}
153+
154+
public long getFsAvailable() {
155+
return fsAvailable;
156+
}
157+
146158
private static StorageType getStorageType(StorageTypeProto proto) throws
147159
IOException {
148160
StorageType storageType;
@@ -186,6 +198,8 @@ public StorageReportProto getProtoBufMessage() throws IOException {
186198
.setFailed(isFailed())
187199
.setFreeSpaceToSpare(getFreeSpaceToSpare())
188200
.setReserved(getReserved())
201+
.setFsCapacity(getFsCapacity())
202+
.setFsAvailable(getFsAvailable())
189203
.build();
190204
}
191205

@@ -240,6 +254,12 @@ public static StorageLocationReport getFromProtobuf(StorageReportProto report)
240254
if (report.hasReserved()) {
241255
builder.setReserved(report.getReserved());
242256
}
257+
if (report.hasFsCapacity()) {
258+
builder.setFsCapacity(report.getFsCapacity());
259+
}
260+
if (report.hasFsAvailable()) {
261+
builder.setFsAvailable(report.getFsAvailable());
262+
}
243263
return builder.build();
244264
}
245265

@@ -254,11 +274,16 @@ public String toString() {
254274
if (failed) {
255275
sb.append(" failed");
256276
} else {
257-
sb.append(" capacity=").append(capacity)
258-
.append(" used=").append(scmUsed)
259-
.append(" available=").append(remaining)
277+
long fsUsed = fsCapacity - fsAvailable;
278+
sb.append(" ozoneCapacity=").append(capacity)
279+
.append(" ozoneUsed=").append(scmUsed)
280+
.append(" ozoneAvailable=").append(remaining)
260281
.append(" minFree=").append(freeSpaceToSpare)
261-
.append(" committed=").append(committed);
282+
.append(" committed=").append(committed)
283+
.append(" reserved=").append(reserved)
284+
.append(" fsCapacity=").append(fsCapacity)
285+
.append(" fsAvailable=").append(fsAvailable)
286+
.append(" fsUsed=").append(fsUsed);
262287
}
263288

264289
return sb.append(" }").toString();
@@ -287,6 +312,8 @@ public static class Builder {
287312
private StorageType storageType;
288313
private String storageLocation;
289314
private long reserved;
315+
private long fsCapacity;
316+
private long fsAvailable;
290317

291318
/**
292319
* Sets the storageId.
@@ -405,6 +432,16 @@ public long getReserved() {
405432
return reserved;
406433
}
407434

435+
public Builder setFsCapacity(long fsCapacity) {
436+
this.fsCapacity = fsCapacity;
437+
return this;
438+
}
439+
440+
public Builder setFsAvailable(long fsAvailable) {
441+
this.fsAvailable = fsAvailable;
442+
return this;
443+
}
444+
408445
/**
409446
* Builds and returns StorageLocationReport instance.
410447
*

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerLogger.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ private static String getMessage(long containerId, StorageVolume sourceVolume,
214214
"SrcVolume=" + sourceVolume,
215215
"DestVolume=" + destinationVolume,
216216
"Size=" + containerSize + " bytes",
217-
"TimeTaken=" + timeTaken + " ms");
217+
"TimeTaken=" + timeTaken + " ms",
218+
"Container is moved from SrcVolume to DestVolume");
218219
}
219220
}

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/StorageVolume.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -480,11 +480,14 @@ protected StorageLocationReport.Builder reportBuilder() {
480480
.setStorageType(storageType);
481481

482482
if (!builder.isFailed()) {
483-
SpaceUsageSource usage = volumeUsage.getCurrentUsage();
483+
SpaceUsageSource.Fixed fsUsage = volumeUsage.realUsage();
484+
SpaceUsageSource usage = volumeUsage.getCurrentUsage(fsUsage);
484485
builder.setCapacity(usage.getCapacity())
485486
.setRemaining(usage.getAvailable())
486487
.setScmUsed(usage.getUsedSpace())
487-
.setReserved(volumeUsage.getReservedInBytes());
488+
.setReserved(volumeUsage.getReservedInBytes())
489+
.setFsCapacity(fsUsage.getCapacity())
490+
.setFsAvailable(fsUsage.getAvailable());
488491
}
489492

490493
return builder;

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeInfoMetrics.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,15 +44,21 @@ public class VolumeInfoMetrics implements MetricsSource {
4444
VolumeInfoMetrics.class.getSimpleName();
4545

4646
private static final MetricsInfo CAPACITY =
47-
Interns.info("Capacity", "Capacity");
47+
Interns.info("OzoneCapacity", "Ozone usable capacity (after reserved space adjustment)");
4848
private static final MetricsInfo AVAILABLE =
49-
Interns.info("Available", "Available Space");
49+
Interns.info("OzoneAvailable", "Ozone available space (after reserved space adjustment)");
5050
private static final MetricsInfo USED =
51-
Interns.info("Used", "Used Space");
51+
Interns.info("OzoneUsed", "Ozone used space");
5252
private static final MetricsInfo RESERVED =
5353
Interns.info("Reserved", "Reserved Space");
5454
private static final MetricsInfo TOTAL_CAPACITY =
55-
Interns.info("TotalCapacity", "Total Capacity");
55+
Interns.info("TotalCapacity", "Ozone capacity + reserved space");
56+
private static final MetricsInfo FS_CAPACITY =
57+
Interns.info("FilesystemCapacity", "Filesystem capacity as reported by the local filesystem");
58+
private static final MetricsInfo FS_AVAILABLE =
59+
Interns.info("FilesystemAvailable", "Filesystem available space as reported by the local filesystem");
60+
private static final MetricsInfo FS_USED =
61+
Interns.info("FilesystemUsed", "Filesystem used space (FilesystemCapacity - FilesystemAvailable)");
5662

5763
private final MetricsRegistry registry;
5864
private final String metricsSourceName;
@@ -185,14 +191,18 @@ public void getMetrics(MetricsCollector collector, boolean all) {
185191
registry.snapshot(builder, all);
186192
VolumeUsage volumeUsage = volume.getVolumeUsage();
187193
if (volumeUsage != null) {
188-
SpaceUsageSource usage = volumeUsage.getCurrentUsage();
194+
SpaceUsageSource.Fixed fsUsage = volumeUsage.realUsage();
195+
SpaceUsageSource usage = volumeUsage.getCurrentUsage(fsUsage);
189196
long reserved = volumeUsage.getReservedInBytes();
190197
builder
191198
.addGauge(CAPACITY, usage.getCapacity())
192199
.addGauge(AVAILABLE, usage.getAvailable())
193200
.addGauge(USED, usage.getUsedSpace())
194201
.addGauge(RESERVED, reserved)
195-
.addGauge(TOTAL_CAPACITY, usage.getCapacity() + reserved);
202+
.addGauge(TOTAL_CAPACITY, usage.getCapacity() + reserved)
203+
.addGauge(FS_CAPACITY, fsUsage.getCapacity())
204+
.addGauge(FS_AVAILABLE, fsUsage.getAvailable())
205+
.addGauge(FS_USED, fsUsage.getUsedSpace());
196206
}
197207
}
198208
}

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,11 @@ SpaceUsageSource.Fixed realUsage() {
124124
* B) avail = fsAvail - Max(reserved - other, 0);
125125
*/
126126
public SpaceUsageSource.Fixed getCurrentUsage() {
127-
final SpaceUsageSource.Fixed real = realUsage();
127+
return getCurrentUsage(realUsage());
128+
}
129+
130+
// use this variant if real usage values are also needed at the caller
131+
public SpaceUsageSource.Fixed getCurrentUsage(SpaceUsageSource.Fixed real) {
128132
return reservedInBytes == 0
129133
? real
130134
: new SpaceUsageSource.Fixed(

‎hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn-overview.html‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,14 @@ <h2>Volume Information</h2>
5353
<th>Directory</th>
5454
<th>Storage Type</th>
5555
<th>Volume Type</th>
56-
<th>Used Space</th>
57-
<th>Available Space</th>
56+
<th>Ozone Capacity</th>
57+
<th>Ozone Used</th>
58+
<th>Ozone Available</th>
5859
<th>Reserved</th>
59-
<th>Total Capacity</th>
60+
<th>Total Capacity (Ozone Capacity + Reserved)</th>
61+
<th>Filesystem Capacity</th>
62+
<th>Filesystem Available</th>
63+
<th>Filesystem Used</th>
6064
<th>Containers</th>
6165
<th>State</th>
6266
</tr>
@@ -66,10 +70,14 @@ <h2>Volume Information</h2>
6670
<td>{{volumeInfo["tag.StorageDirectory"]}}</td>
6771
<td>{{volumeInfo["tag.StorageType"]}}</td>
6872
<td>{{volumeInfo["tag.VolumeType"]}}</td>
69-
<td>{{volumeInfo.Used}}</td>
70-
<td>{{volumeInfo.Available}}</td>
73+
<td>{{volumeInfo.OzoneCapacity}}</td>
74+
<td>{{volumeInfo.OzoneUsed}}</td>
75+
<td>{{volumeInfo.OzoneAvailable}}</td>
7176
<td>{{volumeInfo.Reserved}}</td>
7277
<td>{{volumeInfo.TotalCapacity}}</td>
78+
<td>{{volumeInfo.FilesystemCapacity}}</td>
79+
<td>{{volumeInfo.FilesystemAvailable}}</td>
80+
<td>{{volumeInfo.FilesystemUsed}}</td>
7381
<td>{{volumeInfo.Containers}}</td>
7482
<td>{{volumeInfo["tag.VolumeState"]}}</td>
7583
</tr>

‎hadoop-hdds/container-service/src/main/resources/webapps/hddsDatanode/dn.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,14 @@
3030
.then(function (result) {
3131
ctrl.dnmetrics = result.data.beans;
3232
ctrl.dnmetrics.forEach(volume => {
33-
volume.Used = transform(volume.Used);
34-
volume.Available = transform(volume.Available);
33+
volume.OzoneCapacity = transform(volume.OzoneCapacity);
34+
volume.OzoneUsed = transform(volume.OzoneUsed);
35+
volume.OzoneAvailable = transform(volume.OzoneAvailable);
3536
volume.Reserved = transform(volume.Reserved);
3637
volume.TotalCapacity = transform(volume.TotalCapacity);
38+
volume.FilesystemCapacity = transform(volume.FilesystemCapacity);
39+
volume.FilesystemAvailable = transform(volume.FilesystemAvailable);
40+
volume.FilesystemUsed = transform(volume.FilesystemUsed);
3741
})
3842
});
3943

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestStorageLocationReport.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.container.common.impl;
19+
20+
import static org.assertj.core.api.Assertions.assertThat;
21+
22+
import java.io.IOException;
23+
import org.apache.hadoop.fs.StorageType;
24+
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageReportProto;
25+
import org.junit.jupiter.api.Test;
26+
27+
class TestStorageLocationReport {
28+
29+
@Test
30+
void testStorageReportProtoIncludesFilesystemFieldsAndRoundTrips() throws IOException {
31+
StorageLocationReport report = StorageLocationReport.newBuilder()
32+
.setId("vol-1")
33+
.setStorageLocation("/data/hdds/vol-1")
34+
.setStorageType(StorageType.DISK)
35+
.setCapacity(1000L)
36+
.setScmUsed(100L)
37+
.setRemaining(900L)
38+
.setCommitted(10L)
39+
.setFreeSpaceToSpare(5L)
40+
.setReserved(50L)
41+
.setFsCapacity(2000L)
42+
.setFsAvailable(1500L)
43+
.build();
44+
45+
StorageReportProto proto = report.getProtoBufMessage();
46+
assertThat(proto.hasFsCapacity()).isTrue();
47+
assertThat(proto.hasFsAvailable()).isTrue();
48+
assertThat(proto.getFsCapacity()).isEqualTo(2000L);
49+
assertThat(proto.getFsAvailable()).isEqualTo(1500L);
50+
51+
StorageLocationReport parsed = StorageLocationReport.getFromProtobuf(proto);
52+
assertThat(parsed.getCapacity()).isEqualTo(1000L);
53+
assertThat(parsed.getScmUsed()).isEqualTo(100L);
54+
assertThat(parsed.getRemaining()).isEqualTo(900L);
55+
assertThat(parsed.getReserved()).isEqualTo(50L);
56+
assertThat(parsed.getFsCapacity()).isEqualTo(2000L);
57+
assertThat(parsed.getFsAvailable()).isEqualTo(1500L);
58+
}
59+
}
60+

0 commit comments

Comments
 (0)