From 011f775e3f4fd82f863a5d14d63d9029b8b59cfe Mon Sep 17 00:00:00 2001
From: Lee Rhodes <leerho@gmail.com>
Date: Thu, 26 Mar 2026 18:37:25 -0700
Subject: [PATCH 1/2] Fixes the getRSE() function and updates the
 SketchesCheckstyle.xml.

---
 .../java/org/apache/datasketches/req/BaseReqSketch.java    | 7 +++----
 tools/SketchesCheckstyle.xml                               | 3 +++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
index f9411d5f4..fd44a12df 100644
--- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
+++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java
@@ -71,7 +71,7 @@ abstract class BaseReqSketch implements QuantilesFloatsAPI {
    * @return an a priori estimate of relative standard error (RSE, expressed as a number in [0,1]).
    */
   public static double getRSE(final int k, final double rank, final boolean hra, final long totalN) {
-    return getRankUB(k, 2, rank, 1, hra, totalN); //more conservative to assume > 1 level
+    return getRankUB(k, 2, rank, 1, hra, totalN) - rank; //more conservative to assume > 1 level
   }
 
   @Override
@@ -188,9 +188,8 @@ public boolean isReadOnly() {
    */
   public abstract String viewCompactorDetail(String fmt, boolean allData);
 
-  static boolean exactRank(final int k, final int levels, final double rank,
-      final boolean hra, final long totalN) {
-    final int baseCap = k * INIT_NUMBER_OF_SECTIONS;
+  static boolean exactRank(final int k, final int levels, final double rank, final boolean hra, final long totalN) {
+    final long baseCap = (long)k * INIT_NUMBER_OF_SECTIONS;
     if ((levels == 1) || (totalN <= baseCap)) { return true; }
     final double exactRankThresh = (double)baseCap / totalN;
     return (hra ? (rank >= (1.0 - exactRankThresh)) : (rank <= exactRankThresh));
diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml
index f51987dc5..0ced9d3e3 100644
--- a/tools/SketchesCheckstyle.xml
+++ b/tools/SketchesCheckstyle.xml
@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+          "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN"
+          "https://checkstyle.org/dtds/configuration_1_3.dtd">
 
 <!--
 Licensed to the Apache Software Foundation (ASF) under one

From 3abbe20735cb00c64c3a31cd718ea45f8d8c88d7 Mon Sep 17 00:00:00 2001
From: Lee Rhodes <leerho@gmail.com>
Date: Fri, 27 Mar 2026 11:56:30 -0700
Subject: [PATCH 2/2] Fix getRSE tests

---
 .../org/apache/datasketches/req/ReqSketchOtherTest.java    | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java
index 48b6e582e..4a54c7c69 100644
--- a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java
+++ b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java
@@ -33,9 +33,6 @@
 import org.apache.datasketches.common.SketchesArgumentException;
 import org.apache.datasketches.quantilescommon.FloatsSortedView;
 import org.apache.datasketches.quantilescommon.InequalitySearch;
-import org.apache.datasketches.req.BaseReqSketch;
-import org.apache.datasketches.req.ReqSketch;
-import org.apache.datasketches.req.ReqSketchBuilder;
 import org.testng.annotations.Test;
 
 /**
@@ -114,7 +111,7 @@ public void checkEstimationMode() {
     assertEquals(v, 120.0f);
     final FloatsSortedView aux = sk.getSortedView();
     assertNotNull(aux);
-    assertTrue(BaseReqSketch.getRSE(sk.getK(), .5, false, 120) > 0);
+    assertTrue(BaseReqSketch.getRSE(sk.getK(), .5, false, 120) >= 0);
     assertTrue(sk.getSerializedSizeBytes() > 0);
   }
 
@@ -185,7 +182,7 @@ public void checkEmpty() {
     try { sk.getQuantiles(new double[] {0.5}); fail(); } catch (IllegalArgumentException e) {}
     try { sk.getPMF(new float[] {1f}); fail(); } catch (IllegalArgumentException e) {}
     try { sk.getCDF(new float[] {1f}); fail(); } catch (IllegalArgumentException e) {}
-    assertTrue(BaseReqSketch.getRSE(50, 0.5, true, 0) > 0);
+    assertTrue(BaseReqSketch.getRSE(50, 0.5, true, 0) >= 0);
     assertTrue(sk.getRankUpperBound(0.5, 1) > 0);
   }