diff --git a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java index f9411d5f4..fd44a12df 100644 --- a/src/main/java/org/apache/datasketches/req/BaseReqSketch.java +++ b/src/main/java/org/apache/datasketches/req/BaseReqSketch.java @@ -71,7 +71,7 @@ abstract class BaseReqSketch implements QuantilesFloatsAPI { * @return an a priori estimate of relative standard error (RSE, expressed as a number in [0,1]). */ public static double getRSE(final int k, final double rank, final boolean hra, final long totalN) { - return getRankUB(k, 2, rank, 1, hra, totalN); //more conservative to assume > 1 level + return getRankUB(k, 2, rank, 1, hra, totalN) - rank; //more conservative to assume > 1 level } @Override @@ -188,9 +188,8 @@ public boolean isReadOnly() { */ public abstract String viewCompactorDetail(String fmt, boolean allData); - static boolean exactRank(final int k, final int levels, final double rank, - final boolean hra, final long totalN) { - final int baseCap = k * INIT_NUMBER_OF_SECTIONS; + static boolean exactRank(final int k, final int levels, final double rank, final boolean hra, final long totalN) { + final long baseCap = (long)k * INIT_NUMBER_OF_SECTIONS; if ((levels == 1) || (totalN <= baseCap)) { return true; } final double exactRankThresh = (double)baseCap / totalN; return (hra ? (rank >= (1.0 - exactRankThresh)) : (rank <= exactRankThresh)); diff --git a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java index 48b6e582e..4a54c7c69 100644 --- a/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java +++ b/src/test/java/org/apache/datasketches/req/ReqSketchOtherTest.java @@ -33,9 +33,6 @@ import org.apache.datasketches.common.SketchesArgumentException; import org.apache.datasketches.quantilescommon.FloatsSortedView; import org.apache.datasketches.quantilescommon.InequalitySearch; -import org.apache.datasketches.req.BaseReqSketch; -import org.apache.datasketches.req.ReqSketch; -import org.apache.datasketches.req.ReqSketchBuilder; import org.testng.annotations.Test; /** @@ -114,7 +111,7 @@ public void checkEstimationMode() { assertEquals(v, 120.0f); final FloatsSortedView aux = sk.getSortedView(); assertNotNull(aux); - assertTrue(BaseReqSketch.getRSE(sk.getK(), .5, false, 120) > 0); + assertTrue(BaseReqSketch.getRSE(sk.getK(), .5, false, 120) >= 0); assertTrue(sk.getSerializedSizeBytes() > 0); } @@ -185,7 +182,7 @@ public void checkEmpty() { try { sk.getQuantiles(new double[] {0.5}); fail(); } catch (IllegalArgumentException e) {} try { sk.getPMF(new float[] {1f}); fail(); } catch (IllegalArgumentException e) {} try { sk.getCDF(new float[] {1f}); fail(); } catch (IllegalArgumentException e) {} - assertTrue(BaseReqSketch.getRSE(50, 0.5, true, 0) > 0); + assertTrue(BaseReqSketch.getRSE(50, 0.5, true, 0) >= 0); assertTrue(sk.getRankUpperBound(0.5, 1) > 0); } diff --git a/tools/SketchesCheckstyle.xml b/tools/SketchesCheckstyle.xml index f51987dc5..0ced9d3e3 100644 --- a/tools/SketchesCheckstyle.xml +++ b/tools/SketchesCheckstyle.xml @@ -1,4 +1,7 @@ +