Skip to content

Commit b6049aa

Browse files
TwolveroTwolvero
committed
add test config
1 parent 9f4bb9f commit b6049aa

9 files changed

Lines changed: 273 additions & 25 deletions

File tree

‎src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,8 @@ private CNode getIntermediateInputVector() {
244244
}
245245

246246
private boolean getTemplateType(double sparsityEst, double scalarVal) {
247-
if(!DMLScript.SPARSE_INTERMEDIATE)
248-
return false;
247+
if(DMLScript.SPARSE_INTERMEDIATE)
248+
return true;
249249
else {
250250
switch(_type) {
251251
case VECT_MULT: return sparsityEst < 0.008;

‎src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,8 @@ public String codegen(boolean sparse, GeneratorAPI api) {
143143
}
144144

145145
public boolean getTemplateType(double sparsity) {
146-
if(!DMLScript.SPARSE_INTERMEDIATE)
147-
return false;
146+
if(DMLScript.SPARSE_INTERMEDIATE)
147+
return true;
148148
else {
149149
switch(_type) {
150150
case VECT_SQRT:

‎src/test/java/org/apache/sysds/performance/primitives/SparseRowPerfTest.java‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -232,22 +232,16 @@ public void logResults(double[] sparsityVals, String[] result, boolean sparse, U
232232
}
233233

234234
public static void main(String[] args) {
235-
new SparseRowPerfTest(100, 200000).testBinaryPrimitivePerf(BinType.VECT_MULT_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.HYBRID, SparsityType.SET1);
236-
new SparseRowPerfTest(100, 200000).testBinaryPrimitivePerf(BinType.VECT_MULT_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.HYBRID, SparsityType.SET1);
237-
new SparseRowPerfTest(100, 200000).testBinaryPrimitivePerf(BinType.VECT_DIV_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.HYBRID, SparsityType.SET1);
238-
new SparseRowPerfTest(100, 200000).testBinaryPrimitivePerf(BinType.VECT_DIV_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.HYBRID, SparsityType.SET1);
235+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_MIN_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
236+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_MIN_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
237+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_MIN, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
239238

240-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_MULT_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
241-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_MULT_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
242-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_MULT, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
239+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_PLUS_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
240+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_PLUS_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
241+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_PLUS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
243242

244-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_DIV_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
245-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_DIV_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
246-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_DIV, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
247-
248-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_POW_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
249-
250-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_XOR_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
251-
new SparseRowPerfTest(2000, 10000).testBinaryPrimitivePerf(BinType.VECT_XOR_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
243+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_LESS_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR, TestType.SPARSITY, SparsityType.SET1);
244+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_LESS_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
245+
new SparseRowPerfTest().testBinaryPrimitivePerf(BinType.VECT_LESS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE, TestType.SPARSITY, SparsityType.SET1);
252246
}
253247
}

‎src/test/java/org/apache/sysds/performance/primitives/TypicalExpressionTest.java‎

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import org.apache.sysds.common.Types.ExecMode;
66
import org.apache.sysds.common.Types.ExecType;
77
import org.apache.sysds.hops.OptimizerUtils;
8+
import org.apache.sysds.hops.codegen.cplan.CNodeBinary;
9+
import org.apache.sysds.performance.TimingUtils;
810
import org.apache.sysds.runtime.matrix.data.MatrixValue;
911
import org.apache.sysds.test.AutomatedTestBase;
1012
import org.apache.sysds.test.TestConfiguration;
@@ -13,6 +15,11 @@
1315
import org.junit.Test;
1416

1517
import java.io.File;
18+
import java.io.FileWriter;
19+
import java.io.IOException;
20+
import java.io.PrintWriter;
21+
import java.time.LocalDateTime;
22+
import java.time.format.DateTimeFormatter;
1623
import java.util.HashMap;
1724

1825
public class TypicalExpressionTest extends AutomatedTestBase {
@@ -21,30 +28,106 @@ public class TypicalExpressionTest extends AutomatedTestBase {
2128

2229
private static final String TEST_NAME = "expression";
2330
private static final String TEST_NAME1 = TEST_NAME+"1";
31+
private static final String TEST_NAME2 = TEST_NAME+"2";
2432

2533
private static final String TEST_DIR = "performance/primitives/";
2634
private static final String TEST_CLASS_DIR = TEST_DIR + TypicalExpressionTest.class.getSimpleName() + "/";
2735
private final static String TEST_CONF = "SystemDS-config-codegen.xml";
2836
private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF);
2937

30-
private final static int rows = 500;
31-
private final static int cols = 1000;
3238
private final static double sparsity1 = 0.9;
3339
private final static double sparsity2 = 0.1;
3440
private final static double eps = 1e-8;
41+
private int rows = 500;
42+
private int cols = 1000;
43+
double[] sparsities = new double[] {1, 0.3333, 0.1111, 0.0333, 0.0111, 0.0033, 0.0011};
44+
int warmupRuns = 100;
45+
int repetitions = 100;
3546

3647
@Override
3748
public void setUp() {
3849
TestUtils.clearAssertionInformation();
39-
for(int i=1; i<=1; i++)
50+
for(int i=1; i<=2; i++)
4051
addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) );
4152
}
4253

4354
@Test
44-
public void testSparseNewImplCP() {runSparseExpression(TEST_NAME1, true, true, ExecType.CP);}
55+
public void testSparseNewImpl1CP() {runSparseExpression(TEST_NAME1, true, true, ExecType.CP);}
4556

4657
@Test
47-
public void testOldSparseImplCP() {runSparseExpression(TEST_NAME1, true, false, ExecType.CP);}
58+
public void testOldSparseImpl1CP() {runSparseExpression(TEST_NAME1, true, false, ExecType.CP);}
59+
60+
@Test
61+
public void testSparseNewImpl2CP() {runSparseExpression(TEST_NAME2, true, true, ExecType.CP);}
62+
63+
@Test
64+
public void testOldSparseImpl2CP() {runSparseExpression(TEST_NAME2, true, false, ExecType.CP);}
65+
66+
public static void main(String[] args) {
67+
new TypicalExpressionTest().runBenchmark(TEST_NAME2);
68+
}
69+
70+
public void runBenchmark(String testname) {
71+
setUp();
72+
logResults(runPerfTest(testname, true, ExecType.CP), true);
73+
logResults(runPerfTest(testname, false, ExecType.CP), false);
74+
}
75+
76+
public void runBenchmark(String testname, int rows, int cols) {
77+
setUp();
78+
this.rows = rows;
79+
this.cols = cols;
80+
logResults(runPerfTest(testname, true, ExecType.CP), true);
81+
logResults(runPerfTest(testname, false, ExecType.CP), false);
82+
}
83+
84+
private String[] runPerfTest(String testname, boolean sparseRowVec, ExecType et) {
85+
86+
ExecMode platformOld = setExecMode(et);
87+
88+
String[] resultTime = new String[sparsities.length];
89+
90+
try {
91+
92+
getAndLoadTestConfiguration(testname);
93+
94+
String HOME = SCRIPT_DIR + TEST_DIR;
95+
fullDMLScriptName = HOME + testname + ".dml";
96+
if(sparseRowVec)
97+
programArgs = new String[]{"-explain", "codegen", "-sparseIntermediate", "-args",
98+
input("A"), input("B"), input("V"), output("S")};
99+
else
100+
programArgs = new String[]{"-explain", "codegen", "-args",
101+
input("A"), input("B"), input("V"), output("S")};
102+
103+
104+
fullRScriptName = HOME + TEST_NAME1 + ".R";
105+
rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
106+
107+
for(int i = 0; i < sparsities.length; i++) {
108+
//get a random matrix of values with
109+
double[][] A = getRandomMatrix(rows, cols, 1, 31, sparsities[i], 1234);
110+
double[][] B = getRandomMatrix(rows, cols, 1, 31, sparsities[i], 5678);
111+
double[][] V = getRandomMatrix(rows, 1, 1, 31, sparsities[i], 9876);
112+
writeInputMatrixWithMTD("A", A, true);
113+
writeInputMatrixWithMTD("B", B, true);
114+
writeInputMatrixWithMTD("V", V, true);
115+
116+
TimingUtils.time(() -> runTest(true, false, null, -1), warmupRuns);
117+
double[] result = TimingUtils.time(() -> runTest(true, false, null, -1), repetitions);
118+
119+
resultTime[i] = TimingUtils.stats(result).split("\\+-")[0];
120+
121+
}
122+
123+
}
124+
finally {
125+
resetExecMode(platformOld);
126+
OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
127+
OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
128+
}
129+
return resultTime;
130+
}
48131

49132
private void runSparseExpression(String testname, boolean sparse, boolean sparseRowVec, ExecType et) {
50133

@@ -55,7 +138,7 @@ private void runSparseExpression(String testname, boolean sparse, boolean sparse
55138
getAndLoadTestConfiguration(testname);
56139

57140
String HOME = SCRIPT_DIR + TEST_DIR;
58-
fullDMLScriptName = HOME + TEST_NAME1 + ".dml";
141+
fullDMLScriptName = HOME + testname + ".dml";
59142
if(sparseRowVec)
60143
programArgs = new String[]{"-explain", "codegen", "-sparseIntermediate", "-args",
61144
input("A"), input("B"), input("V"), output("S")};
@@ -92,6 +175,26 @@ private void runSparseExpression(String testname, boolean sparse, boolean sparse
92175

93176
}
94177

178+
public void logResults(String[] result, boolean sparseInterm) {
179+
String currDate = LocalDateTime.now().format(DateTimeFormatter.ofPattern("ddMMyyyy_HHmmss"));
180+
PrintWriter writer = null;
181+
try {
182+
writer = new PrintWriter(new FileWriter("C:\\Users\\tomok\\OneDrive - Technische Universität Berlin\\Bachelorarbeit\\performance\\expression_testing"
183+
+ "expressionTest_"+ currDate + "_" + (sparseInterm ? "sparseInterm" : "denseInterm") + "_" + ".csv"));
184+
}
185+
catch(IOException e) {
186+
throw new RuntimeException(e);
187+
}
188+
writer.printf("%4$s Repetitions: %1$2s, rl: %2$2s, cl: %3$2s with %5$2s%n", repetitions, rows, cols);
189+
writer.printf("%1$2s;%2$2s;%3$2s;%4$2s%n", "Sparsity", "rows", "cols","time in ms");
190+
for(int i = 0; i < sparsities.length; i++) {
191+
writer.printf("%1$2s;%2$2s;%3$2s;%4$2s%n", result[i]);
192+
}
193+
194+
writer.flush();
195+
writer.close();
196+
}
197+
95198
/**
96199
* Override default configuration with custom test configuration to ensure
97200
* scratch space and local temporary directory locations are also updated.

‎src/test/scripts/performance/primitives/SystemDS-config-codegen.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<!--
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
-->
19+
20+
<root>
21+
<sysds.localtmpdir>/tmp/systemds</sysds.localtmpdir>
22+
<sysds.scratch>scratch_space</sysds.scratch>
23+
<sysds.optlevel>7</sysds.optlevel>
24+
<sysds.codegen.enabled>true</sysds.codegen.enabled>
25+
<sysds.codegen.plancache>true</sysds.codegen.plancache>
26+
<sysds.codegen.literals>1</sysds.codegen.literals>
27+
28+
<!-- The number of threads for the spark instance artificially selected-->
29+
<sysds.local.spark.number.threads>16</sysds.local.spark.number.threads>
30+
31+
<sysds.codegen.api>auto</sysds.codegen.api>
32+
</root>

‎src/test/scripts/performance/primitives/expression1.R‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#-------------------------------------------------------------
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
#-------------------------------------------------------------
21+
22+
args <- commandArgs(TRUE)
23+
options(digits=22)
24+
library("Matrix")
25+
library("matrixStats")
26+
27+
A <- as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
28+
v <- as.vector(readMM(paste(args[1], "V.mtx", sep="")))
29+
30+
S = rowMaxs(A*v)
31+
32+
writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep=""));

‎src/test/scripts/performance/primitives/expression1.dml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#-------------------------------------------------------------
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
#-------------------------------------------------------------
21+
22+
A = read($1);
23+
v = read($3);
24+
25+
S = rowMaxs(A*v)
26+
27+
write(S, $4);
28+

‎src/test/scripts/performance/primitives/expression2.R‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#-------------------------------------------------------------
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
#-------------------------------------------------------------
21+
22+
args <- commandArgs(TRUE)
23+
options(digits=22)
24+
library("Matrix")
25+
26+
A <- as.matrix(readMM(paste(args[1], "A.mtx", sep="")))
27+
B <- as.matrix(readMM(paste(args[1], "B.mtx", sep="")))
28+
29+
S = (A!=0) * (B-A)
30+
31+
writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep=""));

‎src/test/scripts/performance/primitives/expression2.dml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#-------------------------------------------------------------
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
#
20+
#-------------------------------------------------------------
21+
22+
A = read($1);
23+
B = read($2);
24+
25+
S = (A!=0) * (B-A)
26+
27+
write(S, $4);
28+

0 commit comments

Comments
 (0)