fix hana insertion

Author: awildturtok

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java

@@ -58,7 +58,7 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) {
 
 	@Override
 	public Expression buildExpression(CTConditionContext context, ConceptElement<?> id) {
-		return new Expression(id, Map.of(field(name(getColumn()), VARCHAR).as("%s_equal".formatted(column)), values.stream().map(DSL::val).collect(Collectors.toSet())));
+		return new Expression(id, Map.of(field(name(getColumn()), VARCHAR(32)).as("%s_equal".formatted(column)), values.stream().map(DSL::val).collect(Collectors.toSet())));
 
 	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java

@@ -15,6 +15,7 @@
 import com.bakdata.conquery.util.CalculatedValue;
 import com.bakdata.conquery.util.CollectionsUtil;
 import com.fasterxml.jackson.annotation.JsonCreator;
+import com.google.common.base.Preconditions;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
@@ -49,8 +50,6 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) {
 
 	@Override
 	public Expression buildExpression(CTConditionContext context, ConceptElement<?> id) {
-		return new Expression(id,
-							  Map.of(context.getConnectorColumn(), values.stream().map(DSL::val).collect(Collectors.toSet()))
-		);
+		return new Expression(id, Map.of(context.getConnectorColumn(), values.stream().map(DSL::val).collect(Collectors.toSet())));
 	}
 }

backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java

@@ -49,6 +49,7 @@
 import org.jooq.SelectConditionStep;
 import org.jooq.SelectJoinStep;
 import org.jooq.Table;
+import org.jooq.exception.DataAccessException;
 
 @Slf4j
 @Data
@@ -79,7 +80,10 @@ private static void assignStatsToPath(ConceptElementId<?> resolvedId, Map<Concep
 	 * collect unique fields used/defined in the expressions.
 	 */
 	private static List<Field<?>> collectAllFields(List<CTCondition.Expression> expressions) {
+
+
 		List<Field<?>> fields = expressions.stream()
+										   //TODO determine length of chars, for now we are relying on a fixed length because it's quite cumbersome
 										   .map(expression -> expression.conditions().keySet())
 										   .flatMap(Collection::stream)
 										   .distinct()
@@ -119,11 +123,13 @@ public void createConceptIdJoinTable(TreeConcept concept) {
 		Name tableName = idsTableName(concept.getName());
 
 		// allFields are the statements to extract values from the underlying tables, we use them to generate the field names
-		List<Field<?>> fieldNames = new ArrayList<>(allFields);
-		fieldNames.addFirst(field(CONCEPT_ID_FIELD.getName(), VARCHAR(findMaxIdLength(expressions))));
+		List<Field<?>> fields = new ArrayList<>();
+
+		fields.addAll(allFields);
+		fields.addFirst(field(CONCEPT_ID_FIELD.getName(), VARCHAR(findMaxIdLength(expressions))));
 
-		createConceptIdsTable(tableName, fieldNames);
-		insertConceptIdMappings(tableName, fieldNames, rows, dslContext);
+		createConceptIdsTable(tableName, fields);
+		insertConceptIdMappings(tableName, fields, rows, dslContext);
 	}
 
 	@NotNull
@@ -211,11 +217,19 @@ private Name idsTableName(@NotBlank String name) {
 	private void insertConceptIdMappings(Name tableName, List<Field<?>> fieldNames, List<RowN> rows, DSLContext dsl) {
 		log.info("BEGIN inserting {} rows into {}", rows.size(), tableName);
 
-		InsertValuesStepN<Record> insertConceptTable = dsl.insertInto(table(tableName))
-														  .columns(fieldNames)
-														  .valuesOfRows(rows);
+		// We're using batching here because some DBMS don't allow mass inserts.
+		// There's a chance, we rework this to use a prepared statement with lots of bindings under the hood. But that needs to rework the entire stream of rows.
+		List<InsertValuesStepN<?>> inserts = new ArrayList<>(rows.size());
+
+		for (RowN row : rows) {
+			inserts.add(dsl.insertInto(table(tableName))
+						   .columns(fieldNames)
+						   .values(row));
+		}
+
+		dsl.batch(inserts)
+				.execute();
 
-		insertConceptTable.execute();
 
 		log.trace("DONE inserting into {}", tableName);
 	}
@@ -224,17 +238,23 @@ private void insertConceptIdMappings(Name tableName, List<Field<?>> fieldNames,
 	 * Drop the table, then recreate it.
 	 * TODO add an index.
 	 */
-	private void createConceptIdsTable(Name tableName, List<Field<?>> fieldNames) {
+	private void createConceptIdsTable(Name tableName, List<Field<?>> fields) {
 
-		log.debug("Creating table {} with fields {}", tableName, fieldNames);
+		log.debug("Creating table {} with fields {}", tableName, fields);
 
-		dslContext.dropTableIfExists(tableName)
-				  .cascade()
-				  .execute();
+		try {
+			dslContext.dropTable(tableName)
+					  .cascade()
+					  .execute();
+		}
+		catch (DataAccessException exception) {
+			// Likely it doesn't exist. Some DBMS just don't support drop-IfExists so this is the next best thing :^)
+			log.trace("Failed to drop table {}", tableName, exception);
+		}
 
 		CreateTableElementListStep createTable =
 				dslContext.createTable(tableName)
-						  .columns(fieldNames);
+						  .columns(fields);
 
 
 		createTable.execute();