diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ef02745..fe4c371c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ jobs: executor: java17 steps: - run_maven: - maven_subcommand: test + maven_subcommand: verify - slack/notify: event: fail diff --git a/.gitignore b/.gitignore index ae1ac990..edbffa9b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ target/ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* +.DS_Store diff --git a/checkstyle.xml b/checkstyle.xml index 76f872ed..f5e7412d 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -181,8 +181,8 @@ --> - @@ -67,6 +67,7 @@ test + org.apache.maven.plugins diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java index 752b53c9..0820c159 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java @@ -24,11 +24,13 @@ package com.kingsrook.qqq.backend.module.rdbms; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.interfaces.CountInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSCountAction; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSDeleteAction; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSInsertAction; import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSQueryAction; @@ -74,6 +76,18 @@ public class RDBMSBackendModule implements QBackendModuleInterface + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return (new RDBMSCountAction()); + } + + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 0f1c7110..85166328 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -25,8 +25,13 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.io.Serializable; import java.sql.Connection; import java.sql.SQLException; -import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.model.actions.AbstractQTableRequest; +import com.kingsrook.qqq.backend.core.model.actions.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; @@ -92,7 +97,7 @@ public abstract class AbstractRDBMSAction ** Handle obvious problems with values - like empty string for integer should be null. ** *******************************************************************************/ - protected Serializable scrubValue(QFieldMetaData field, Serializable value) + protected Serializable scrubValue(QFieldMetaData field, Serializable value, boolean isInsert) { if("".equals(value)) { @@ -103,14 +108,201 @@ public abstract class AbstractRDBMSAction } } - ////////////////////////////////////////////////////// - // todo - let this come from something in the field // - ////////////////////////////////////////////////////// - if(value == null && (field.getName().equals("createDate") || field.getName().equals("modifyDate"))) - { - value = OffsetDateTime.now(); - } - return (value); } + + + + /******************************************************************************* + ** If the table has a field with the given name, then set the given value in the + ** given record. + *******************************************************************************/ + protected void setValueIfTableHasField(QRecord record, QTableMetaData table, String fieldName, Serializable value) + { + QFieldMetaData field = table.getField(fieldName); + if(field != null) + { + record.setValue(fieldName, value); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected String makeWhereClause(QTableMetaData table, List criteria, List params) throws IllegalArgumentException + { + List clauses = new ArrayList<>(); + for(QFilterCriteria criterion : criteria) + { + QFieldMetaData field = table.getField(criterion.getFieldName()); + List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); + String column = getColumnName(field); + String clause = column; + Integer expectedNoOfParams = null; + switch(criterion.getOperator()) + { + case EQUALS: + { + clause += " = ? "; + expectedNoOfParams = 1; + break; + } + case NOT_EQUALS: + { + clause += " != ? "; + expectedNoOfParams = 1; + break; + } + case IN: + { + clause += " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ") "; + break; + } + case NOT_IN: + { + clause += " NOT IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ") "; + break; + } + case STARTS_WITH: + { + clause += " LIKE ? "; + editFirstValue(values, (s -> s + "%")); + expectedNoOfParams = 1; + break; + } + case ENDS_WITH: + { + clause += " LIKE ? "; + editFirstValue(values, (s -> "%" + s)); + expectedNoOfParams = 1; + break; + } + case CONTAINS: + { + clause += " LIKE ? "; + editFirstValue(values, (s -> "%" + s + "%")); + expectedNoOfParams = 1; + break; + } + case NOT_STARTS_WITH: + { + clause += " NOT LIKE ? "; + editFirstValue(values, (s -> s + "%")); + expectedNoOfParams = 1; + break; + } + case NOT_ENDS_WITH: + { + clause += " NOT LIKE ? "; + editFirstValue(values, (s -> "%" + s)); + expectedNoOfParams = 1; + break; + } + case NOT_CONTAINS: + { + clause += " NOT LIKE ? "; + editFirstValue(values, (s -> "%" + s + "%")); + expectedNoOfParams = 1; + break; + } + case LESS_THAN: + { + clause += " < ? "; + expectedNoOfParams = 1; + break; + } + case LESS_THAN_OR_EQUALS: + { + clause += " <= ? "; + expectedNoOfParams = 1; + break; + } + case GREATER_THAN: + { + clause += " > ? "; + expectedNoOfParams = 1; + break; + } + case GREATER_THAN_OR_EQUALS: + { + clause += " >= ? "; + expectedNoOfParams = 1; + break; + } + case IS_BLANK: + { + clause += " IS NULL "; + if(isString(field.getType())) + { + clause += " OR " + column + " = '' "; + } + expectedNoOfParams = 0; + break; + } + case IS_NOT_BLANK: + { + clause += " IS NOT NULL "; + if(isString(field.getType())) + { + clause += " AND " + column + " !+ '' "; + } + expectedNoOfParams = 0; + break; + } + case BETWEEN: + { + clause += " BETWEEN ? AND ? "; + expectedNoOfParams = 2; + break; + } + case NOT_BETWEEN: + { + clause += " NOT BETWEEN ? AND ? "; + expectedNoOfParams = 2; + break; + } + default: + { + throw new IllegalArgumentException("Unexpected operator: " + criterion.getOperator()); + } + } + clauses.add("(" + clause + ")"); + if(expectedNoOfParams != null) + { + if(!expectedNoOfParams.equals(values.size())) + { + throw new IllegalArgumentException("Incorrect number of values given for criteria [" + field.getName() + "]"); + } + } + + params.addAll(values); + } + + return (String.join(" AND ", clauses)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void editFirstValue(List values, Function editFunction) + { + if(values.size() > 0) + { + values.set(0, editFunction.apply(String.valueOf(values.get(0)))); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isString(QFieldType fieldType) + { + return fieldType == QFieldType.STRING || fieldType == QFieldType.TEXT || fieldType == QFieldType.HTML || fieldType == QFieldType.PASSWORD; + } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java new file mode 100644 index 00000000..d0970a73 --- /dev/null +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.count.CountRequest; +import com.kingsrook.qqq.backend.core.model.actions.count.CountResult; +import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterface +{ + private static final Logger LOG = LogManager.getLogger(RDBMSCountAction.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public CountResult execute(CountRequest countRequest) throws QException + { + try + { + QTableMetaData table = countRequest.getTable(); + String tableName = getTableName(table); + + String sql = "SELECT count(*) as record_count FROM " + tableName; + + QQueryFilter filter = countRequest.getFilter(); + List params = new ArrayList<>(); + if(filter != null && CollectionUtils.nullSafeHasContents(filter.getCriteria())) + { + sql += " WHERE " + makeWhereClause(table, filter.getCriteria(), params); + } + + // todo sql customization - can edit sql and/or param list + + CountResult rs = new CountResult(); + + try(Connection connection = getConnection(countRequest)) + { + QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) -> + { + ResultSetMetaData metaData = resultSet.getMetaData(); + if(resultSet.next()) + { + rs.setCount(resultSet.getInt("record_count")); + } + + }), params); + } + + return rs; + } + catch(Exception e) + { + LOG.warn("Error executing count", e); + throw new QException("Error executing count", e); + } + } + +} diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index 3495a621..ee75105d 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -65,16 +65,18 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // todo sql customization - can edit sql and/or param list - Connection connection = getConnection(deleteRequest); - QueryManager.executeUpdateForRowCount(connection, sql, params); - List outputRecords = new ArrayList<>(); - rs.setRecords(outputRecords); - for(Serializable primaryKey : deleteRequest.getPrimaryKeys()) + try(Connection connection = getConnection(deleteRequest)) { - QRecord qRecord = new QRecord().withTableName(deleteRequest.getTableName()).withValue("id", primaryKey); - // todo uh, identify any errors? - QRecord outputRecord = new QRecord(qRecord); - outputRecords.add(outputRecord); + QueryManager.executeUpdateForRowCount(connection, sql, params); + List outputRecords = new ArrayList<>(); + rs.setRecords(outputRecords); + for(Serializable primaryKey : deleteRequest.getPrimaryKeys()) + { + QRecord qRecord = new QRecord().withTableName(deleteRequest.getTableName()).withValue("id", primaryKey); + // todo uh, identify any errors? + QRecord outputRecord = new QRecord(qRecord); + outputRecords.add(outputRecord); + } } return rs; diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index c55fff9f..0176f96c 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.io.Serializable; import java.sql.Connection; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -36,6 +37,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* @@ -43,22 +46,38 @@ import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; *******************************************************************************/ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInterface { + private static final Logger LOG = LogManager.getLogger(RDBMSInsertAction.class); + + /******************************************************************************* ** *******************************************************************************/ public InsertResult execute(InsertRequest insertRequest) throws QException { + InsertResult rs = new InsertResult(); + if(CollectionUtils.nullSafeIsEmpty(insertRequest.getRecords())) { - throw (new QException("Request to insert 0 records.")); + LOG.info("Insert request called with 0 records. Returning with no-op"); + rs.setRecords(new ArrayList<>()); + return (rs); + } + + QTableMetaData table = insertRequest.getTable(); + Instant now = Instant.now(); + + for(QRecord record : insertRequest.getRecords()) + { + /////////////////////////////////////////// + // todo .. better (not hard-coded names) // + /////////////////////////////////////////// + setValueIfTableHasField(record, table, "createDate", now); + setValueIfTableHasField(record, table, "modifyDate", now); } try { - InsertResult rs = new InsertResult(); - QTableMetaData table = insertRequest.getTable(); - List insertableFields = table.getFields().values().stream() .filter(field -> !field.getName().equals("id")) // todo - intent here is to avoid non-insertable fields. .toList(); @@ -74,42 +93,41 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte StringBuilder sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); List params = new ArrayList<>(); - int recordIndex = 0; - for(QRecord record : insertRequest.getRecords()) + try(Connection connection = getConnection(insertRequest)) { - if(recordIndex++ > 0) + for(List page : CollectionUtils.getPages(insertRequest.getRecords(), QueryManager.PAGE_SIZE)) { - sql.append(","); + int recordIndex = 0; + for(QRecord record : page) + { + if(recordIndex++ > 0) + { + sql.append(","); + } + sql.append("(").append(questionMarks).append(")"); + for(QFieldMetaData field : insertableFields) + { + Serializable value = record.getValue(field.getName()); + value = scrubValue(field, value, true); + params.add(value); + } + } + + // todo sql customization - can edit sql and/or param list + // todo - non-serial-id style tables + // todo - other generated values, e.g., createDate... maybe need to re-select? + List idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params); + List outputRecords = new ArrayList<>(); + rs.setRecords(outputRecords); + int index = 0; + for(QRecord record : insertRequest.getRecords()) + { + Integer id = idList.get(index++); + QRecord outputRecord = new QRecord(record); + outputRecord.setValue(table.getPrimaryKeyField(), id); + outputRecords.add(outputRecord); + } } - sql.append("(").append(questionMarks).append(")"); - for(QFieldMetaData field : insertableFields) - { - Serializable value = record.getValue(field.getName()); - value = scrubValue(field, value); - - params.add(value); - } - } - - // todo sql customization - can edit sql and/or param list - - // QueryResult rs = new QueryResult(); - // List records = new ArrayList<>(); - // rs.setRecords(records); - - // todo - non-serial-id style tables - // todo - other generated values, e.g., createDate... maybe need to re-select? - Connection connection = getConnection(insertRequest); - List idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params); - List outputRecords = new ArrayList<>(); - rs.setRecords(outputRecords); - int index = 0; - for(QRecord record : insertRequest.getRecords()) - { - Integer id = idList.get(index++); - QRecord outputRecord = new QRecord(record); - outputRecord.setValue(table.getPrimaryKeyField(), id); - outputRecords.add(outputRecord); } return rs; diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 6c892ee8..1a7d3d3f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -56,6 +56,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { private static final Logger LOG = LogManager.getLogger(RDBMSQueryAction.class); + + /******************************************************************************* ** *******************************************************************************/ @@ -102,29 +104,31 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List records = new ArrayList<>(); rs.setRecords(records); - Connection connection = getConnection(queryRequest); - QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) -> + try(Connection connection = getConnection(queryRequest)) { - ResultSetMetaData metaData = resultSet.getMetaData(); - while(resultSet.next()) + QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) -> { - // todo - should refactor this for view etc to use too. - // todo - Add display values (String labels for possibleValues, formatted #'s, etc) - QRecord record = new QRecord(); - records.add(record); - record.setTableName(table.getName()); - LinkedHashMap values = new LinkedHashMap<>(); - record.setValues(values); - - for(int i = 1; i <= metaData.getColumnCount(); i++) + ResultSetMetaData metaData = resultSet.getMetaData(); + while(resultSet.next()) { - QFieldMetaData qFieldMetaData = fieldList.get(i - 1); - Serializable value = getValue(qFieldMetaData, resultSet, i); - values.put(qFieldMetaData.getName(), value); - } - } + // todo - should refactor this for view etc to use too. + // todo - Add display values (String labels for possibleValues, formatted #'s, etc) + QRecord record = new QRecord(); + records.add(record); + record.setTableName(table.getName()); + LinkedHashMap values = new LinkedHashMap<>(); + record.setValues(values); - }), params); + for(int i = 1; i <= metaData.getColumnCount(); i++) + { + QFieldMetaData qFieldMetaData = fieldList.get(i - 1); + Serializable value = getValue(qFieldMetaData, resultSet, i); + values.put(qFieldMetaData.getName(), value); + } + } + + }), params); + } return rs; } @@ -177,186 +181,6 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf - /******************************************************************************* - ** - *******************************************************************************/ - private String makeWhereClause(QTableMetaData table, List criteria, List params) throws IllegalArgumentException - { - List clauses = new ArrayList<>(); - for(QFilterCriteria criterion : criteria) - { - QFieldMetaData field = table.getField(criterion.getFieldName()); - List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); - String column = getColumnName(field); - String clause = column; - Integer expectedNoOfParams = null; - switch(criterion.getOperator()) - { - case EQUALS: - { - clause += " = ? "; - expectedNoOfParams = 1; - break; - } - case NOT_EQUALS: - { - clause += " != ? "; - expectedNoOfParams = 1; - break; - } - case IN: - { - clause += " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ") "; - break; - } - case NOT_IN: - { - clause += " NOT IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ") "; - break; - } - case STARTS_WITH: - { - clause += " LIKE ? "; - editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - break; - } - case ENDS_WITH: - { - clause += " LIKE ? "; - editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - break; - } - case CONTAINS: - { - clause += " LIKE ? "; - editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - break; - } - case NOT_STARTS_WITH: - { - clause += " NOT LIKE ? "; - editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - break; - } - case NOT_ENDS_WITH: - { - clause += " NOT LIKE ? "; - editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - break; - } - case NOT_CONTAINS: - { - clause += " NOT LIKE ? "; - editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - break; - } - case LESS_THAN: - { - clause += " < ? "; - expectedNoOfParams = 1; - break; - } - case LESS_THAN_OR_EQUALS: - { - clause += " <= ? "; - expectedNoOfParams = 1; - break; - } - case GREATER_THAN: - { - clause += " > ? "; - expectedNoOfParams = 1; - break; - } - case GREATER_THAN_OR_EQUALS: - { - clause += " >= ? "; - expectedNoOfParams = 1; - break; - } - case IS_BLANK: - { - clause += " IS NULL "; - if(isString(field.getType())) - { - clause += " OR " + column + " = '' "; - } - expectedNoOfParams = 0; - break; - } - case IS_NOT_BLANK: - { - clause += " IS NOT NULL "; - if(isString(field.getType())) - { - clause += " AND " + column + " !+ '' "; - } - expectedNoOfParams = 0; - break; - } - case BETWEEN: - { - clause += " BETWEEN ? AND ? "; - expectedNoOfParams = 2; - break; - } - case NOT_BETWEEN: - { - clause += " NOT BETWEEN ? AND ? "; - expectedNoOfParams = 2; - break; - } - default: - { - throw new IllegalArgumentException("Unexpected operator: " + criterion.getOperator()); - } - } - clauses.add("(" + clause + ")"); - if(expectedNoOfParams != null) - { - if(!expectedNoOfParams.equals(values.size())) - { - throw new IllegalArgumentException("Incorrect number of values given for criteria [" + field.getName() + "]"); - } - } - - params.addAll(values); - } - - return (String.join(" AND ", clauses)); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private void editFirstValue(List values, Function editFunction) - { - if(values.size() > 0) - { - values.set(0, editFunction.apply(String.valueOf(values.get(0)))); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private boolean isString(QFieldType fieldType) - { - return fieldType == QFieldType.STRING || fieldType == QFieldType.TEXT || fieldType == QFieldType.HTML || fieldType == QFieldType.PASSWORD; - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index c9bdb280..30a40eca 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -24,8 +24,12 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.io.Serializable; import java.sql.Connection; +import java.sql.SQLException; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; @@ -34,80 +38,227 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /******************************************************************************* - ** + ** Only the fields which exist in the record's values map will be updated. + ** Note the difference between a field being in the value map, with a null value, + ** vs. not being in the map. If the field (its key) is in the value map, with a + ** null value, then the field will be updated to NULL. But if it's not in the + ** map, then it'll be ignored. This would be to do a PATCH type operation, vs a + ** PUT. See https://rapidapi.com/blog/put-vs-patch/ *******************************************************************************/ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInterface { + private static final Logger LOG = LogManager.getLogger(RDBMSUpdateAction.class); + + /******************************************************************************* ** *******************************************************************************/ public UpdateResult execute(UpdateRequest updateRequest) throws QException { - try + UpdateResult rs = new UpdateResult(); + + if(CollectionUtils.nullSafeIsEmpty(updateRequest.getRecords())) { - UpdateResult rs = new UpdateResult(); - QTableMetaData table = updateRequest.getTable(); + LOG.info("Update request called with 0 records. Returning with no-op"); + rs.setRecords(new ArrayList<>()); + return (rs); + } - List outputRecords = new ArrayList<>(); - rs.setRecords(outputRecords); + QTableMetaData table = updateRequest.getTable(); + Instant now = Instant.now(); - // todo - sql batch for performance - // todo - if setting a bunch of records to have the same value, a single update where id IN? - Connection connection = getConnection(updateRequest); - int recordIndex = 0; - for(QRecord record : updateRequest.getRecords()) + List outputRecords = new ArrayList<>(); + rs.setRecords(outputRecords); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // we want to do batch updates. But, since we only update the columns columns that // + // are present in each record, it means we may have different update SQL for each // + // record. So, we will first "hash" up the records by their list of fields being updated. // + ///////////////////////////////////////////////////////////////////////////////////////////// + ListingHash, QRecord> recordsByFieldBeingUpdated = new ListingHash<>(); + for(QRecord record : updateRequest.getRecords()) + { + //////////////////////////////////////////// + // todo .. better (not a hard-coded name) // + //////////////////////////////////////////// + setValueIfTableHasField(record, table, "modifyDate", now); + + List updatableFields = table.getFields().values().stream() + .map(QFieldMetaData::getName) + // todo - intent here is to avoid non-updateable fields - but this + // should be like based on field.isUpdatable once that attribute exists + .filter(name -> !name.equals("id")) + .filter(name -> record.getValues().containsKey(name)) + .toList(); + recordsByFieldBeingUpdated.add(updatableFields, record); + + ////////////////////////////////////////////////////////////////////////////// + // go ahead and put the record into the output list at this point in time, // + // so that the output list's order matches the input list order // + // note that if we want to capture updated values (like modify dates), then // + // we may want a map of primary key to output record, for easy updating. // + ////////////////////////////////////////////////////////////////////////////// + QRecord outputRecord = new QRecord(record); + outputRecords.add(outputRecord); + } + + try(Connection connection = getConnection(updateRequest)) + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // process each distinct list of fields being updated (e.g., each different SQL statement) // + ///////////////////////////////////////////////////////////////////////////////////////////// + for(List fieldsBeingUpdated : recordsByFieldBeingUpdated.keySet()) { - List updateableFields = table.getFields().values().stream() - .filter(field -> !field.getName().equals("id")) // todo - intent here is to avoid non-updateable fields. - .filter(field -> record.getValues().containsKey(field.getName())) - .toList(); - - String columns = updateableFields.stream() - .map(f -> this.getColumnName(f) + " = ?") - .collect(Collectors.joining(", ")); - - String tableName = getTableName(table); - StringBuilder sql = new StringBuilder("UPDATE ").append(tableName) - .append(" SET ").append(columns) - .append(" WHERE ").append(getColumnName(table.getField(table.getPrimaryKeyField()))).append(" = ?"); - - // todo sql customization - can edit sql and/or param list - - QRecord outputRecord = new QRecord(record); - outputRecords.add(outputRecord); - - try - { - List params = new ArrayList<>(); - for(QFieldMetaData field : updateableFields) - { - Serializable value = record.getValue(field.getName()); - value = scrubValue(field, value); - params.add(value); - } - params.add(record.getValue(table.getPrimaryKeyField())); - - QueryManager.executeUpdate(connection, sql.toString(), params); - // todo - auto-updated values, e.g., modifyDate... maybe need to re-select? - } - catch(Exception e) - { - // todo - how to communicate errors??? outputRecord.setErrors(new ArrayList<>(List.of(e))); - throw new QException("Error executing update: " + e.getMessage(), e); - } + updateRecordsWithMatchingListOfFields(connection, table, recordsByFieldBeingUpdated.get(fieldsBeingUpdated), fieldsBeingUpdated); } return rs; } catch(Exception e) { + // todo - how to communicate errors??? outputRecord.setErrors(new ArrayList<>(List.of(e))); throw new QException("Error executing update: " + e.getMessage(), e); } } + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateRecordsWithMatchingListOfFields(Connection connection, QTableMetaData table, List recordList, List fieldsBeingUpdated) throws SQLException + { + //////////////////////////////////////////////////////////////////////////////// + // check for an optimization - if all of the records have the same values for // + // all fields being updated, just do 1 update, with an IN list on the ids. // + //////////////////////////////////////////////////////////////////////////////// + if(areAllValuesBeingUpdatedTheSame(recordList, fieldsBeingUpdated)) + { + updateRecordsWithMatchingValuesAndFields(connection, table, recordList, fieldsBeingUpdated); + return; + } + + String sql = writeUpdateSQLPrefix(table, fieldsBeingUpdated) + " = ?"; + + // todo sql customization? - let each table have custom sql and/or param list + + //////////////////////////////////////////////////////// + // build the list of list of values, from the records // + //////////////////////////////////////////////////////// + List> values = new ArrayList<>(); + for(QRecord record : recordList) + { + List rowValues = new ArrayList<>(); + values.add(rowValues); + + for(String fieldName : fieldsBeingUpdated) + { + Serializable value = record.getValue(fieldName); + value = scrubValue(table.getField(fieldName), value, false); + rowValues.add(value); + } + rowValues.add(record.getValue(table.getPrimaryKeyField())); + } + + //////////////////////////////////////////////////////////////////////////////// + // let query manager do the batch updates - note that it will internally page // + //////////////////////////////////////////////////////////////////////////////// + QueryManager.executeBatchUpdate(connection, sql, values); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String writeUpdateSQLPrefix(QTableMetaData table, List fieldsBeingUpdated) + { + String columns = fieldsBeingUpdated.stream() + .map(f -> this.getColumnName(table.getField(f)) + " = ?") + .collect(Collectors.joining(", ")); + + String tableName = getTableName(table); + return ("UPDATE " + tableName + + " SET " + columns + + " WHERE " + getColumnName(table.getField(table.getPrimaryKeyField())) + " "); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void updateRecordsWithMatchingValuesAndFields(Connection connection, QTableMetaData table, List recordList, List fieldsBeingUpdated) throws SQLException + { + for(List page : CollectionUtils.getPages(recordList, QueryManager.PAGE_SIZE)) + { + String sql = writeUpdateSQLPrefix(table, fieldsBeingUpdated) + " IN (" + StringUtils.join(",", Collections.nCopies(page.size(), "?")) + ")"; + + // todo sql customization? - let each table have custom sql and/or param list + + //////////////////////////////////////////////////////////////// + // values in the update clause can come from the first record // + //////////////////////////////////////////////////////////////// + QRecord record0 = page.get(0); + List params = new ArrayList<>(); + for(String fieldName : fieldsBeingUpdated) + { + Serializable value = record0.getValue(fieldName); + value = scrubValue(table.getField(fieldName), value, false); + params.add(value); + } + + ////////////////////////////////////////////////////////////////////// + // values in the where clause (in list) are the id from each record // + ////////////////////////////////////////////////////////////////////// + for(QRecord record : page) + { + params.add(record.getValue(table.getPrimaryKeyField())); + } + + ///////////////////////////////////// + // let query manager do the update // + ///////////////////////////////////// + QueryManager.executeUpdate(connection, sql, params); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private boolean areAllValuesBeingUpdatedTheSame(List recordList, List fieldsBeingUpdated) + { + if(recordList.size() == 1) + { + return (true); + } + + QRecord record0 = recordList.get(0); + for(int i = 1; i < recordList.size(); i++) + { + QRecord record = recordList.get(i); + for(String fieldName : fieldsBeingUpdated) + { + if(!Objects.equals(record0.getValue(fieldName), record.getValue(fieldName))) + { + return (false); + } + } + } + + return (true); + } + } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 43c6a1bc..6f2488e1 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -45,9 +45,9 @@ public class ConnectionManager { case "aurora": { - //TODO AWS version not working and why ssl=false required? + // TODO aws-mysql-jdbc driver not working when running on AWS + // jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=CONVERT_TO_NULL"; jdbcURL = "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; - //jdbcURL = "jdbc:mysql:aws://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; break; } case "mysql": @@ -57,7 +57,7 @@ public class ConnectionManager } case "h2": { - jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL"; + jdbcURL = "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; break; } default: diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index 6fa85207..78d126e0 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -22,13 +22,12 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; +import java.io.Serializable; import java.math.BigDecimal; -import java.math.BigInteger; import java.sql.Connection; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; @@ -43,13 +42,12 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.commons.lang.NotImplementedException; /******************************************************************************* @@ -57,10 +55,17 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class QueryManager { - private static final int PAGE_SIZE = 2000; - private static final int MS_PER_SEC = 1000; + public static final int PAGE_SIZE = 2000; + private static final int MS_PER_SEC = 1000; private static final int NINETEEN_HUNDRED = 1900; + private static boolean collectStatistics = false; + + private static final Map statistics = Collections.synchronizedMap(new HashMap<>()); + + public static final String STAT_QUERIES_RAN = "queriesRan"; + public static final String STAT_BATCHES_RAN = "batchesRan"; + /******************************************************************************* @@ -83,12 +88,13 @@ public class QueryManager public static void executeStatement(Connection connection, String sql, ResultSetProcessor procesor, Object... params) throws SQLException { PreparedStatement statement = null; - ResultSet resultSet = null; + ResultSet resultSet = null; try { statement = prepareStatementAndBindParams(connection, sql, params); statement.execute(); + incrementStatistic(STAT_QUERIES_RAN); resultSet = statement.getResultSet(); procesor.processResultSet(resultSet); @@ -114,6 +120,8 @@ public class QueryManager *******************************************************************************/ public static void executeStatementForeachResult(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* PreparedStatement statement = null; ResultSet resultSet = null; @@ -145,6 +153,7 @@ public class QueryManager resultSet.close(); } } + */ } @@ -155,6 +164,8 @@ public class QueryManager @SuppressWarnings("unchecked") public static T executeStatementForSingleValue(Connection connection, Class returnClass, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.execute(); ResultSet resultSet = statement.getResultSet(); @@ -203,6 +214,7 @@ public class QueryManager { return (null); } + */ } @@ -212,6 +224,8 @@ public class QueryManager *******************************************************************************/ public static Map executeStatementForSingleRow(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.execute(); ResultSet resultSet = statement.getResultSet(); @@ -231,6 +245,7 @@ public class QueryManager { return (null); } + */ } @@ -240,6 +255,8 @@ public class QueryManager *******************************************************************************/ public static SimpleEntity executeStatementForSimpleEntity(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.execute(); ResultSet resultSet = statement.getResultSet(); @@ -251,6 +268,7 @@ public class QueryManager { return (null); } + */ } @@ -260,6 +278,8 @@ public class QueryManager *******************************************************************************/ public static List> executeStatementForRows(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* List> rs = new ArrayList<>(); PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); @@ -278,6 +298,7 @@ public class QueryManager } return (rs); + */ } @@ -287,6 +308,8 @@ public class QueryManager *******************************************************************************/ public static List executeStatementForSimpleEntityList(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* List rs = new ArrayList<>(); PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); @@ -300,6 +323,7 @@ public class QueryManager } return (rs); + */ } @@ -309,6 +333,8 @@ public class QueryManager *******************************************************************************/ public static SimpleEntity buildSimpleEntity(ResultSet resultSet) throws SQLException { + throw (new NotImplementedException()); + /* SimpleEntity row = new SimpleEntity(); ResultSetMetaData metaData = resultSet.getMetaData(); @@ -317,6 +343,7 @@ public class QueryManager row.put(metaData.getColumnName(i), getObject(resultSet, i)); } return row; + */ } @@ -328,6 +355,7 @@ public class QueryManager { PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.executeUpdate(); + incrementStatistic(STAT_QUERIES_RAN); return (statement); } @@ -340,6 +368,7 @@ public class QueryManager { PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); statement.executeUpdate(); + incrementStatistic(STAT_QUERIES_RAN); return (statement); } @@ -350,10 +379,13 @@ public class QueryManager *******************************************************************************/ public static void executeUpdateVoid(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) { statement.executeUpdate(); } + */ } @@ -363,10 +395,13 @@ public class QueryManager *******************************************************************************/ public static void executeUpdateVoid(Connection connection, String sql, List params) throws SQLException { + throw (new NotImplementedException()); + /* try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) { statement.executeUpdate(); } + */ } @@ -379,6 +414,7 @@ public class QueryManager try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) { statement.executeUpdate(); + incrementStatistic(STAT_QUERIES_RAN); return (statement.getUpdateCount()); } } @@ -390,11 +426,14 @@ public class QueryManager *******************************************************************************/ public static Integer executeUpdateForRowCount(Connection connection, String sql, List params) throws SQLException { + throw (new NotImplementedException()); + /* try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) { statement.executeUpdate(); return (statement.getUpdateCount()); } + */ } @@ -404,6 +443,8 @@ public class QueryManager *******************************************************************************/ public static Integer executeInsertForGeneratedId(Connection connection, String sql, Object... params) throws SQLException { + throw (new NotImplementedException()); + /* try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) { bindParams(params, statement); @@ -418,12 +459,13 @@ public class QueryManager return (null); } } + */ } /******************************************************************************* - ** todo - needs unit test + ** todo - needs (specific) unit test *******************************************************************************/ public static List executeInsertForGeneratedIds(Connection connection, String sql, List params) throws SQLException { @@ -433,6 +475,7 @@ public class QueryManager bindParams(params.toArray(), statement); statement.executeUpdate(); ResultSet generatedKeys = statement.getGeneratedKeys(); + incrementStatistic(STAT_QUERIES_RAN); while(generatedKeys.next()) { rs.add(getInteger(generatedKeys, 1)); @@ -448,6 +491,8 @@ public class QueryManager *******************************************************************************/ public static void executeInsertForList(Connection connection, List entityList) throws SQLException { + throw (new NotImplementedException()); + /* List> pages = CollectionUtils.getPages(entityList, PAGE_SIZE); for(List page : pages) { @@ -474,6 +519,7 @@ public class QueryManager page.clear(); } pages.clear(); + */ } @@ -483,6 +529,8 @@ public class QueryManager *******************************************************************************/ public static Integer executeInsert(Connection connection, SimpleEntity entity) throws SQLException { + throw (new NotImplementedException()); + /* ArrayList columns = new ArrayList<>(entity.keySet()); String sql = "INSERT INTO " + entity.getTableName() + "(" + StringUtils.join(",", columns) + ") VALUES (" + columns.stream().map(s -> "?").collect(Collectors.joining(",")) + ")"; @@ -493,6 +541,33 @@ public class QueryManager } return (executeInsertForGeneratedId(connection, sql, params)); + */ + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException + { + for(List> page : CollectionUtils.getPages(values, PAGE_SIZE)) + { + PreparedStatement updatePS = connection.prepareStatement(updateSQL); + for(List row : page) + { + Object[] params = new Object[row.size()]; + for(int i = 0; i < row.size(); i++) + { + params[i] = row.get(i); + } + + bindParams(updatePS, params); + updatePS.addBatch(); + } + updatePS.executeBatch(); + incrementStatistic(STAT_BATCHES_RAN); + } } @@ -570,12 +645,13 @@ public class QueryManager @SuppressWarnings("unchecked") public static int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException { - if(value instanceof TypeValuePair) + /* if(value instanceof TypeValuePair) { bindParamTypeValuePair(statement, index, (TypeValuePair) value); return (1); } - else if(value instanceof Integer) + else*/ + if(value instanceof Integer) { bindParam(statement, index, (Integer) value); return (1); @@ -627,8 +703,8 @@ public class QueryManager } else if(value instanceof Collection) { - Collection collection = (Collection) value; - int paramsBound = 0; + Collection collection = (Collection) value; + int paramsBound = 0; for(Object o : collection) { paramsBound += bindParamObject(statement, (index + paramsBound), o); @@ -670,24 +746,23 @@ public class QueryManager } } - - /******************************************************************************* ** *******************************************************************************/ + /* public static TypeValuePair param(Class c, T v) { return (new TypeValuePair<>(c, v)); } - - + */ /******************************************************************************* ** *******************************************************************************/ + /* private static void bindParamTypeValuePair(PreparedStatement statement, int index, TypeValuePair value) throws SQLException { - Object v = value.getValue(); + Object v = value.getValue(); Class t = value.getType(); if(t.equals(Integer.class)) @@ -731,6 +806,7 @@ public class QueryManager throw (new SQLException("Unexpected value type [" + t.getSimpleName() + "] in bindParamTypeValuePair.")); } } + */ @@ -848,7 +924,7 @@ public class QueryManager else { LocalDateTime localDateTime = value.atTime(0, 0); - Timestamp timestamp = new Timestamp(localDateTime.atZone(ZoneId.systemDefault()).toEpochSecond() * MS_PER_SEC); // TimeStamp expects millis, not seconds, after epoch + Timestamp timestamp = new Timestamp(localDateTime.atZone(ZoneId.systemDefault()).toEpochSecond() * MS_PER_SEC); // TimeStamp expects millis, not seconds, after epoch statement.setTimestamp(index, timestamp); } } @@ -1044,12 +1120,15 @@ public class QueryManager *******************************************************************************/ public static BigDecimal getBigDecimal(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* BigDecimal value = resultSet.getBigDecimal(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1074,12 +1153,15 @@ public class QueryManager *******************************************************************************/ public static Date getDate(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Date value = resultSet.getDate(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1104,6 +1186,8 @@ public class QueryManager *******************************************************************************/ public static Calendar getCalendar(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { @@ -1112,6 +1196,7 @@ public class QueryManager Calendar rs = Calendar.getInstance(); rs.setTimeInMillis(value.getTime()); return (rs); + */ } @@ -1121,6 +1206,8 @@ public class QueryManager *******************************************************************************/ public static Calendar getCalendar(ResultSet resultSet, int column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { @@ -1129,6 +1216,7 @@ public class QueryManager Calendar rs = Calendar.getInstance(); rs.setTimeInMillis(value.getTime()); return (rs); + */ } @@ -1139,6 +1227,8 @@ public class QueryManager @SuppressWarnings("deprecation") public static LocalDate getLocalDate(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { @@ -1147,6 +1237,7 @@ public class QueryManager LocalDate date = LocalDate.of(value.getYear() + NINETEEN_HUNDRED, value.getMonth() + 1, value.getDate()); return (date); + */ } @@ -1157,6 +1248,8 @@ public class QueryManager @SuppressWarnings("deprecation") public static LocalDateTime getLocalDateTime(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { @@ -1165,6 +1258,7 @@ public class QueryManager LocalDateTime dateTime = LocalDateTime.of(value.getYear() + NINETEEN_HUNDRED, value.getMonth() + 1, value.getDate(), value.getHours(), value.getMinutes(), value.getSeconds(), 0); return (dateTime); + */ } @@ -1193,6 +1287,8 @@ public class QueryManager @SuppressWarnings("deprecation") public static OffsetDateTime getOffsetDateTime(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { @@ -1201,6 +1297,7 @@ public class QueryManager OffsetDateTime dateTime = OffsetDateTime.of(value.getYear() + NINETEEN_HUNDRED, value.getMonth() + 1, value.getDate(), value.getHours(), value.getMinutes(), value.getSeconds(), 0, OffsetDateTime.now().getOffset()); return (dateTime); + */ } @@ -1210,12 +1307,15 @@ public class QueryManager *******************************************************************************/ public static Boolean getBoolean(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Boolean value = resultSet.getBoolean(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1225,12 +1325,15 @@ public class QueryManager *******************************************************************************/ public static Boolean getBoolean(ResultSet resultSet, int column) throws SQLException { + throw (new NotImplementedException()); + /* Boolean value = resultSet.getBoolean(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1240,12 +1343,15 @@ public class QueryManager *******************************************************************************/ public static Long getLong(ResultSet resultSet, int column) throws SQLException { + throw (new NotImplementedException()); + /* long value = resultSet.getLong(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1255,12 +1361,15 @@ public class QueryManager *******************************************************************************/ public static Long getLong(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* long value = resultSet.getLong(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1270,12 +1379,15 @@ public class QueryManager *******************************************************************************/ public static Timestamp getTimestamp(ResultSet resultSet, int column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1285,12 +1397,15 @@ public class QueryManager *******************************************************************************/ public static Timestamp getTimestamp(ResultSet resultSet, String column) throws SQLException { + throw (new NotImplementedException()); + /* Timestamp value = resultSet.getTimestamp(column); if(resultSet.wasNull()) { return (null); } return (value); + */ } @@ -1304,7 +1419,10 @@ public class QueryManager *******************************************************************************/ public static Integer findIdForDaysAgo(Connection connection, String tableName, String dateFieldName, int goalDaysAgo) throws SQLException { + throw (new NotImplementedException()); + /* return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalDaysAgo, ChronoUnit.DAYS)); + */ } @@ -1314,8 +1432,11 @@ public class QueryManager *******************************************************************************/ public static Integer findIdForTimestamp(Connection connection, String tableName, String dateFieldName, LocalDateTime timestamp) throws SQLException { + throw (new NotImplementedException()); + /* long between = ChronoUnit.SECONDS.between(timestamp, LocalDateTime.now()); return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, (int) between, ChronoUnit.SECONDS)); + */ } @@ -1325,6 +1446,8 @@ public class QueryManager *******************************************************************************/ public static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, ChronoUnit unit) throws SQLException { + throw (new NotImplementedException()); + /* Integer maxId = executeStatementForSingleValue(connection, Integer.class, "SELECT MAX(id) FROM " + tableName); Integer minId = executeStatementForSingleValue(connection, Integer.class, "SELECT MIN(id) FROM " + tableName); @@ -1340,6 +1463,7 @@ public class QueryManager // Logger.logDebug("For [" + tableName + "], using min id [" + idForGoal + "], which is from [" + foundUnitsAgo + "] Units[" + unit + "] ago."); return (idForGoal); + */ } @@ -1349,6 +1473,8 @@ public class QueryManager *******************************************************************************/ private static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, Integer minId, Integer maxId, ChronoUnit unit) throws SQLException { + throw (new NotImplementedException()); + /* Integer midId = minId + ((maxId - minId) / 2); if(midId.equals(minId) || midId.equals(maxId)) { @@ -1368,6 +1494,7 @@ public class QueryManager { return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, minId, midId, unit)); } + */ } @@ -1377,6 +1504,8 @@ public class QueryManager *******************************************************************************/ private static long getTimeUnitAgo(Connection connection, String tableName, String dateFieldName, Integer id, ChronoUnit unit) throws SQLException { + throw (new NotImplementedException()); + /* LocalDateTime now = LocalDateTime.now(); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1395,61 +1524,100 @@ public class QueryManager // System.out.println("Unit[" + unit + "]'s ago: " + diff); return diff; } + */ + } + + /******************************************************************************* + ** + *******************************************************************************/ + // public static class TypeValuePair + // { + // private Class type; + // private T value; + + // /******************************************************************************* + // ** + // *******************************************************************************/ + // @SuppressWarnings("unchecked") + // public TypeValuePair(T value) + // { + // this.value = value; + // this.type = (Class) value.getClass(); + // } + + // /******************************************************************************* + // ** + // *******************************************************************************/ + // public TypeValuePair(Class type, T value) + // { + // this.type = type; + // this.value = value; + // } + + // /******************************************************************************* + // ** + // *******************************************************************************/ + // public T getValue() + // { + // return (value); + // } + + // /******************************************************************************* + // ** + // *******************************************************************************/ + // public Class getType() + // { + // return (type); + // } + + // } + + + + /******************************************************************************* + ** Setter for collectStatistics + ** + *******************************************************************************/ + public static void setCollectStatistics(boolean collectStatistics) + { + QueryManager.collectStatistics = collectStatistics; } /******************************************************************************* + ** Increment a statistic ** *******************************************************************************/ - public static class TypeValuePair + public static void incrementStatistic(String statName) { - private Class type; - private T value; - - - - /******************************************************************************* - ** - *******************************************************************************/ - @SuppressWarnings("unchecked") - public TypeValuePair(T value) + if(collectStatistics) { - this.value = value; - this.type = (Class) value.getClass(); + statistics.putIfAbsent(statName, 0); + statistics.put(statName, statistics.get(statName) + 1); } + } - /******************************************************************************* - ** - *******************************************************************************/ - public TypeValuePair(Class type, T value) - { - this.type = type; - this.value = value; - } + /******************************************************************************* + ** clear the map of statistics + ** + *******************************************************************************/ + public static void resetStatistics() + { + statistics.clear(); + } - /******************************************************************************* - ** - *******************************************************************************/ - public T getValue() - { - return (value); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public Class getType() - { - return (type); - } - + /******************************************************************************* + ** Getter for statistics + ** + *******************************************************************************/ + public static Map getStatistics() + { + return statistics; } } diff --git a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 627c7557..5425c3e1 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -52,6 +52,18 @@ public class RDBMSBackendMetaData extends QBackendMetaData + /******************************************************************************* + ** Fluent setter, override to help fluent flows + *******************************************************************************/ + @Override + public RDBMSBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + /******************************************************************************* ** Getter for vendor ** diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index 1e9596db..2953addc 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -58,8 +58,7 @@ public class TestUtils .withVendor("h2") .withHostName("mem") .withDatabaseName("test_database") - .withUsername("sa") - .withPassword(""); + .withUsername("sa"); rdbmsBackendMetaData.setName("default"); return (rdbmsBackendMetaData); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java index a70632bf..3c03a72a 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java @@ -45,16 +45,18 @@ public class RDBMSActionTest @SuppressWarnings("unchecked") protected void primeTestDatabase() throws Exception { - ConnectionManager connectionManager = new ConnectionManager(); - Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); - InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/prime-test-database.sql"); - assertNotNull(primeTestDatabaseSqlStream); - List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); - lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); - String joinedSQL = String.join("\n", lines); - for(String sql : joinedSQL.split(";")) + ConnectionManager connectionManager = new ConnectionManager(); + try(Connection connection = connectionManager.getConnection(TestUtils.defineBackend())) { - QueryManager.executeUpdate(connection, sql); + InputStream primeTestDatabaseSqlStream = RDBMSActionTest.class.getResourceAsStream("/prime-test-database.sql"); + assertNotNull(primeTestDatabaseSqlStream); + List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } } } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java new file mode 100644 index 00000000..62869e7e --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java @@ -0,0 +1,121 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.count.CountRequest; +import com.kingsrook.qqq.backend.core.model.actions.count.CountResult; +import com.kingsrook.qqq.backend.core.model.actions.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.query.QQueryFilter; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RDBMSCountActionTest extends RDBMSActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + super.primeTestDatabase(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredCount() throws QException + { + CountRequest countRequest = initCountRequest(); + CountResult countResult = new RDBMSCountAction().execute(countRequest); + Assertions.assertEquals(5, countResult.getCount(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQueryCount() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountRequest countRequest = initCountRequest(); + countRequest.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + CountResult countResult = new RDBMSCountAction().execute(countRequest); + Assertions.assertEquals(1, countResult.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountRequest countRequest = initCountRequest(); + countRequest.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + CountResult countResult = new RDBMSCountAction().execute(countRequest); + Assertions.assertEquals(4, countResult.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountRequest initCountRequest() + { + CountRequest countRequest = new CountRequest(); + countRequest.setInstance(TestUtils.defineInstance()); + countRequest.setTableName(TestUtils.defineTablePerson().getName()); + return countRequest; + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java index 852d7a51..386af624 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java @@ -22,7 +22,9 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; +import java.util.Collections; import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -50,6 +52,34 @@ public class RDBMSInsertActionTest extends RDBMSActionTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertNullList() throws QException + { + InsertRequest insertRequest = initInsertRequest(); + insertRequest.setRecords(null); + InsertResult insertResult = new RDBMSInsertAction().execute(insertRequest); + assertEquals(0, insertResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertEmptyList() throws QException + { + InsertRequest insertRequest = initInsertRequest(); + insertRequest.setRecords(Collections.emptyList()); + InsertResult insertResult = new RDBMSInsertAction().execute(insertRequest); + assertEquals(0, insertResult.getRecords().size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java index 0cbe75a2..14114ddc 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java @@ -22,11 +22,17 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest; import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -47,6 +53,49 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); + + QueryManager.setCollectStatistics(true); + QueryManager.resetStatistics(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + public void afterEach() throws Exception + { + QueryManager.resetStatistics(); + QueryManager.setCollectStatistics(false); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateNullList() throws QException + { + UpdateRequest updateRequest = initUpdateRequest(); + updateRequest.setRecords(null); + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); + assertEquals(0, updateResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateEmptyList() throws QException + { + UpdateRequest updateRequest = initUpdateRequest(); + updateRequest.setRecords(Collections.emptyList()); + new RDBMSUpdateAction().execute(updateRequest); + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); + assertEquals(0, updateResult.getRecords().size()); } @@ -65,7 +114,11 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest .withValue("email", "jamestk@starfleet.net") .withValue("birthDate", "2210-05-20"); updateRequest.setRecords(List.of(record)); + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); + Map statistics = QueryManager.getStatistics(); + assertEquals(1, statistics.get(QueryManager.STAT_QUERIES_RAN)); + assertEquals(1, updateResult.getRecords().size(), "Should return 1 row"); assertEquals(2, updateResult.getRecords().get(0).getValue("id"), "Should have id=2 in the row"); // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); @@ -94,7 +147,7 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest ** *******************************************************************************/ @Test - public void testUpdateMany() throws Exception + public void testUpdateManyWithDifferentColumnsAndValues() throws Exception { UpdateRequest updateRequest = initUpdateRequest(); QRecord record1 = new QRecord().withTableName("person") @@ -108,11 +161,24 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest .withValue("firstName", "Wilt") .withValue("birthDate", null); - updateRequest.setRecords(List.of(record1, record2)); + QRecord record3 = new QRecord().withTableName("person") + .withValue("id", 5) + .withValue("firstName", "Richard") + .withValue("birthDate", null); + + updateRequest.setRecords(List.of(record1, record2, record3)); + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); - assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); + + // this test runs one batch and one regular query + Map statistics = QueryManager.getStatistics(); + assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); + assertEquals(1, statistics.get(QueryManager.STAT_QUERIES_RAN)); + + assertEquals(3, updateResult.getRecords().size(), "Should return 3 rows"); assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + assertEquals(5, updateResult.getRecords().get(2).getValue("id"), "Should have expected ids in the row"); // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> { int rowsFound = 0; @@ -137,6 +203,110 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest } assertEquals(1, rowsFound); })); + runTestSql("SELECT * FROM person WHERE last_name = 'Richardson'", (rs -> { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(5, rs.getInt("id")); + assertEquals("Richard", rs.getString("first_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsDifferentValues() throws Exception + { + UpdateRequest updateRequest = initUpdateRequest(); + QRecord record1 = new QRecord().withTableName("person") + .withValue("id", 1) + .withValue("firstName", "Darren") + .withValue("lastName", "From Bewitched") + .withValue("birthDate", "1900-01-01"); + + QRecord record2 = new QRecord().withTableName("person") + .withValue("id", 3) + .withValue("firstName", "Wilt") + .withValue("lastName", "Tim's Uncle") + .withValue("birthDate", null); + + updateRequest.setRecords(List.of(record1, record2)); + + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); + Map statistics = QueryManager.getStatistics(); + assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); + + assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); + assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); + assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(1, rs.getInt("id")); + assertEquals("Darren", rs.getString("first_name")); + assertEquals("From Bewitched", rs.getString("last_name")); + assertEquals("1900-01-01", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Tim''s Uncle'", (rs -> { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(3, rs.getInt("id")); + assertEquals("Wilt", rs.getString("first_name")); + assertEquals("Tim's Uncle", rs.getString("last_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsSameValues() throws Exception + { + UpdateRequest updateRequest = initUpdateRequest(); + List records = new ArrayList<>(); + for(int i = 1; i <= 5; i++) + { + records.add(new QRecord().withTableName("person") + .withValue("id", i) + .withValue("birthDate", "1999-09-09")); + } + + updateRequest.setRecords(records); + + UpdateResult updateResult = new RDBMSUpdateAction().execute(updateRequest); + Map statistics = QueryManager.getStatistics(); + assertEquals(1, statistics.get(QueryManager.STAT_QUERIES_RAN)); + + assertEquals(5, updateResult.getRecords().size(), "Should return 5 rows"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE id <= 5", (rs -> { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals("1999-09-09", rs.getString("birth_date")); + } + assertEquals(5, rowsFound); + })); } diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java new file mode 100644 index 00000000..a51a041b --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManagerTest.java @@ -0,0 +1,113 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.jdbc; + + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.Collections; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for ConnectionManager + *******************************************************************************/ +@Disabled("This was okay for POC, but shouldn't run in CI") +class ConnectionManagerTest +{ + @Test + public void test() throws SQLException + { + Connection connection = new ConnectionManager().getConnection(getAuroraBacked()); + assertNotNull(connection); + + String sql = """ + insert into raw_parcel_invoice_line_ups (id, create_date, modify_date, parcel_invoice_line_id, account_country, account_number, account_split_payment_indicator, account_tax_id, alternate_invoice_amount, alternate_invoice_number, alternate_invoicing_currency_code, bol__number_1, bol__number_2, bol__number_3, bol__number_4, bol__number_5, basis_currency_code, basis_value, bill_option_code, + billed_weight, billed_weight_type, billed_weight_unit_of_measure, cccd_number, cpc_code, carrier_name_clinical_trial_identification_number__sds_id, charge_category_code, charge_category_detail_code, charge_classification_code, charge_description, charge_description_code, charge_source, charged_unit_quantity, class_number, contact_name, container_type, + corrected_zone, currency_variance_amount, customer_reference_number, customs_number, customs_office_name, cycle_date, cycle_number, declaration_number, declared_freight_class, detail_class, detail_keyed_billed_dimension, detail_keyed_billed_unit_of_measure, detail_keyed_dim, detail_keyed_unit_of_measure, direct_shipment_date, document_number, document_type, + duty_amount, duty_rate, duty_value, eft_date, eori_number, epu, entered_currency_code, entered_value, entered_weight, entered_weight_unit_of_measure, entry_date, entry_number, entry_port, entry_type, exchange_rate, excise_tax_amount, excise_tax_rate, export_place, foreign_trade_reference_number, freight_sequence_number, gst_amount, gst_rate, + goods_description, import_tax_id, incentive_amount, invoice_amount, invoice_currency_code, invoice_date, invoice_due_date, invoice_exchange_rate, invoice_level_charge, invoice_number, invoice_remit_amount, invoice_type_code, invoice_type_detail_code, item_quantity, item_quantity_unit_of_measure, job_number, lead_shipment_number, line_item_number, + master_air_waybill_number, miscellaneous_address_1_address_line_1, miscellaneous_address_1_address_line_2, miscellaneous_address_1_city, miscellaneous_address_1_company_name, miscellaneous_address_1_country, miscellaneous_address_1_name, miscellaneous_address_1_postal, miscellaneous_address_1_state, miscellaneous_address_2_address_line_1, + miscellaneous_address_2_address_line_2, miscellaneous_address_2_city, miscellaneous_address_2_company_name, miscellaneous_address_2_country, miscellaneous_address_2_name, miscellaneous_address_2_postal, miscellaneous_address_2_state, miscellaneous_address_qual_1, miscellaneous_address_qual_2, miscellaneous_currency_code, miscellaneous_incentive_amount, + miscellaneous_line_1, miscellaneous_line_10, miscellaneous_line_11, miscellaneous_line_2, miscellaneous_line_3, miscellaneous_line_4, miscellaneous_line_5, miscellaneous_line_7, miscellaneous_line_8, miscellaneous_line_9, miscellaneous_net_amount, nmfc, net_amount, office_number, order_in_council, origin_country, original_service_description, + original_shipment_package_quantity, original_tracking_number, other_amount, other_basis_amount, other_customs_number, other_customs_number_indicator, other_rate, oversize_quantity, po__number_1, po__number_10, po__number_2, po__number_3, po__number_4, po__number_5, po__number_6, po__number_7, po__number_8, po__number_9, package_dimension_unit_of_measure, + package_dimensions, package_quantity, package_reference_number_1, package_reference_number_2, package_reference_number_3, package_reference_number_4, package_reference_number_5, payer_role_code, pickup_record_number, place_holder_46, place_holder_47, place_holder_48, place_holder_52, place_holder_53, place_holder_54, place_holder_55, place_holder_56, + place_holder_57, place_holder_58, place_holder_59, promo_discount_alias, promo_discount_applied_indicator, raw_dimension_unit_of_measure, raw_dimensions, receiver_address_line_1, receiver_address_line_2, receiver_city, receiver_company_name, receiver_country, receiver_name, receiver_postal, receiver_state, recipient_number, scc_scale_weight, + sds_delivery_date, sds_error_code, sds_match_level_cd, sds_rnr_date, sima_access, scale_weight_unit_of_measure, scale_weight_quantity, sender_address_line_1, sender_address_line_2, sender_city, sender_company_name, sender_country, sender_name, sender_postal, sender_state, shipment_date, shipment_delivery_date, shipment_description, shipment_export_date, + shipment_import_date, shipment_reference_number_1, shipment_reference_number_2, shipment_release_date, shipment_value_amount, sold_to_address_line_1, sold_to_address_line_2, sold_to_city, sold_to_company_name, sold_to_country, sold_to_name, sold_to_postal, sold_to_state, store_number, tariff_code, tariff_rate, tariff_treatment_number, tax_indicator, + tax_type, tax_value, tax_variance_amount, tax_law_article_basis_amount, tax_law_article_number, third_party_address_line_1, third_party_address_line_2, third_party_city, third_party_company_name, third_party_country, third_party_name, third_party_postal, third_party_state, total_customs_amount, total_value_for_duty, tracking_number, + transaction_currency_code, transaction_date, transport_mode, type_code_1, type_code_2, type_detail_code_1, type_detail_code_2, type_detail_value_1, type_detail_value_2, unit_of_measure, vat_amount, vat_basis_amount, vat_rate, validation_date, version, weight, world_ease_number, zone, data_lake_id) + values + """; + + String values = """ + (0, '2022-06-13 13:07:52.0', '2022-06-13 13:07:52.0', null, 'US', '00000F2098', null, null, 0, null, null, null, null, null, null, null, null, 0.00, null, 0, null, null, null, null, + null, 'MIS', 'SVCH', 'FRT', 'Service Charge', null, null, 0, null, null, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0, 0, 0, null, null, null, null, 0, + 0, null, null, null, null, null, 0, 0, 0, null, null, 0, 0, 0, null, null, 0.00, 36, 'USD', '2022-01-01', '2022-01-10', 0, 0, '0000000F2098012', null, 'E', null, 0, null, null, null, + 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, null, null, null, null, 0, null, 36.00, null, null, null, null, 0, null, 0, + 0, null, null, 0, 0, null, null, null, null, null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, '00000F2098', 0, null, null, null, null, 0, null, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, null, null, null, null, + 0, null, null, null, 0, 0, 0, null, null, null, null, null, null, null, null, null, 0, 0, null, 'USD', '2022-01-01', null, null, null, null, null, null, null, null, 0, 0, 0, null, 2, null, null, null, 'XXXXXX') + """; + sql += String.join(",", Collections.nCopies(1000, values)); + + for(int i = 0; i < 10; i++) + { + System.out.println("== Cycle " + i); + QueryManager.executeUpdate(connection, "BEGIN WORK"); + System.out.println("Begin work..."); + Integer insertCount = QueryManager.executeUpdateForRowCount(connection, sql); + System.out.println("Inserted: " + insertCount); + Integer deleteCount = QueryManager.executeUpdateForRowCount(connection, "DELETE from raw_parcel_invoice_line_ups WHERE data_lake_id='XXXXXX'"); + System.out.println("Deleted: " + deleteCount); + + boolean commit = true; + if(commit) + { + QueryManager.executeUpdate(connection, "COMMIT WORK"); + System.out.println("Commit."); + } + else + { + QueryManager.executeUpdate(connection, "ROLLBACK WORK"); + System.out.println("Rollback."); + } + } + } + + + + private RDBMSBackendMetaData getAuroraBacked() + { + return new RDBMSBackendMetaData() + .withName("aurora-test") + .withVendor("aurora") + .withHostName("nf-one-development-aurora.cwuhqcx1inwx.us-east-2.rds.amazonaws.com") + .withPort(3306) + .withDatabaseName("nutrifresh_one") + .withUsername("nf_admin") + .withPassword("%!2rwcH+fb#WgPg"); + } +} diff --git a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaDataTest.java b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaDataTest.java index 028e8081..83d2d3db 100644 --- a/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaDataTest.java +++ b/src/test/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaDataTest.java @@ -49,7 +49,7 @@ class RDBMSBackendMetaDataTest System.out.println(JsonUtils.prettyPrint(json)); System.out.println(json); String expectToContain = """ - "backends":{"default":{"hostName":"mem","password":"","databaseName":"test_database\""""; + "backends":{"default":{"hostName":"mem","databaseName":"test_database\""""; assertTrue(json.contains(expectToContain)); }