From a75530b466c0b78b59042be309e2f1b800746d31 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 30 May 2023 10:13:04 -0500 Subject: [PATCH] Updates for more heavy-field handling --- .../core/actions/audits/DMLAuditAction.java | 59 ++++-- .../core/actions/reporting/ExportAction.java | 15 +- .../core/actions/tables/QueryAction.java | 10 + .../core/instances/QInstanceEnricher.java | 2 +- .../qqq/backend/core/model/data/QRecord.java | 6 +- .../model/metadata/fields/FieldAdornment.java | 18 ++ .../model/metadata/fields/QFieldMetaData.java | 24 +++ .../columnstats/ColumnStatsStep.java | 5 + .../rdbms/actions/RDBMSQueryAction.java | 78 ++++++-- .../qqq/backend/module/rdbms/TestUtils.java | 1 + .../rdbms/actions/RDBMSQueryActionTest.java | 33 ++++ .../test/resources/prime-test-database.sql | 13 +- .../javalin/QJavalinImplementation.java | 178 ++++++++++++++---- 13 files changed, 362 insertions(+), 80 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java index 8c2f747e..b532fecd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/audits/DMLAuditAction.java @@ -187,32 +187,57 @@ public class DMLAuditAction extends AbstractQActionFunction(table.getFields().values()); } - ////////////////////////////////////////// - // add fields for possible value labels // - ////////////////////////////////////////// List returnList = new ArrayList<>(); for(QFieldMetaData field : fieldList) { + ///////////////////////////////////////////////////////////////////////////////////////// + // skip heavy fields. they aren't fetched, and we generally think we don't want them. // + ///////////////////////////////////////////////////////////////////////////////////////// + if(field.getIsHeavy()) + { + continue; + } + returnList.add(field); + + ////////////////////////////////////////// + // add fields for possible value labels // + ////////////////////////////////////////// if(StringUtils.hasContent(field.getPossibleValueSourceName())) { returnList.add(new QFieldMetaData(field.getName() + ":possibleValueLabel", QFieldType.STRING).withLabel(field.getLabel() + " Name")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java index 0bf8e33c..5ba0002b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java @@ -81,6 +81,16 @@ public class QueryAction { ActionHelper.validateSession(queryInput); + if(queryInput.getTableName() == null) + { + throw (new QException("Table name was not specified in query input")); + } + + if(queryInput.getTable() == null) + { + throw (new QException("A table named [" + queryInput.getTableName() + "] was not found in the active QInstance")); + } + postQueryRecordCustomizer = QCodeLoader.getTableCustomizer(AbstractPostQueryCustomizer.class, queryInput.getTable(), TableCustomizers.POST_QUERY_RECORD.getRole()); this.queryInput = queryInput; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 38213836..465434bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -735,7 +735,7 @@ public class QInstanceEnricher QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() .withName("upload") .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withIsRequired(true)) + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) .withComponent(new QFrontendComponentMetaData() .withType(QComponentType.HELP_TEXT) .withValue("previewText", "file upload instructions") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 00fbeb47..f826e6ff 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -69,7 +69,11 @@ public class QRecord implements Serializable private Map> associatedRecords = new HashMap<>(); - public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT = "jsonSourceObject"; + //////////////////////////////////////////////// + // well-known keys for the backendDetails map // + //////////////////////////////////////////////// + public static final String BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT = "jsonSourceObject"; // String of JSON + public static final String BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS = "heavyFieldLengths"; // Map diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java index 6bf68f48..2ca70af1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.io.Serializable; import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.utils.Pair; @@ -116,6 +118,22 @@ public class FieldAdornment + /******************************************************************************* + ** + *******************************************************************************/ + @JsonIgnore + public Optional getValue(String key) + { + if(key != null && values != null) + { + return (Optional.ofNullable(values.get(key))); + } + + return (Optional.empty()); + } + + + /******************************************************************************* ** Setter for values ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 298ffaf4..23f5b992 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.github.hervian.reflection.Fun; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -534,6 +535,29 @@ public class QFieldMetaData implements Cloneable + /******************************************************************************* + ** Getter for adornments + ** + *******************************************************************************/ + @JsonIgnore + public Optional getAdornment(AdornmentType adornmentType) + { + if(adornmentType != null && adornments != null) + { + for(FieldAdornment adornment : adornments) + { + if(adornmentType.equals(adornment.getType())) + { + return Optional.of((adornment)); + } + } + } + + return (Optional.empty()); + } + + + /******************************************************************************* ** Setter for adornments ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java index dee5e4d1..9cfe210e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/columnstats/ColumnStatsStep.java @@ -134,6 +134,11 @@ public class ColumnStatsStep implements BackendStep throw (new QException("Could not find field by name: " + fieldName)); } + if(field.getType().equals(QFieldType.BLOB)) + { + throw (new QException("Column stats are not supported for this field's data type.")); + } + //////////////////////////////////////////// // do a count query grouped by this field // //////////////////////////////////////////// diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index c82a18d4..968901b0 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -29,8 +29,10 @@ import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -43,8 +45,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; @@ -112,9 +116,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf //////////////////////////////////////////////////////////////////////////// // build the list of fields that will be processed in the result-set loop // //////////////////////////////////////////////////////////////////////////// - List fieldList = new ArrayList<>(table.getFields().values().stream() - .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields())) - .toList()); + List fieldList = new ArrayList<>(table.getFields().values().stream().toList()); for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryInput.getQueryJoins())) { if(queryJoin.getSelect()) @@ -123,10 +125,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf String tableNameOrAlias = queryJoin.getJoinTableOrItsAlias(); for(QFieldMetaData joinField : joinTable.getFields().values()) { - if(filterOutHeavyFieldsIfNeeded(joinField, queryInput.getShouldFetchHeavyFields())) - { - fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName())); - } + fieldList.add(joinField.clone().withName(tableNameOrAlias + "." + joinField.getName())); } } } @@ -153,9 +152,22 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf for(int i = 1; i <= metaData.getColumnCount(); i++) { - QFieldMetaData qFieldMetaData = fieldList.get(i - 1); - Serializable value = getFieldValueFromResultSet(qFieldMetaData, resultSet, i); - values.put(qFieldMetaData.getName(), value); + QFieldMetaData field = fieldList.get(i - 1); + + if(!queryInput.getShouldFetchHeavyFields() && field.getIsHeavy()) + { + /////////////////////////////////////////////////////////////////////////////////// + // if this is a non-fetched heavy field (e.g., we just fetched its length), then // + // get the value here as an INTEGER, not a BLOB or whatever the field would be // + /////////////////////////////////////////////////////////////////////////////////// + Serializable fieldLength = getFieldValueFromResultSet(QFieldType.INTEGER, resultSet, i); + setHeavyFieldLengthInRecordBackendDetails(record, field, fieldLength); + } + else + { + Serializable value = getFieldValueFromResultSet(field, resultSet, i); + values.put(field.getName(), value); + } } queryOutput.addRecord(record); @@ -195,6 +207,27 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static void setHeavyFieldLengthInRecordBackendDetails(QRecord record, QFieldMetaData field, Serializable fieldLength) + { + if(record.getBackendDetails() == null) + { + record.setBackendDetails(new HashMap<>()); + } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(field.getName(), fieldLength); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -210,8 +243,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List fieldList = new ArrayList<>(table.getFields().values()); String columns = fieldList.stream() - .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields())) - .map(field -> escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field))) + .map(field -> Pair.of(field, escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field)))) + .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields())) .collect(Collectors.joining(", ")); StringBuilder rs = new StringBuilder(clausePrefix).append(columns); @@ -229,7 +262,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf List joinFieldList = new ArrayList<>(joinTable.getFields().values()); String joinColumns = joinFieldList.stream() .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields())) - .map(field -> escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field))) + .map(field -> Pair.of(field, escapeIdentifier(tableNameOrAlias) + "." + escapeIdentifier(getColumnName(field)))) + .map(pair -> wrapHeavyFieldsWithLengthFunctionIfNeeded(pair, queryInput.getShouldFetchHeavyFields())) .collect(Collectors.joining(", ")); rs.append(", ").append(joinColumns); } @@ -254,6 +288,24 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf + /******************************************************************************* + ** if we're not fetching heavy fields, instead just get their length. this + ** method wraps the field 'sql name' (e.g., column_name or table_name.column_name) + ** with the LENGTH() function, if needed. + *******************************************************************************/ + private String wrapHeavyFieldsWithLengthFunctionIfNeeded(Pair fieldAndSqlName, boolean shouldFetchHeavyFields) + { + QFieldMetaData field = fieldAndSqlName.getA(); + String sqlName = fieldAndSqlName.getB(); + if(!shouldFetchHeavyFields && field.getIsHeavy()) + { + return ("LENGTH(" + sqlName + ")"); + } + return (sqlName); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java index a2479159..2a9b32f4 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java @@ -167,6 +167,7 @@ public class TestUtils .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary")) .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked")) + .withField(new QFieldMetaData("homeTown", QFieldType.STRING).withBackendName("home_town")) .withBackendDetails(new RDBMSTableBackendDetails() .withTableName("person")); } diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java index 3d49f858..4e49d82a 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java @@ -22,10 +22,12 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; +import java.io.Serializable; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -1663,4 +1665,35 @@ public class RDBMSQueryActionTest extends RDBMSActionTest .hasSize(1); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testHeavyFields() throws QException + { + ////////////////////////////////////////////////////////// + // set homeTown field as heavy - so it won't be fetched // + ////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON) + .getField("homeTown") + .withIsHeavy(true); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + List records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("No records should have the heavy homeTown field set").noneMatch(r -> r.getValue("homeTown") != null); + assertThat(records).describedAs("Some records should have a homeTown length backend detail set").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") != null); + assertThat(records).describedAs("Some records should have a null homeTown length backend").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") == null); + + ////////////////////////////////////////////// + // re-do the query, requesting heavy fields // + ////////////////////////////////////////////// + queryInput.setShouldFetchHeavyFields(true); + records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null); + + } + } diff --git a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql index 92c69194..87620a4c 100644 --- a/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql +++ b/qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql @@ -32,14 +32,15 @@ CREATE TABLE person email VARCHAR(250) NOT NULL, is_employed BOOLEAN, annual_salary DECIMAL(12,2), - days_worked INTEGER + days_worked INTEGER, + home_town VARCHAR(80) ); -INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27); -INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124); -INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0); -INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99); -INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0, 'Decatur'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99, 'Texas'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232, null); DROP TABLE IF EXISTS personal_id_card; CREATE TABLE personal_id_card diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java index 665047b3..77aaf73a 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinImplementation.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.javalin; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.io.Serializable; @@ -116,6 +117,7 @@ import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import io.javalin.Javalin; import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.Context; +import io.javalin.http.UploadedFile; import org.apache.commons.io.FileUtils; import org.eclipse.jetty.http.HttpStatus; import org.json.JSONArray; @@ -619,38 +621,17 @@ public class QJavalinImplementation updateInput.setTableName(table); PermissionsHelper.checkTablePermissionThrowing(updateInput, TablePermissionSubType.EDIT); + QTableMetaData tableMetaData = qInstance.getTable(table); + + QJavalinAccessLogger.logStart("update", logPair("table", table), logPair("primaryKey", primaryKey)); List recordList = new ArrayList<>(); QRecord record = new QRecord(); record.setTableName(table); recordList.add(record); - Map map = context.bodyAsClass(Map.class); - for(Map.Entry entry : map.entrySet()) - { - String fieldName = ValueUtils.getValueAsString(entry.getKey()); - Object value = entry.getValue(); - - if(StringUtils.hasContent(String.valueOf(value))) - { - record.setValue(fieldName, (Serializable) value); - } - else if("".equals(value)) - { - /////////////////////////////////////////////////////////////////////////////////////////////////// - // if frontend sent us an empty string - put a null in the record's value map. // - // this could potentially be changed to be type-specific (e.g., store an empty-string for STRING // - // fields, but null for INTEGER, etc) - but, who really wants empty-string in database anyway? // - /////////////////////////////////////////////////////////////////////////////////////////////////// - record.setValue(fieldName, null); - } - } - - QTableMetaData tableMetaData = qInstance.getTable(table); record.setValue(tableMetaData.getPrimaryKeyField(), primaryKey); - - QJavalinAccessLogger.logStart("update", logPair("table", table), logPair("primaryKey", primaryKey)); - + setRecordValuesForInsertOrUpdate(context, tableMetaData, record); updateInput.setRecords(recordList); UpdateAction updateAction = new UpdateAction(); @@ -668,6 +649,87 @@ public class QJavalinImplementation + /******************************************************************************* + ** + *******************************************************************************/ + private static void setRecordValuesForInsertOrUpdate(Context context, QTableMetaData tableMetaData, QRecord record) throws IOException + { + ///////////////////////// + // process form params // + ///////////////////////// + for(Map.Entry> formParam : context.formParamMap().entrySet()) + { + String fieldName = formParam.getKey(); + List values = formParam.getValue(); + if(CollectionUtils.nullSafeHasContents(values)) + { + String value = values.get(0); + if(StringUtils.hasContent(value)) + { + record.setValue(fieldName, value); + } + else + { + record.setValue(fieldName, null); + } + } + else + { + // is this ever hit? + record.setValue(fieldName, null); + } + } + + //////////////////////////// + // process uploaded files // + //////////////////////////// + for(Map.Entry> entry : CollectionUtils.nonNullMap(context.uploadedFileMap()).entrySet()) + { + String fieldName = entry.getKey(); + List uploadedFiles = entry.getValue(); + if(uploadedFiles.size() > 0) + { + UploadedFile uploadedFile = uploadedFiles.get(0); + try(InputStream content = uploadedFile.content()) + { + record.setValue(fieldName, content.readAllBytes()); + } + + QFieldMetaData blobField = tableMetaData.getField(fieldName); + blobField.getAdornment(AdornmentType.FILE_DOWNLOAD).ifPresent(adornment -> + { + adornment.getValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD).ifPresent(fileNameFieldName -> + { + record.setValue(ValueUtils.getValueAsString(fileNameFieldName), uploadedFile.filename()); + }); + }); + } + } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the record has any blob fields, and we're clearing them out (present in the values list, and set to null), // + // and they have a file-name field associated with them, then also clear out that file-name field // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData field : tableMetaData.getFields().values()) + { + if(field.getType().equals(QFieldType.BLOB)) + { + field.getAdornment(AdornmentType.FILE_DOWNLOAD).ifPresent(adornment -> + { + adornment.getValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD).ifPresent(fileNameFieldName -> + { + if(record.getValues().containsKey(field.getName()) && record.getValue(field.getName()) == null) + { + record.setValue(ValueUtils.getValueAsString(fileNameFieldName), null); + } + }); + }); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -682,20 +744,13 @@ public class QJavalinImplementation QJavalinAccessLogger.logStart("insert", logPair("table", tableName)); PermissionsHelper.checkTablePermissionThrowing(insertInput, TablePermissionSubType.INSERT); + QTableMetaData tableMetaData = qInstance.getTable(tableName); List recordList = new ArrayList<>(); QRecord record = new QRecord(); record.setTableName(tableName); recordList.add(record); - - Map map = context.bodyAsClass(Map.class); - for(Map.Entry entry : map.entrySet()) - { - if(StringUtils.hasContent(String.valueOf(entry.getValue()))) - { - record.setValue(String.valueOf(entry.getKey()), (Serializable) entry.getValue()); - } - } + setRecordValuesForInsertOrUpdate(context, tableMetaData, record); insertInput.setRecords(recordList); InsertAction insertAction = new InsertAction(); @@ -703,14 +758,14 @@ public class QJavalinImplementation if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getErrors())) { - throw (new QUserFacingException("Error inserting " + qInstance.getTable(tableName).getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0))); + throw (new QUserFacingException("Error inserting " + tableMetaData.getLabel() + ": " + insertOutput.getRecords().get(0).getErrors().get(0))); } if(CollectionUtils.nullSafeHasContents(insertOutput.getRecords().get(0).getWarnings())) { - throw (new QUserFacingException("Warning inserting " + qInstance.getTable(tableName).getLabel() + ": " + insertOutput.getRecords().get(0).getWarnings().get(0))); + throw (new QUserFacingException("Warning inserting " + tableMetaData.getLabel() + ": " + insertOutput.getRecords().get(0).getWarnings().get(0))); } - QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (insertOutput.getRecords().get(0).getValue(qInstance.getTable(tableName).getPrimaryKeyField())))); + QJavalinAccessLogger.logEndSuccess(logPair("primaryKey", () -> (insertOutput.getRecords().get(0).getValue(tableMetaData.getPrimaryKeyField())))); context.result(JsonUtils.toJson(insertOutput)); } catch(Exception e) @@ -795,7 +850,7 @@ public class QJavalinImplementation // - tableLabel primaryKey fieldLabel // // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot) // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - Optional fileDownloadAdornment = field.getAdornments().stream().filter(a -> a.getType().equals(AdornmentType.FILE_DOWNLOAD)).findFirst(); + Optional fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD); Map adornmentValues = Collections.emptyMap(); if(fileDownloadAdornment.isPresent()) @@ -809,6 +864,11 @@ public class QJavalinImplementation for(QRecord record : records) { + if(!doesFieldHaveValue(field, record)) + { + continue; + } + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); String fileName = null; @@ -826,7 +886,8 @@ public class QJavalinImplementation { @SuppressWarnings("unchecked") // instance validation should make this safe! List fileNameFormatFields = (List) adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS); - fileName = QValueFormatter.formatStringWithValues(fileNameFormat, fileNameFormatFields); + List values = fileNameFormatFields.stream().map(f -> ValueUtils.getValueAsString(record.getValue(f))).toList(); + fileName = QValueFormatter.formatStringWithValues(fileNameFormat, values); } } @@ -855,6 +916,45 @@ public class QJavalinImplementation + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean doesFieldHaveValue(QFieldMetaData field, QRecord record) + { + boolean fieldHasValue = false; + + try + { + if(record.getValue(field.getName()) != null) + { + fieldHasValue = true; + } + else if(field.getIsHeavy()) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // heavy fields that weren't fetched - they should have a backend-detail specifying their length (or null if null) // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map heavyFieldLengths = (Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS); + if(heavyFieldLengths != null) + { + Integer fieldLength = ValueUtils.getValueAsInteger(heavyFieldLengths.get(field.getName())); + if(fieldLength != null && fieldLength > 0) + { + fieldHasValue = true; + } + } + } + } + catch(Exception e) + { + LOG.info("Error checking if field has value", e, logPair("fieldName", field.getName()), logPair("record", record)); + } + + return fieldHasValue; + } + + + /******************************************************************************* ** *******************************************************************************/