From 7491e5f819d0c40d9adf74116c48a96aced00fce Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 22 May 2023 16:05:34 -0500 Subject: [PATCH 1/5] api association fixes; mostly about propagating ids/fkeys, and having fields (in maps) as expected field types (cherry pick 74d003ed) --- .../core/actions/tables/InsertAction.java | 5 ++- .../core/actions/tables/QueryAction.java | 9 +++-- .../core/actions/tables/UpdateAction.java | 14 ++++++- .../ValidateRecordSecurityLockHelper.java | 11 ++++-- .../qqq/api/actions/QRecordApiAdapter.java | 5 ++- .../api/javalin/QJavalinApiHandlerTest.java | 37 +++++++++++++++++++ 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 8a532325..3b730336 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -52,6 +52,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -60,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -183,7 +185,8 @@ public class InsertAction extends AbstractQActionFunction queryOutputRecords) throws QException { - LOG.info("In manageAssociations for " + queryInput.getTableName() + " with " + queryOutputRecords.size() + " records"); QTableMetaData table = queryInput.getTable(); for(Association association : CollectionUtils.nonNullList(table.getAssociations())) { @@ -149,9 +149,10 @@ public class QueryAction Set values = new HashSet<>(); for(QRecord record : queryOutputRecords) { - Serializable value = record.getValue(joinOn.getLeftField()); - values.add(value); - outerResultMap.add(List.of(value), record); + Serializable value = record.getValue(joinOn.getLeftField()); + Serializable valueAsType = ValueUtils.getValueAsFieldType(table.getField(joinOn.getLeftField()).getType(), value); + values.add(valueAsType); + outerResultMap.add(List.of(valueAsType), record); } filter.addCriteria(new QFilterCriteria(joinOn.getRightField(), QCriteriaOperator.IN, new ArrayList<>(values))); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index 17663aa7..a02d223e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -296,7 +297,8 @@ public class UpdateAction ////////////////////////////////////////////////////////////////////////////////////////////////////////// for(JoinOn joinOn : join.getJoinOns()) { - associatedRecord.setValue(joinOn.getRightField(), record.getValue(joinOn.getLeftField())); + QFieldType type = table.getField(joinOn.getLeftField()).getType(); + associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField()))); } nextLevelInserts.add(associatedRecord); } @@ -307,6 +309,16 @@ public class UpdateAction /////////////////////////////////////////////////////////////////////////////// idsBeingUpdated.add(associatedId); nextLevelUpdates.add(associatedRecord); + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // make sure the child record being updated has its join fields populated (same as an insert). // + // this will make the next update action much happier // + ///////////////////////////////////////////////////////////////////////////////////////////////// + for(JoinOn joinOn : join.getJoinOns()) + { + QFieldType type = table.getField(joinOn.getLeftField()).getType(); + associatedRecord.setValue(joinOn.getRightField(), ValueUtils.getValueAsFieldType(type, record.getValue(joinOn.getLeftField()))); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java index ae44f8fe..94067332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/ValidateRecordSecurityLockHelper.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -113,9 +114,10 @@ public class ValidateRecordSecurityLockHelper //////////////////////////////////////////////////////////////////////////////////////////////////////////////// // else look for the joined record - if it isn't found, assume a fail - else validate security value if found // //////////////////////////////////////////////////////////////////////////////////////////////////////////////// - QJoinMetaData leftMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0)); - QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1)); - QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable()); + QJoinMetaData leftMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(0)); + QJoinMetaData rightMostJoin = QContext.getQInstance().getJoin(recordSecurityLock.getJoinNameChain().get(recordSecurityLock.getJoinNameChain().size() - 1)); + QTableMetaData rightMostJoinTable = QContext.getQInstance().getTable(rightMostJoin.getRightTable()); + QTableMetaData leftMostJoinTable = QContext.getQInstance().getTable(leftMostJoin.getLeftTable()); for(List inputRecordPage : CollectionUtils.getPages(records, 500)) { @@ -153,7 +155,8 @@ public class ValidateRecordSecurityLockHelper for(JoinOn joinOn : rightMostJoin.getJoinOns()) { - Serializable inputRecordValue = inputRecord.getValue(joinOn.getRightField()); + QFieldType type = rightMostJoinTable.getField(joinOn.getRightField()).getType(); + Serializable inputRecordValue = ValueUtils.getValueAsFieldType(type, inputRecord.getValue(joinOn.getRightField())); inputRecordJoinValues.add(inputRecordValue); subFilter.addCriteria(inputRecordValue == null diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java index 412a0ad5..687fe2aa 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/actions/QRecordApiAdapter.java @@ -183,7 +183,10 @@ public class QRecordApiAdapter { if(subObject instanceof JSONObject subJsonObject) { - QRecord subRecord = apiJsonObjectToQRecord(subJsonObject, association.getAssociatedTableName(), apiName, apiVersion, includePrimaryKey); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure to always include primary keys (boolean param) on calls for children - to help determine insert/update cases // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord subRecord = apiJsonObjectToQRecord(subJsonObject, association.getAssociatedTableName(), apiName, apiVersion, true); qRecord.withAssociatedRecord(association.getName(), subRecord); } else diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java index 34832bdb..3ea783a6 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java @@ -892,6 +892,42 @@ class QJavalinApiHandlerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testUpdateAssociations() throws QException + { + insert1Order3Lines4LineExtrinsicsAnd1OrderExtrinsic(); + + HttpResponse response = Unirest.patch(BASE_URL + "/api/" + VERSION + "/order/1") + .body(""" + {"orderLines": + [ + {"id": 1, "lineNumber": 1, "sku": "BASIC1", "quantity": 47}, + {"id": 2}, + {"id": 3} + ] + } + """) + .asString(); + assertErrorResponse(HttpStatus.NO_CONTENT_204, null, response); + + QRecord orderRecord = getRecord(TestUtils.TABLE_NAME_ORDER, 1); + List orderLines = orderRecord.getAssociatedRecords().get("orderLines"); + assertThat(orderLines) + .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty() + .withFailMessage("Should have a line with id 1").filteredOn(r -> r.getValueInteger("id").equals(1)).hasSize(1) + .withFailMessage("line with id 1 should have quantity updated to 47").first().matches(r -> r.getValue("quantity").equals(47)); + + assertThat(orderLines) + .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty() + .withFailMessage("Should have a line with id 2").filteredOn(r -> r.getValueInteger("id").equals(2)).hasSize(1) + .withFailMessage("line with id 2 should have original quantity (42)").first().matches(r -> r.getValue("quantity").equals(42)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -1252,6 +1288,7 @@ class QJavalinApiHandlerTest extends BaseTest GetInput getInput = new GetInput(); getInput.setTableName(tableName); getInput.setPrimaryKey(id); + getInput.setIncludeAssociations(true); GetOutput getOutput = new GetAction().execute(getInput); QRecord record = getOutput.getRecord(); return record; From de74455de710ab57ec692710ddd5bf36d76b3275 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 23 May 2023 10:11:54 -0500 Subject: [PATCH 2/5] more changes to help reduce warnings that should probably be infos in loggly --- .../core/actions/async/AsyncJobManager.java | 8 +++++++- .../core/actions/tables/InsertAction.java | 2 +- .../core/actions/tables/UpdateAction.java | 2 +- .../qqq/backend/core/logging/QLogger.java | 10 ++++++++++ .../module/api/actions/BaseAPIActionUtil.java | 19 ++++++++++++++++--- 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java index de4716b7..2c16afd0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java @@ -33,11 +33,13 @@ import java.util.concurrent.TimeoutException; import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider; import com.kingsrook.qqq.backend.core.state.StateProviderInterface; import com.kingsrook.qqq.backend.core.state.StateType; import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; +import org.apache.logging.log4j.Level; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -151,7 +153,11 @@ public class AsyncJobManager asyncJobStatus.setState(AsyncJobState.ERROR); asyncJobStatus.setCaughtException(e); getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus); - LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid())); + + ////////////////////////////////////////////////////// + // if user facing, just log an info, warn otherwise // + ////////////////////////////////////////////////////// + LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid())); throw (new CompletionException(e)); } finally diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 3b730336..1177bbe3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -111,7 +111,7 @@ public class InsertAction extends AbstractQActionFunction errors = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList(); if(CollectionUtils.nullSafeHasContents(errors)) { - LOG.warn("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10))); + LOG.info("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10))); } manageAssociations(table, insertOutput.getRecords(), insertInput.getTransaction()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java index a02d223e..79518f36 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java @@ -112,7 +112,7 @@ public class UpdateAction List errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream()).toList(); if(CollectionUtils.nullSafeHasContents(errors)) { - LOG.warn("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10))); + LOG.info("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10))); } manageAssociations(updateInput); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index 8e5a25bf..d057b8fb 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -137,6 +137,16 @@ public class QLogger + /******************************************************************************* + ** + *******************************************************************************/ + public void log(Level level, String message, Throwable t, LogPair... logPairs) + { + logger.log(level, makeJsonString(message, t, logPairs)); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index c2617519..bb764769 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -505,8 +505,20 @@ public class BaseAPIActionUtil int statusCode = response.getStatusCode(); String resultString = response.getContent(); - String errorMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; - LOG.error("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); + + if("GET".equals(request.getMethod())) + { + //////////////////////////////////////////////////////////////////////////////////////// + // bad gateways are not our fault and don't happen often, so just log an info on them // + //////////////////////////////////////////////////////////////////////////////////////// + if(statusCode == HttpStatus.SC_BAD_GATEWAY) + { + LOG.info("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); + return; + } + } + + LOG.warn("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); if("GET".equals(request.getMethod())) { @@ -516,7 +528,8 @@ public class BaseAPIActionUtil } } - throw (new QException(errorMessage)); + String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; + throw (new QException(warningMessage)); } From 76b102b8112e562e29b32a46baccf8e1ef866e28 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 24 May 2023 21:28:52 -0500 Subject: [PATCH 3/5] updated to still log info on api gateway error, but still throw exception --- .../module/api/actions/BaseAPIActionUtil.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index bb764769..7b8769e7 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -506,26 +506,23 @@ public class BaseAPIActionUtil int statusCode = response.getStatusCode(); String resultString = response.getContent(); - if("GET".equals(request.getMethod())) - { - //////////////////////////////////////////////////////////////////////////////////////// - // bad gateways are not our fault and don't happen often, so just log an info on them // - //////////////////////////////////////////////////////////////////////////////////////// - if(statusCode == HttpStatus.SC_BAD_GATEWAY) - { - LOG.info("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); - return; - } - } - - LOG.warn("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); - + boolean didLog = false; if("GET".equals(request.getMethod())) { if(statusCode == HttpStatus.SC_NOT_FOUND) { return; } + else if(statusCode == HttpStatus.SC_BAD_GATEWAY) + { + LOG.info("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); + didLog = true; + } + } + + if(!didLog) + { + LOG.warn("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "..."))); } String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString; From 515e04ecfe372253b00824156124aa4f74515c97 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 25 May 2023 10:22:04 -0500 Subject: [PATCH 4/5] Update api json mapping to include null & empty values --- .../qqq/backend/core/utils/JsonUtils.java | 23 +++++++++++-- .../qqq/backend/core/utils/JsonUtilsTest.java | 21 ++++++++++++ .../qqq/api/javalin/QJavalinApiHandler.java | 34 ++++++++++++++----- .../api/javalin/QJavalinApiHandlerTest.java | 6 +++- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java index 281b5620..ecaf07c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java @@ -28,6 +28,7 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -62,11 +63,29 @@ public class JsonUtils ** *******************************************************************************/ public static String toJson(Object object) + { + return (toJson(object, null)); + } + + + + /******************************************************************************* + ** Serialize any object into a JSON String - with customizations on the Jackson + ** ObjectMapper. + ** + ** Internally using jackson - so jackson annotations apply! + ** + *******************************************************************************/ + public static String toJson(Object object, Consumer objectMapperCustomizer) { try { - ObjectMapper mapper = newObjectMapper(); - String jsonResult = mapper.writeValueAsString(object); + ObjectMapper mapper = newObjectMapper(); + if(objectMapperCustomizer != null) + { + objectMapperCustomizer.accept(mapper); + } + String jsonResult = mapper.writeValueAsString(object); return (jsonResult); } catch(JsonProcessingException e) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/JsonUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/JsonUtilsTest.java index de9121b3..f049ad44 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/JsonUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/JsonUtilsTest.java @@ -28,6 +28,7 @@ import java.math.BigDecimal; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonInclude; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; @@ -38,6 +39,7 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -68,6 +70,24 @@ class JsonUtilsTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_toJsonQRecordInputWithNullValues() + { + QRecord qRecord = getQRecord(); + String json = JsonUtils.toJson(qRecord, objectMapper -> + { + objectMapper.setSerializationInclusion(JsonInclude.Include.ALWAYS); + }); + + assertThat(json).contains(""" + "values":{"foo":"Foo","bar":3.14159,"baz":null}"""); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -213,6 +233,7 @@ class JsonUtilsTest extends BaseTest qRecord.setValues(values); values.put("foo", "Foo"); values.put("bar", new BigDecimal("3.14159")); + values.put("baz", null); return qRecord; } diff --git a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java index 5bcd5f02..98484dce 100644 --- a/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java +++ b/qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonInclude; import com.kingsrook.qqq.api.actions.ApiImplementation; import com.kingsrook.qqq.api.actions.GenerateOpenApiSpecAction; import com.kingsrook.qqq.api.model.APILog; @@ -297,7 +298,7 @@ public class QJavalinApiHandler } context.contentType(ContentType.APPLICATION_JSON); - context.result(JsonUtils.toJson(rs)); + context.result(toJson(rs)); } @@ -312,7 +313,7 @@ public class QJavalinApiHandler rs.put("currentVersion", apiInstanceMetaData.getCurrentVersion().toString()); context.contentType(ContentType.APPLICATION_JSON); - context.result(JsonUtils.toJson(rs)); + context.result(toJson(rs)); } @@ -655,7 +656,7 @@ public class QJavalinApiHandler Map outputRecord = ApiImplementation.get(apiInstanceMetaData, version, tableApiName, primaryKey); QJavalinAccessLogger.logEndSuccess(); - String resultString = JsonUtils.toJson(outputRecord); + String resultString = toJson(outputRecord); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -668,6 +669,21 @@ public class QJavalinApiHandler + /******************************************************************************* + ** Define standard way we'll make JSON objects for the API. + ** + ** Specifically, changes QQQ's default to include null values + *******************************************************************************/ + private static String toJson(Object object) + { + return JsonUtils.toJson(object, mapper -> + { + mapper.setSerializationInclusion(JsonInclude.Include.ALWAYS); + }); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -880,7 +896,7 @@ public class QJavalinApiHandler Map output = ApiImplementation.query(apiInstanceMetaData, version, tableApiName, context.queryParamMap()); QJavalinAccessLogger.logEndSuccess(logPair("recordCount", () -> ((List) output.get("records")).size()), QJavalinAccessLogger.logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS)); - String resultString = JsonUtils.toJson(output); + String resultString = toJson(output); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -911,7 +927,7 @@ public class QJavalinApiHandler QJavalinAccessLogger.logEndSuccess(); context.status(HttpStatus.Code.CREATED.getCode()); - String resultString = JsonUtils.toJson(outputRecord); + String resultString = toJson(outputRecord); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -941,7 +957,7 @@ public class QJavalinApiHandler List> response = ApiImplementation.bulkInsert(apiInstanceMetaData, version, tableApiName, context.body()); context.status(HttpStatus.Code.MULTI_STATUS.getCode()); - String resultString = JsonUtils.toJson(response); + String resultString = toJson(response); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -972,7 +988,7 @@ public class QJavalinApiHandler QJavalinAccessLogger.logEndSuccess(logPair("recordCount", response.size())); context.status(HttpStatus.Code.MULTI_STATUS.getCode()); - String resultString = JsonUtils.toJson(response); + String resultString = toJson(response); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -1003,7 +1019,7 @@ public class QJavalinApiHandler QJavalinAccessLogger.logEndSuccess(logPair("recordCount", response.size())); context.status(HttpStatus.Code.MULTI_STATUS.getCode()); - String resultString = JsonUtils.toJson(response); + String resultString = toJson(response); context.result(resultString); storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString)); } @@ -1176,7 +1192,7 @@ public class QJavalinApiHandler /////////////////////////// } - String responseBody = JsonUtils.toJson(Map.of("error", errorMessage)); + String responseBody = toJson(Map.of("error", errorMessage)); context.result(responseBody); if(apiLog != null) diff --git a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java index 3ea783a6..6dc6f93f 100644 --- a/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java +++ b/qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java @@ -202,6 +202,8 @@ class QJavalinApiHandlerTest extends BaseTest assertEquals(1, jsonObject.getInt("id")); assertEquals("Homer", jsonObject.getString("firstName")); assertEquals("Simpson", jsonObject.getString("lastName")); + assertTrue(jsonObject.isNull("noOfShoes")); + assertFalse(jsonObject.has("someNonField")); } @@ -376,6 +378,8 @@ class QJavalinApiHandlerTest extends BaseTest assertEquals(1, jsonObject.getInt("id")); assertEquals("Homer", jsonObject.getString("firstName")); assertEquals("Simpson", jsonObject.getString("lastName")); + assertTrue(jsonObject.isNull("noOfShoes")); + assertFalse(jsonObject.has("someNonField")); } @@ -959,7 +963,7 @@ class QJavalinApiHandlerTest extends BaseTest assertEquals(HttpStatus.BAD_REQUEST_400, jsonArray.getJSONObject(2).getInt("statusCode")); assertEquals("Error updating Person: Missing value in primary key field", jsonArray.getJSONObject(2).getString("error")); - assertFalse(jsonArray.getJSONObject(2).has("id")); + assertTrue(jsonArray.getJSONObject(2).isNull("id")); assertEquals(HttpStatus.NOT_FOUND_404, jsonArray.getJSONObject(3).getInt("statusCode")); assertEquals("Error updating Person: No record was found to update for Id = 256", jsonArray.getJSONObject(3).getString("error")); From 518575885548b043cadfac219074a62b91327e13 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 25 May 2023 10:55:10 -0500 Subject: [PATCH 5/5] add check for "snapshot-BRANCH_SLUG" arg to update-all-qqq-deps.sh -- if given, then we look for branchy versions, else we use main versions [skip ci] --- qqq-dev-tools/bin/get-latest-snapshot.sh | 6 ++++-- qqq-dev-tools/bin/update-all-qqq-deps.sh | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/qqq-dev-tools/bin/get-latest-snapshot.sh b/qqq-dev-tools/bin/get-latest-snapshot.sh index 232842d5..b36e5ae4 100755 --- a/qqq-dev-tools/bin/get-latest-snapshot.sh +++ b/qqq-dev-tools/bin/get-latest-snapshot.sh @@ -97,7 +97,7 @@ else artifact=$1 version=$2 - if [ "$version" == "-l" ]; then + if [ "$version" == "-s" ]; then useSlug=$(checkForBranchBuild $artifact) if [ "$useSlug" == "1" ]; then version=$SLUG @@ -105,9 +105,11 @@ else version=$CURRENT_VERSION fi echo "Using $version for $artifact" >&2 + elif [ "$version" == "-l" ]; then + version=$CURRENT_VERSION fi - if [ -z "$ar^tifact" -o -z "$version" ]; then + if [ -z "$artifact" -o -z "$version" ]; then echo "Usage: $0 artifact snapshot-version-prefix" echo " or: $0 artifact -l (latest of CURRENT_VERSION, or branch-slug, if it has been deployed)" echo " or: $0 -i (interactive mode)" diff --git a/qqq-dev-tools/bin/update-all-qqq-deps.sh b/qqq-dev-tools/bin/update-all-qqq-deps.sh index 846eefeb..517e5a69 100755 --- a/qqq-dev-tools/bin/update-all-qqq-deps.sh +++ b/qqq-dev-tools/bin/update-all-qqq-deps.sh @@ -13,9 +13,14 @@ fi CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)" MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST +getLatestSnapshotArg="-l" +if [[ "$1" == "snapshot-BRANCH_SLUG" ]]; then + getLatestSnapshotArg="-s" +fi + for module in $(cat $MODULE_LIST_FILE); do echo "Updating $module..." - version=$(get-latest-snapshot.sh $module -l) + version=$(get-latest-snapshot.sh $module $getLatestSnapshotArg) update-dep.sh $module $version -q done