Merge branch 'dev' into feature/CTLE-434-oms-update-business-logic

# Conflicts:
#	qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/async/AsyncJobManager.java

@@ -33,11 +33,13 @@ import java.util.concurrent.TimeoutException;
 import com.kingsrook.qqq.backend.core.context.CapturedContext;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
 import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
 import com.kingsrook.qqq.backend.core.state.StateType;
 import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
+import org.apache.logging.log4j.Level;
 import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
 
 
@@ -151,7 +153,11 @@ public class AsyncJobManager
          asyncJobStatus.setState(AsyncJobState.ERROR);
          asyncJobStatus.setCaughtException(e);
          getStateProvider().put(uuidAndTypeStateKey, asyncJobStatus);
-         LOG.warn("Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
+
+         //////////////////////////////////////////////////////
+         // if user facing, just log an info, warn otherwise //
+         //////////////////////////////////////////////////////
+         LOG.log((e instanceof QUserFacingException) ? Level.INFO : Level.WARN, "Job ended with an exception", e, logPair("jobId", uuidAndTypeStateKey.getUuid()));
          throw (new CompletionException(e));
       }
       finally

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java

@@ -116,7 +116,7 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
       List<String> errors = insertOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in insertAction", logPair("tableName", table.getName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
       //////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/UpdateAction.java

@@ -119,7 +119,7 @@ public class UpdateAction
       List<String> errors = updateOutput.getRecords().stream().flatMap(r -> r.getErrors().stream().map(Object::toString)).toList();
       if(CollectionUtils.nullSafeHasContents(errors))
       {
-         LOG.warn("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
+         LOG.info("Errors in updateAction", logPair("tableName", updateInput.getTableName()), logPair("errorCount", errors.size()), errors.size() < 10 ? logPair("errors", errors) : logPair("first10Errors", errors.subList(0, 10)));
       }
 
       /////////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java

@@ -137,6 +137,16 @@ public class QLogger
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void log(Level level, String message, Throwable t, LogPair... logPairs)
+   {
+      logger.log(level, makeJsonString(message, t, logPairs));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/JsonUtils.java

@@ -28,6 +28,7 @@ import java.time.LocalDate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -62,11 +63,29 @@ public class JsonUtils
     **
     *******************************************************************************/
    public static String toJson(Object object)
+   {
+      return (toJson(object, null));
+   }
+
+
+
+   /*******************************************************************************
+    ** Serialize any object into a JSON String - with customizations on the Jackson
+    ** ObjectMapper.
+    **
+    ** Internally using jackson - so jackson annotations apply!
+    **
+    *******************************************************************************/
+   public static String toJson(Object object, Consumer<ObjectMapper> objectMapperCustomizer)
    {
       try
       {
-         ObjectMapper mapper     = newObjectMapper();
+         ObjectMapper mapper = newObjectMapper();
-         String       jsonResult = mapper.writeValueAsString(object);
+         if(objectMapperCustomizer != null)
+         {
+            objectMapperCustomizer.accept(mapper);
+         }
+         String jsonResult = mapper.writeValueAsString(object);
          return (jsonResult);
       }
       catch(JsonProcessingException e)

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/JsonUtilsTest.java

@@ -28,6 +28,7 @@ import java.math.BigDecimal;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.kingsrook.qqq.backend.core.BaseTest;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
@@ -38,6 +39,7 @@ import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -68,6 +70,24 @@ class JsonUtilsTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   public void test_toJsonQRecordInputWithNullValues()
+   {
+      QRecord qRecord = getQRecord();
+      String json = JsonUtils.toJson(qRecord, objectMapper ->
+      {
+         objectMapper.setSerializationInclusion(JsonInclude.Include.ALWAYS);
+      });
+      
+      assertThat(json).contains("""
+         "values":{"foo":"Foo","bar":3.14159,"baz":null}""");
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -213,6 +233,7 @@ class JsonUtilsTest extends BaseTest
       qRecord.setValues(values);
       values.put("foo", "Foo");
       values.put("bar", new BigDecimal("3.14159"));
+      values.put("baz", null);
       return qRecord;
    }
 

qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java

@@ -517,18 +517,28 @@ public class BaseAPIActionUtil
 
       int    statusCode   = response.getStatusCode();
       String resultString = response.getContent();
-      String errorMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString;
-      LOG.error("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "...")));
 
+      boolean didLog = false;
       if("GET".equals(request.getMethod()))
       {
          if(statusCode == HttpStatus.SC_NOT_FOUND)
          {
             return;
          }
+         else if(statusCode == HttpStatus.SC_BAD_GATEWAY)
+         {
+            LOG.info("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "...")));
+            didLog = true;
+         }
       }
 
-      throw (new QException(errorMessage));
+      if(!didLog)
+      {
+         LOG.warn("HTTP " + request.getMethod() + " failed", logPair("table", table.getName()), logPair("statusCode", statusCode), logPair("responseContent", StringUtils.safeTruncate(resultString, 1024, "...")));
+      }
+
+      String warningMessage = "HTTP " + request.getMethod() + " for table [" + table.getName() + "] failed with status " + statusCode + ": " + resultString;
+      throw (new QException(warningMessage));
    }
 
 

qqq-dev-tools/bin/get-latest-snapshot.sh

@@ -97,7 +97,7 @@ else
 
    artifact=$1
    version=$2
-   if [ "$version" == "-l" ]; then
+   if [ "$version" == "-s" ]; then
       useSlug=$(checkForBranchBuild $artifact)
       if [ "$useSlug" == "1" ]; then
          version=$SLUG
@@ -105,9 +105,11 @@ else
          version=$CURRENT_VERSION
       fi
       echo "Using $version for $artifact" >&2
+   elif [ "$version" == "-l" ]; then
+      version=$CURRENT_VERSION
    fi
 
-   if [ -z "$ar^tifact" -o -z "$version" ]; then
+   if [ -z "$artifact" -o -z "$version" ]; then
       echo "Usage: $0 artifact snapshot-version-prefix"
       echo "   or: $0 artifact -l (latest of CURRENT_VERSION, or branch-slug, if it has been deployed)"
       echo "   or: $0 -i (interactive mode)"

qqq-dev-tools/bin/update-all-qqq-deps.sh

@@ -13,9 +13,14 @@ fi
 CURRENT_VERSION="$(cat $QQQ_DEV_TOOLS_DIR/CURRENT-SNAPSHOT-VERSION)"
 MODULE_LIST_FILE=$QQQ_DEV_TOOLS_DIR/MODULE_LIST
 
+getLatestSnapshotArg="-l"
+if [[ "$1" == "snapshot-BRANCH_SLUG" ]]; then
+   getLatestSnapshotArg="-s"
+fi
+
 for module in $(cat $MODULE_LIST_FILE); do
    echo "Updating $module..."
-   version=$(get-latest-snapshot.sh $module -l)
+   version=$(get-latest-snapshot.sh $module $getLatestSnapshotArg)
    update-dep.sh $module $version -q
 done
 

qqq-middleware-api/src/main/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandler.java

@@ -33,6 +33,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.kingsrook.qqq.api.actions.ApiImplementation;
 import com.kingsrook.qqq.api.actions.GenerateOpenApiSpecAction;
 import com.kingsrook.qqq.api.model.APILog;
@@ -297,7 +298,7 @@ public class QJavalinApiHandler
       }
 
       context.contentType(ContentType.APPLICATION_JSON);
-      context.result(JsonUtils.toJson(rs));
+      context.result(toJson(rs));
    }
 
 
@@ -312,7 +313,7 @@ public class QJavalinApiHandler
       rs.put("currentVersion", apiInstanceMetaData.getCurrentVersion().toString());
 
       context.contentType(ContentType.APPLICATION_JSON);
-      context.result(JsonUtils.toJson(rs));
+      context.result(toJson(rs));
    }
 
 
@@ -655,7 +656,7 @@ public class QJavalinApiHandler
          Map<String, Serializable> outputRecord = ApiImplementation.get(apiInstanceMetaData, version, tableApiName, primaryKey);
 
          QJavalinAccessLogger.logEndSuccess();
-         String resultString = JsonUtils.toJson(outputRecord);
+         String resultString = toJson(outputRecord);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -668,6 +669,21 @@ public class QJavalinApiHandler
 
 
 
+   /*******************************************************************************
+    ** Define standard way we'll make JSON objects for the API.
+    **
+    ** Specifically, changes QQQ's default to include null values
+    *******************************************************************************/
+   private static String toJson(Object object)
+   {
+      return JsonUtils.toJson(object, mapper ->
+      {
+         mapper.setSerializationInclusion(JsonInclude.Include.ALWAYS);
+      });
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -880,7 +896,7 @@ public class QJavalinApiHandler
          Map<String, Serializable> output = ApiImplementation.query(apiInstanceMetaData, version, tableApiName, context.queryParamMap());
 
          QJavalinAccessLogger.logEndSuccess(logPair("recordCount", () -> ((List<?>) output.get("records")).size()), QJavalinAccessLogger.logPairIfSlow("filter", filter, SLOW_LOG_THRESHOLD_MS));
-         String resultString = JsonUtils.toJson(output);
+         String resultString = toJson(output);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -911,7 +927,7 @@ public class QJavalinApiHandler
 
          QJavalinAccessLogger.logEndSuccess();
          context.status(HttpStatus.Code.CREATED.getCode());
-         String resultString = JsonUtils.toJson(outputRecord);
+         String resultString = toJson(outputRecord);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -941,7 +957,7 @@ public class QJavalinApiHandler
          List<Map<String, Serializable>> response = ApiImplementation.bulkInsert(apiInstanceMetaData, version, tableApiName, context.body());
 
          context.status(HttpStatus.Code.MULTI_STATUS.getCode());
-         String resultString = JsonUtils.toJson(response);
+         String resultString = toJson(response);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -972,7 +988,7 @@ public class QJavalinApiHandler
 
          QJavalinAccessLogger.logEndSuccess(logPair("recordCount", response.size()));
          context.status(HttpStatus.Code.MULTI_STATUS.getCode());
-         String resultString = JsonUtils.toJson(response);
+         String resultString = toJson(response);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -1003,7 +1019,7 @@ public class QJavalinApiHandler
 
          QJavalinAccessLogger.logEndSuccess(logPair("recordCount", response.size()));
          context.status(HttpStatus.Code.MULTI_STATUS.getCode());
-         String resultString = JsonUtils.toJson(response);
+         String resultString = toJson(response);
          context.result(resultString);
          storeApiLog(apiLog.withStatusCode(context.statusCode()).withResponseBody(resultString));
       }
@@ -1176,7 +1192,7 @@ public class QJavalinApiHandler
          ///////////////////////////
       }
 
-      String responseBody = JsonUtils.toJson(Map.of("error", errorMessage));
+      String responseBody = toJson(Map.of("error", errorMessage));
       context.result(responseBody);
 
       if(apiLog != null)

qqq-middleware-api/src/test/java/com/kingsrook/qqq/api/javalin/QJavalinApiHandlerTest.java

@@ -201,6 +201,8 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertTrue(jsonObject.isNull("noOfShoes"));
+      assertFalse(jsonObject.has("someNonField"));
    }
 
 
@@ -374,6 +376,8 @@ class QJavalinApiHandlerTest extends BaseTest
       assertEquals(1, jsonObject.getInt("id"));
       assertEquals("Homer", jsonObject.getString("firstName"));
       assertEquals("Simpson", jsonObject.getString("lastName"));
+      assertTrue(jsonObject.isNull("noOfShoes"));
+      assertFalse(jsonObject.has("someNonField"));
    }
 
 
@@ -938,42 +942,6 @@ class QJavalinApiHandlerTest extends BaseTest
 
 
 
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testUpdateAssociations() throws QException
-   {
-      insert1Order3Lines4LineExtrinsicsAnd1OrderExtrinsic();
-
-      HttpResponse<String> response = Unirest.patch(BASE_URL + "/api/" + VERSION + "/order/1")
-         .body("""
-            {"orderLines":
-               [
-                  {"id": 1, "lineNumber": 1, "sku": "BASIC1", "quantity": 47},
-                  {"id": 2},
-                  {"id": 3}
-               ]
-            }
-            """)
-         .asString();
-      assertErrorResponse(HttpStatus.NO_CONTENT_204, null, response);
-
-      QRecord       orderRecord = getRecord(TestUtils.TABLE_NAME_ORDER, 1);
-      List<QRecord> orderLines  = orderRecord.getAssociatedRecords().get("orderLines");
-      assertThat(orderLines)
-         .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty()
-         .withFailMessage("Should have a line with id 1").filteredOn(r -> r.getValueInteger("id").equals(1)).hasSize(1)
-         .withFailMessage("line with id 1 should have quantity updated to 47").first().matches(r -> r.getValue("quantity").equals(47));
-
-      assertThat(orderLines)
-         .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty()
-         .withFailMessage("Should have a line with id 2").filteredOn(r -> r.getValueInteger("id").equals(2)).hasSize(1)
-         .withFailMessage("line with id 2 should have original quantity (42)").first().matches(r -> r.getValue("quantity").equals(42));
-   }
-
-
-
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1025,6 +993,42 @@ class QJavalinApiHandlerTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testUpdateAssociations() throws QException
+   {
+      insert1Order3Lines4LineExtrinsicsAnd1OrderExtrinsic();
+
+      HttpResponse<String> response = Unirest.patch(BASE_URL + "/api/" + VERSION + "/order/1")
+         .body("""
+            {"orderLines":
+               [
+                  {"id": 1, "lineNumber": 1, "sku": "BASIC1", "quantity": 47},
+                  {"id": 2},
+                  {"id": 3}
+               ]
+            }
+            """)
+         .asString();
+      assertErrorResponse(HttpStatus.NO_CONTENT_204, null, response);
+
+      QRecord       orderRecord = getRecord(TestUtils.TABLE_NAME_ORDER, 1);
+      List<QRecord> orderLines  = orderRecord.getAssociatedRecords().get("orderLines");
+      assertThat(orderLines)
+         .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty()
+         .withFailMessage("Should have a line with id 1").filteredOn(r -> r.getValueInteger("id").equals(1)).hasSize(1)
+         .withFailMessage("line with id 1 should have quantity updated to 47").first().matches(r -> r.getValue("quantity").equals(47));
+
+      assertThat(orderLines)
+         .withFailMessage("order lines should be found on order").isNotNull().isNotEmpty()
+         .withFailMessage("Should have a line with id 2").filteredOn(r -> r.getValueInteger("id").equals(2)).hasSize(1)
+         .withFailMessage("line with id 2 should have original quantity (42)").first().matches(r -> r.getValue("quantity").equals(42));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -1055,7 +1059,7 @@ class QJavalinApiHandlerTest extends BaseTest
 
       assertEquals(HttpStatus.BAD_REQUEST_400, jsonArray.getJSONObject(2).getInt("statusCode"));
       assertEquals("Error updating Person: Missing value in primary key field", jsonArray.getJSONObject(2).getString("error"));
-      assertFalse(jsonArray.getJSONObject(2).has("id"));
+      assertTrue(jsonArray.getJSONObject(2).isNull("id"));
 
       assertEquals(HttpStatus.NOT_FOUND_404, jsonArray.getJSONObject(3).getInt("statusCode"));
       assertEquals("Error updating Person: No record was found to update for Id = 256", jsonArray.getJSONObject(3).getString("error"));