QQQ-14 update removeNonNumericValuesFromMappedRecords to handle commas (strip them); added test

src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
 
 
+import java.io.Serializable;
 import java.math.BigDecimal;
 import java.util.ArrayList;
 import java.util.List;
@@ -49,6 +50,11 @@ public class BasicETLTransformFunction implements FunctionBody
 {
    private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class);
 
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
    @Override
    public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
    {
@@ -79,6 +85,9 @@ public class BasicETLTransformFunction implements FunctionBody
       QTableMetaData table         = runFunctionRequest.getInstance().getTable(tableName);
       List<QRecord>  mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
 
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
+      // todo - should this be conditional, e.g., driven by a field, or an opt-in customization function? //
+      //////////////////////////////////////////////////////////////////////////////////////////////////////
       removeNonNumericValuesFromMappedRecords(table, mappedRecords);
 
       runFunctionResult.setRecords(mappedRecords);
@@ -87,33 +96,18 @@ public class BasicETLTransformFunction implements FunctionBody
 
 
    /*******************************************************************************
-    **
+    ** Note:  package-private for direct unit-testability
     *******************************************************************************/
-   private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
+   void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
    {
       for(QRecord record : records)
       {
          for(QFieldMetaData field : table.getFields().values())
          {
-            Object value = record.getValue(field.getName());
+            Serializable value = record.getValue(field.getName());
             if(value != null && StringUtils.hasContent(String.valueOf(value)))
             {
-               try
+               record.setValue(field.getName(), tryToParseNumber(field.getType(), value));
-               {
-                  if(field.getType().equals(QFieldType.INTEGER))
-                  {
-                     Integer.parseInt(String.valueOf(value));
-                  }
-                  else if(field.getType().equals(QFieldType.DECIMAL))
-                  {
-                     new BigDecimal(String.valueOf(value));
-                  }
-               }
-               catch(NumberFormatException nfe)
-               {
-                  LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]");
-                  record.setValue(field.getName(), null);
-               }
             }
          }
       }
@@ -121,6 +115,63 @@ public class BasicETLTransformFunction implements FunctionBody
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private Serializable tryToParseNumber(QFieldType fieldType, Serializable value)
+   {
+      if(value == null)
+      {
+         //////////////////////////////
+         // null input = null output //
+         //////////////////////////////
+         return (null);
+      }
+
+      ////////////////////////////////////////////////////
+      // get a string version of the value to work with //
+      ////////////////////////////////////////////////////
+      String stringValue = String.valueOf(value);
+      try
+      {
+         ////////////////////////////////////////////////////////////////////////////////
+         // based on field type, try to parse - noting bad formatted values will throw //
+         ////////////////////////////////////////////////////////////////////////////////
+         if(fieldType.equals(QFieldType.INTEGER))
+         {
+            Integer.parseInt(stringValue);
+         }
+         else if(fieldType.equals(QFieldType.DECIMAL))
+         {
+            new BigDecimal(stringValue);
+         }
+
+         //////////////////////////////////////////////////////////////////////////////////
+         // if we got through the parsing without throwing, then we can return the value //
+         //////////////////////////////////////////////////////////////////////////////////
+         return (value);
+      }
+      catch(NumberFormatException nfe)
+      {
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         // upon number format exception, look for commas - if found, strip them out and try again (recursive call) //
+         /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+         if(stringValue.contains(","))
+         {
+            stringValue = stringValue.replaceAll(",", "");
+            return (tryToParseNumber(fieldType, stringValue));
+         }
+
+         /////////////////////////////////////////////////////////////////////////////////
+         // else, if no comma, then we want a null value here, rather than a non-number //
+         /////////////////////////////////////////////////////////////////////////////////
+         LOG.debug("Nulling out value " + value + " that will not work in a " + fieldType + " field");
+         return (null);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java

@@ -0,0 +1,76 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2022.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
+import org.junit.jupiter.api.Test;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+
+/*******************************************************************************
+ ** Unit test for BasicETLTransformFunction
+ *******************************************************************************/
+class BasicETLTransformFunctionTest
+{
+
+   /*******************************************************************************
+    ** Test the removeNonNumericValuesFromMappedRecords function
+    *******************************************************************************/
+   @Test
+   void testRemoveNonNumericValuesFromMappedRecords()
+   {
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, null));
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "foo"));
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1foo"));
+      assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1"));
+      assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000"));
+      assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000,000"));
+
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, null));
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "foo"));
+      assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1foo"));
+      assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1"));
+      assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000"));
+      assertEquals("1.0", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1.0"));
+      assertEquals("1000.00", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000.00"));
+      assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000,000"));
+   }
+
+
+
+   private String doRemoveNonNumericValuesFromMappedRecords(QFieldType fieldType, String inputValue)
+   {
+      String field = "field";
+      QTableMetaData table = new QTableMetaData()
+         .withField(new QFieldMetaData(field, fieldType));
+
+      List<QRecord> records = List.of(new QRecord().withValue(field, inputValue));
+      new BasicETLTransformFunction().removeNonNumericValuesFromMappedRecords(table, records);
+      return (records.get(0).getValueString(field));
+   }
+}