From 845555030faa17419ccd9d0615e14b0e1c1b280e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 30 Jun 2022 07:41:58 -0500 Subject: [PATCH] QQQ-14 update removeNonNumericValuesFromMappedRecords to handle commas (strip them); added test --- .../etl/basic/BasicETLTransformFunction.java | 89 +++++++++++++++---- .../basic/BasicETLTransformFunctionTest.java | 76 ++++++++++++++++ 2 files changed, 146 insertions(+), 19 deletions(-) create mode 100644 src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java diff --git a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java index 88bd9be5..4d19d2d3 100644 --- a/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java +++ b/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; +import java.io.Serializable; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -49,6 +50,11 @@ public class BasicETLTransformFunction implements FunctionBody { private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class); + + + /******************************************************************************* + ** + *******************************************************************************/ @Override public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException { @@ -79,6 +85,9 @@ public class BasicETLTransformFunction implements FunctionBody QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName); List mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping); + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // todo - should this be conditional, e.g., driven by a field, or an opt-in customization function? // + ////////////////////////////////////////////////////////////////////////////////////////////////////// removeNonNumericValuesFromMappedRecords(table, mappedRecords); runFunctionResult.setRecords(mappedRecords); @@ -87,33 +96,18 @@ public class BasicETLTransformFunction implements FunctionBody /******************************************************************************* - ** + ** Note: package-private for direct unit-testability *******************************************************************************/ - private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List records) + void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List records) { for(QRecord record : records) { for(QFieldMetaData field : table.getFields().values()) { - Object value = record.getValue(field.getName()); + Serializable value = record.getValue(field.getName()); if(value != null && StringUtils.hasContent(String.valueOf(value))) { - try - { - if(field.getType().equals(QFieldType.INTEGER)) - { - Integer.parseInt(String.valueOf(value)); - } - else if(field.getType().equals(QFieldType.DECIMAL)) - { - new BigDecimal(String.valueOf(value)); - } - } - catch(NumberFormatException nfe) - { - LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]"); - record.setValue(field.getName(), null); - } + record.setValue(field.getName(), tryToParseNumber(field.getType(), value)); } } } @@ -121,6 +115,63 @@ public class BasicETLTransformFunction implements FunctionBody + /******************************************************************************* + ** + *******************************************************************************/ + private Serializable tryToParseNumber(QFieldType fieldType, Serializable value) + { + if(value == null) + { + ////////////////////////////// + // null input = null output // + ////////////////////////////// + return (null); + } + + //////////////////////////////////////////////////// + // get a string version of the value to work with // + //////////////////////////////////////////////////// + String stringValue = String.valueOf(value); + try + { + //////////////////////////////////////////////////////////////////////////////// + // based on field type, try to parse - noting bad formatted values will throw // + //////////////////////////////////////////////////////////////////////////////// + if(fieldType.equals(QFieldType.INTEGER)) + { + Integer.parseInt(stringValue); + } + else if(fieldType.equals(QFieldType.DECIMAL)) + { + new BigDecimal(stringValue); + } + + ////////////////////////////////////////////////////////////////////////////////// + // if we got through the parsing without throwing, then we can return the value // + ////////////////////////////////////////////////////////////////////////////////// + return (value); + } + catch(NumberFormatException nfe) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // upon number format exception, look for commas - if found, strip them out and try again (recursive call) // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(stringValue.contains(",")) + { + stringValue = stringValue.replaceAll(",", ""); + return (tryToParseNumber(fieldType, stringValue)); + } + + ///////////////////////////////////////////////////////////////////////////////// + // else, if no comma, then we want a null value here, rather than a non-number // + ///////////////////////////////////////////////////////////////////////////////// + LOG.debug("Nulling out value " + value + " that will not work in a " + fieldType + " field"); + return (null); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java new file mode 100644 index 00000000..f517803f --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for BasicETLTransformFunction + *******************************************************************************/ +class BasicETLTransformFunctionTest +{ + + /******************************************************************************* + ** Test the removeNonNumericValuesFromMappedRecords function + *******************************************************************************/ + @Test + void testRemoveNonNumericValuesFromMappedRecords() + { + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, null)); + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "foo")); + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1foo")); + assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1")); + assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000")); + assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000,000")); + + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, null)); + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "foo")); + assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1foo")); + assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1")); + assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000")); + assertEquals("1.0", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1.0")); + assertEquals("1000.00", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000.00")); + assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000,000")); + } + + + + private String doRemoveNonNumericValuesFromMappedRecords(QFieldType fieldType, String inputValue) + { + String field = "field"; + QTableMetaData table = new QTableMetaData() + .withField(new QFieldMetaData(field, fieldType)); + + List records = List.of(new QRecord().withValue(field, inputValue)); + new BasicETLTransformFunction().removeNonNumericValuesFromMappedRecords(table, records); + return (records.get(0).getValueString(field)); + } +} \ No newline at end of file