QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-14 update removeNonNumericValuesFromMappedRecords to handle commas (strip them); added test

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-30 07:41:58 -05:00
parent 253dfaf497
commit 845555030f
2 changed files with 146 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunction.java
View File

@ -22,6 +22,7 @@
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
@ -49,6 +50,11 @@ public class BasicETLTransformFunction implements FunctionBody
{
private static final Logger LOG = LogManager.getLogger(BasicETLTransformFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
@ -79,6 +85,9 @@ public class BasicETLTransformFunction implements FunctionBody
QTableMetaData table = runFunctionRequest.getInstance().getTable(tableName);
List<QRecord> mappedRecords = applyMapping(runFunctionRequest.getRecords(), table, keyBasedFieldMapping);
//////////////////////////////////////////////////////////////////////////////////////////////////////
// todo - should this be conditional, e.g., driven by a field, or an opt-in customization function? //
//////////////////////////////////////////////////////////////////////////////////////////////////////
removeNonNumericValuesFromMappedRecords(table, mappedRecords);
runFunctionResult.setRecords(mappedRecords);
@ -87,35 +96,77 @@ public class BasicETLTransformFunction implements FunctionBody
/*******************************************************************************
**
** Note: package-private for direct unit-testability
*******************************************************************************/
private void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
void removeNonNumericValuesFromMappedRecords(QTableMetaData table, List<QRecord> records)
{
for(QRecord record : records)
{
for(QFieldMetaData field : table.getFields().values())
{
Object value = record.getValue(field.getName());
Serializable value = record.getValue(field.getName());
if(value != null && StringUtils.hasContent(String.valueOf(value)))
{
record.setValue(field.getName(), tryToParseNumber(field.getType(), value));
}
}
}
}
/*******************************************************************************
**
*******************************************************************************/
private Serializable tryToParseNumber(QFieldType fieldType, Serializable value)
{
if(value == null)
{
//////////////////////////////
// null input = null output //
//////////////////////////////
return (null);
}
////////////////////////////////////////////////////
// get a string version of the value to work with //
////////////////////////////////////////////////////
String stringValue = String.valueOf(value);
try
{
if(field.getType().equals(QFieldType.INTEGER))
////////////////////////////////////////////////////////////////////////////////
// based on field type, try to parse - noting bad formatted values will throw //
////////////////////////////////////////////////////////////////////////////////
if(fieldType.equals(QFieldType.INTEGER))
{
Integer.parseInt(String.valueOf(value));
Integer.parseInt(stringValue);
}
else if(field.getType().equals(QFieldType.DECIMAL))
else if(fieldType.equals(QFieldType.DECIMAL))
{
new BigDecimal(String.valueOf(value));
new BigDecimal(stringValue);
}
//////////////////////////////////////////////////////////////////////////////////
// if we got through the parsing without throwing, then we can return the value //
//////////////////////////////////////////////////////////////////////////////////
return (value);
}
catch(NumberFormatException nfe)
{
LOG.info("Removing non-numeric value [" + value + "] from field [" + field.getName() + "]");
record.setValue(field.getName(), null);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
// upon number format exception, look for commas - if found, strip them out and try again (recursive call) //
/////////////////////////////////////////////////////////////////////////////////////////////////////////////
if(stringValue.contains(","))
{
stringValue = stringValue.replaceAll(",", "");
return (tryToParseNumber(fieldType, stringValue));
}
/////////////////////////////////////////////////////////////////////////////////
// else, if no comma, then we want a null value here, rather than a non-number //
/////////////////////////////////////////////////////////////////////////////////
LOG.debug("Nulling out value " + value + " that will not work in a " + fieldType + " field");
return (null);
}
}

76
src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLTransformFunctionTest.java Normal file
View File

@ -0,0 +1,76 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BasicETLTransformFunction
*******************************************************************************/
class BasicETLTransformFunctionTest
{
/*******************************************************************************
** Test the removeNonNumericValuesFromMappedRecords function
*******************************************************************************/
@Test
void testRemoveNonNumericValuesFromMappedRecords()
{
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, null));
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "foo"));
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1foo"));
assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1"));
assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000"));
assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.INTEGER, "1,000,000"));
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, null));
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "foo"));
assertNull(doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1foo"));
assertEquals("1", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1"));
assertEquals("1000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000"));
assertEquals("1.0", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1.0"));
assertEquals("1000.00", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000.00"));
assertEquals("1000000", doRemoveNonNumericValuesFromMappedRecords(QFieldType.DECIMAL, "1,000,000"));
}
private String doRemoveNonNumericValuesFromMappedRecords(QFieldType fieldType, String inputValue)
{
String field = "field";
QTableMetaData table = new QTableMetaData()
.withField(new QFieldMetaData(field, fieldType));
List<QRecord> records = List.of(new QRecord().withValue(field, inputValue));
new BasicETLTransformFunction().removeNonNumericValuesFromMappedRecords(table, records);
return (records.get(0).getValueString(field));
}
}