QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-781 Add option to treat CSV headers as field names (rather than working with a table's fields)

Browse Source
This commit is contained in:
darin.kelkhoff
2024-01-08 12:30:43 -06:00
parent 93dcee9f61
commit 56a2099515
2 changed files with 139 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java
View File

@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFieldMapping;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -133,11 +134,17 @@ public class CsvToQRecordAdapter
CSVFormat.DEFAULT
.withFirstRecordAsHeader()
.withIgnoreHeaderCase()
.withIgnoreEmptyLines()
.withTrim());
List<String> headers = csvParser.getHeaderNames();
headers = makeHeadersUnique(headers);
////////////////////////////////////////
// used by csv-headers-as-field-names //
////////////////////////////////////////
Map<String, QFieldMetaData> csvHeaderFieldMapping = buildCsvHeaderFieldMappingIfNeeded(inputWrapper, headers);
Iterator<CSVRecord> csvIterator = csvParser.iterator();
int recordCount = 0;
while(csvIterator.hasNext())
@ -160,11 +167,27 @@ public class CsvToQRecordAdapter
QRecord qRecord = new QRecord();
try
{
for(QFieldMetaData field : table.getFields().values())
if(inputWrapper.getCsvHeadersAsFieldNames())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
/////////////////////////////////////////////////////////////////////////////////////////
// in csv-headers-as-field-names mode, don't mess with table, and don't do any mapping //
/////////////////////////////////////////////////////////////////////////////////////////
for(Map.Entry<String, String> entry : csvValues.entrySet())
{
setValue(inputWrapper, qRecord, csvHeaderFieldMapping.get(entry.getKey()), entry.getValue());
}
}
else
{
///////////////////////////////////////
// otherwise, fields come from table //
///////////////////////////////////////
for(QFieldMetaData field : table.getFields().values())
{
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
}
}
runRecordCustomizer(recordCustomizer, qRecord);
@ -247,6 +270,26 @@ public class CsvToQRecordAdapter
/*******************************************************************************
**
*******************************************************************************/
private Map<String, QFieldMetaData> buildCsvHeaderFieldMappingIfNeeded(InputWrapper inputWrapper, List<String> headers)
{
Map<String, QFieldMetaData> csvHeaderFieldMapping = null;
if(inputWrapper.getCsvHeadersAsFieldNames())
{
csvHeaderFieldMapping = new HashMap<>();
for(String header : headers)
{
header = adjustHeaderCase(header, inputWrapper);
csvHeaderFieldMapping.put(header, new QFieldMetaData(header, QFieldType.STRING));
}
}
return csvHeaderFieldMapping;
}
/*******************************************************************************
**
*******************************************************************************/
@ -376,7 +419,8 @@ public class CsvToQRecordAdapter
private Integer limit;
private boolean doCorrectValueTypes = false;
private boolean caseSensitiveHeaders = false;
private boolean caseSensitiveHeaders = false;
private boolean csvHeadersAsFieldNames = false;
@ -618,6 +662,40 @@ public class CsvToQRecordAdapter
/*******************************************************************************
** Getter for csvHeadersAsFieldNames
**
*******************************************************************************/
public boolean getCsvHeadersAsFieldNames()
{
return csvHeadersAsFieldNames;
}
/*******************************************************************************
** Setter for csvHeadersAsFieldNames
**
*******************************************************************************/
public void setCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
{
this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
}
/*******************************************************************************
** Fluent setter for csvHeadersAsFieldNames
**
*******************************************************************************/
public InputWrapper withCsvHeadersAsFieldNames(boolean csvHeadersAsFieldNames)
{
this.csvHeadersAsFieldNames = csvHeadersAsFieldNames;
return (this);
}
/*******************************************************************************
** Getter for doCorrectValueTypes
**

56
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java
View File

@ -465,4 +465,60 @@ class CsvToQRecordAdapterTest extends BaseTest
assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCsvHeadersAsFields() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
.withCsvHeadersAsFieldNames(true)
.withCaseSensitiveHeaders(true)
.withCsv("""
firstName,birthDate,favoriteShapeId
John,1980,1
Paul,1970-06-15,green
"""));
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
QRecord qRecord = qRecords.get(0);
assertEquals("John", qRecord.getValue("firstName"));
assertEquals("1980", qRecord.getValue("birthDate"));
assertEquals("1", qRecord.getValue("favoriteShapeId"));
qRecord = qRecords.get(1);
assertEquals("Paul", qRecord.getValue("firstName"));
assertEquals("1970-06-15", qRecord.getValue("birthDate"));
assertEquals("green", qRecord.getValue("favoriteShapeId"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCsvHeadersAsFieldsDuplicatedNames() throws QException
{
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
.withCsvHeadersAsFieldNames(true)
.withCaseSensitiveHeaders(true)
.withCsv("""
orderId,sku,sku
10001,BASIC1,BASIC2
"""));
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
QRecord qRecord = qRecords.get(0);
assertEquals("10001", qRecord.getValue("orderId"));
assertEquals("BASIC1", qRecord.getValue("sku"));
assertEquals("BASIC2", qRecord.getValue("sku 2"));
}
}