QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-23 23:48:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 - Put rows & rowNos in backend details during bulk-load. assert about those. also add tests (and fixes to mapping) for no-header use-cases

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-27 12:13:15 -06:00
parent 17fc976877
commit 6ed9dfd498
8 changed files with 464 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java Normal file
View File

@ -0,0 +1,112 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
/*******************************************************************************
** Utility methods for working with records in a bulk-load.
**
** Originally added for working with backendDetails around the source rows.
*******************************************************************************/
public class BulkLoadRecordUtils
{
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public static QRecord addBackendDetailsAboutFileRows(QRecord record, BulkLoadFileRow fileRow)
{
return (addBackendDetailsAboutFileRows(record, new ArrayList<>(List.of(fileRow))));
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public static QRecord addBackendDetailsAboutFileRows(QRecord record, ArrayList<BulkLoadFileRow> fileRows)
{
if(CollectionUtils.nullSafeHasContents(fileRows))
{
Integer firstRowNo = fileRows.get(0).getRowNo();
Integer lastRowNo = fileRows.get(fileRows.size() - 1).getRowNo();
if(Objects.equals(firstRowNo, lastRowNo))
{
record.addBackendDetail("rowNos", "Row " + firstRowNo);
}
else
{
record.addBackendDetail("rowNos", "Rows " + firstRowNo + "-" + lastRowNo);
}
}
else
{
record.addBackendDetail("rowNos", "Rows ?");
}
record.addBackendDetail("fileRows", fileRows);
return (record);
}
/***************************************************************************
**
***************************************************************************/
public static String getRowNosString(QRecord record)
{
return (record.getBackendDetailString("rowNos"));
}
/***************************************************************************
**
***************************************************************************/
@SuppressWarnings("unchecked")
public static ArrayList<BulkLoadFileRow> getFileRows(QRecord record)
{
return (ArrayList<BulkLoadFileRow>) record.getBackendDetail("fileRows");
}
/***************************************************************************
**
***************************************************************************/
public static List<Integer> getFileRowNos(QRecord record)
{
return (getFileRows(record).stream().map(row -> row.getRowNo()).toList());
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java
View File

@ -60,6 +60,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row);
for(QFieldMetaData field : table.getFields().values())
{

25
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java
View File

@ -29,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.google.gson.reflect.TypeToken;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@ -67,8 +68,8 @@ public class TallRowsToRecord implements RowsToRecordInterface
List<QRecord> rs = new ArrayList<>();
List<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
ArrayList<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
String associationNameChain = "";
@ -82,6 +83,9 @@ public class TallRowsToRecord implements RowsToRecordInterface
groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null);
}
////////////////////////
// this is suspect... //
////////////////////////
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
@ -108,18 +112,19 @@ public class TallRowsToRecord implements RowsToRecordInterface
//////////////////////////////////////////////////////////////
// not first, and not a match, so we can finish this record //
//////////////////////////////////////////////////////////////
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord);
rs.add(record);
//////////////////////////////////////////////////////////////////////////////////////////////////////
// we need to push this row back onto the fileToRows object, so it'll be handled in the next record //
//////////////////////////////////////////////////////////////////////////////////////////////////////
fileToRowsInterface.unNext();
////////////////////////////////////////
// reset these record-specific values //
////////////////////////////////////////
rowsForCurrentRecord = new ArrayList<>();
recordGroupByValues = null;
//////////////////////////////////////////////////////////////////////////////////////////////////////
// we need to push this row back onto the fileToRows object, so it'll be handled in the next record //
//////////////////////////////////////////////////////////////////////////////////////////////////////
fileToRowsInterface.unNext();
}
}
@ -129,7 +134,8 @@ public class TallRowsToRecord implements RowsToRecordInterface
/////////////////////////////////////////////////////////////////////////////////////////////////
if(!rowsForCurrentRecord.isEmpty())
{
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord);
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping, table);
@ -156,6 +162,7 @@ public class TallRowsToRecord implements RowsToRecordInterface
private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List<BulkLoadFileRow> rows) throws QException
{
QRecord record = new QRecord();
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken<ArrayList<BulkLoadFileRow>>() {}));
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java
View File

@ -90,7 +90,9 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem
//////////////////////////////////////////////////////
// start by building the record with its own fields //
//////////////////////////////////////////////////////
QRecord record = new QRecord();
QRecord record = new QRecord();
BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row);
boolean hadAnyValuesInRow = false;
for(QFieldMetaData field : table.getFields().values())
{

34
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java
View File

@ -157,7 +157,7 @@ public class BulkInsertMapping implements Serializable
}
else if(fieldNameToIndexMap != null)
{
return (fieldNameToIndexMap);
return (getFieldIndexesForNoHeaderUseCase(table, associationNameChain, wideAssociationIndexes));
}
throw (new QException("Mapping was not properly configured."));
@ -165,6 +165,38 @@ public class BulkInsertMapping implements Serializable
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
private Map<String, Integer> getFieldIndexesForNoHeaderUseCase(QTableMetaData table, String associationNameChain, List<Integer> wideAssociationIndexes)
{
Map<String, Integer> rs = new HashMap<>();
String wideAssociationSuffix = "";
if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes))
{
wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
Integer index = fieldNameToIndexMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix);
if(index != null)
{
rs.put(field.getName(), index);
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/