QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 do case-insensitive lookups of possible values by label

Browse Source
This commit is contained in:
darin.kelkhoff
2024-12-26 19:11:41 -06:00
parent a4499219c8
commit 7f67eda2e3
2 changed files with 52 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java
View File

@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.ListingHash;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.collections.CaseInsensitiveKeyMap;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@ -167,24 +168,57 @@ public class BulkLoadValueMapper
/***************************************************************************
** Given a listingHash of Strings from the bulk-load file, to QRecords,
** we will either:
** - make sure the value set for the field is valid in the PV type
** - or put an error in the record (leaving the original value from the file in the field)
**
** We'll do potentially 2 possible-value searches - the first "by id" -
** type-converting the input strings to the PV's id type. Then, if any
** values weren't found by id, a second search by "labels"... which might
** be a bit suspicious, e.g., if the PV has a multi-field label...
***************************************************************************/
private static void handlePossibleValues(QFieldMetaData field, ListingHash<String, QRecord> fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException
{
QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName());
////////////////////////////////////////////////////////////
// String from file -> List<QRecord> that have that value //
////////////////////////////////////////////////////////////
Set<String> values = fieldPossibleValueToRecordMap.keySet();
/////////////////////////////////////////////////////////////////////////////////
// String from file -> Integer (for example) - that is the type-converted //
// version of the PVS's idType (but before any lookups were done with that id) //
// e.g., "42" -> 42 //
// e.g., "SOME_CONST" -> "SOME_CONST" (for PVS w/ string ids) //
/////////////////////////////////////////////////////////////////////////////////
Map<String, Serializable> valuesToValueInPvsIdTypeMap = new HashMap<>();
Map<String, QPossibleValue<?>> valuesFound = new HashMap<>();
Set<Serializable> valuesNotFound = new HashSet<>();
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// String versions of EITHER ids or values found in searchPossibleValueSource call (depending on what was searched by) //
// e.g., "42" -> QPossibleValue(42, "Forty-two") (when searched by id) //
// e.g., "Forty-Two" -> QPossibleValue(42, "Forty-two") (when searched by label) //
// e.g., "SOME_CONST" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by id) //
// e.g., "Some Const" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by label) //
// goal being - file could have "42" or "Forty-Two" (or "forty two") and those would all map to QPossibleValue(42, "Forty-two") //
// or - file could have "SOME_CONST" or "Some Const" (or "some const") and those would all map to QPossibleValue("SOME_CONST", "Some Const") //
// this is also why using CaseInsensitiveKeyMap! //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CaseInsensitiveKeyMap<QPossibleValue<?>> valuesFoundAsStrings = new CaseInsensitiveKeyMap<>();
/////////////////////////////////////////////////////////
// String values (from file) that still need looked up //
/////////////////////////////////////////////////////////
Set<String> valuesNotFound = new HashSet<>();
////////////////////////////////////////////////////////
// do a search, trying to use all given values as ids //
////////////////////////////////////////////////////////
ArrayList<Serializable> idList = new ArrayList<>();
SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
ArrayList<Serializable> idList = new ArrayList<>();
for(String value : values)
{
Serializable valueInPvsIdType = value;
@ -202,7 +236,7 @@ public class BulkLoadValueMapper
valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType);
idList.add(valueInPvsIdType);
valuesNotFound.add(valueInPvsIdType);
valuesNotFound.add(value);
}
searchPossibleValueSourceInput.setIdList(idList);
@ -211,12 +245,12 @@ public class BulkLoadValueMapper
SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
////////////////////////////////////////////////////////////////////////////////////////////////////
// for each possible value found, remove it from the set of ones not-found, and store it as a hit //
// for each possible value found, store it as a hit, and remove it from the set of ones not-found //
////////////////////////////////////////////////////////////////////////////////////////////////////
for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
{
String valueAsString = ValueUtils.getValueAsString(possibleValue.getId());
valuesFound.put(valueAsString, possibleValue);
valuesFoundAsStrings.put(valueAsString, possibleValue);
valuesNotFound.remove(valueAsString);
}
@ -227,15 +261,14 @@ public class BulkLoadValueMapper
{
searchPossibleValueSourceInput = new SearchPossibleValueSourceInput();
searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName());
List<String> labelList = valuesNotFound.stream().map(ValueUtils::getValueAsString).toList();
searchPossibleValueSourceInput.setLabelList(labelList);
searchPossibleValueSourceInput.setLimit(valuesNotFound.size());
searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound));
searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); // todo - a little sus... leaves no room for any dupes, which, can they happen?
LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0)));
LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next()));
searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput);
for(QPossibleValue<?> possibleValue : searchPossibleValueSourceOutput.getResults())
{
valuesFound.put(possibleValue.getLabel(), possibleValue);
valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue);
valuesNotFound.remove(possibleValue.getLabel());
}
}
@ -251,9 +284,9 @@ public class BulkLoadValueMapper
for(QRecord record : entry.getValue())
{
if(valuesFound.containsKey(pvsIdAsString))
if(valuesFoundAsStrings.containsKey(pvsIdAsString))
{
record.setValue(field.getName(), valuesFound.get(pvsIdAsString).getId());
record.setValue(field.getName(), valuesFoundAsStrings.get(pvsIdAsString).getId());
}
else
{

4
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java
View File

@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
@ -148,6 +149,9 @@ class BulkLoadValueMapperTest extends BaseTest
testPossibleValue(new BigDecimal("1.0"), 1, false);
testPossibleValue("IL", 1, false);
BackendQueryFilterUtils.setCaseSensitive(true);
testPossibleValue("il", 1, false);
testPossibleValue(512, 512, true); // an id, but not in the PVS
testPossibleValue("USA", "USA", true);
testPossibleValue(true, true, true);