mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 05:01:07 +00:00
CE-604 Bulk load improvements - type handling (via csv adapter) and better errors
This commit is contained in:
@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVParser;
|
||||
import org.apache.commons.csv.CSVRecord;
|
||||
@ -156,14 +158,21 @@ public class CsvToQRecordAdapter
|
||||
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
QRecord qRecord = new QRecord();
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
try
|
||||
{
|
||||
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
|
||||
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
|
||||
qRecord.setValue(field.getName(), csvValues.get(fieldSource));
|
||||
}
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
|
||||
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
|
||||
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
|
||||
}
|
||||
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
|
||||
}
|
||||
addRecord(qRecord);
|
||||
|
||||
recordCount++;
|
||||
@ -202,13 +211,20 @@ public class CsvToQRecordAdapter
|
||||
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
QRecord qRecord = new QRecord();
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
try
|
||||
{
|
||||
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
|
||||
qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
|
||||
}
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
|
||||
setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex));
|
||||
}
|
||||
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
|
||||
}
|
||||
addRecord(qRecord);
|
||||
|
||||
recordCount++;
|
||||
@ -231,6 +247,23 @@ public class CsvToQRecordAdapter
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString)
|
||||
{
|
||||
if(inputWrapper.doCorrectValueTypes)
|
||||
{
|
||||
qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString));
|
||||
}
|
||||
else
|
||||
{
|
||||
qRecord.setValue(field.getName(), valueString);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -341,6 +374,7 @@ public class CsvToQRecordAdapter
|
||||
private AbstractQFieldMapping<?> mapping;
|
||||
private Consumer<QRecord> recordCustomizer;
|
||||
private Integer limit;
|
||||
private boolean doCorrectValueTypes = false;
|
||||
|
||||
private boolean caseSensitiveHeaders = false;
|
||||
|
||||
@ -582,6 +616,40 @@ public class CsvToQRecordAdapter
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public boolean getDoCorrectValueTypes()
|
||||
{
|
||||
return doCorrectValueTypes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setDoCorrectValueTypes(boolean doCorrectValueTypes)
|
||||
{
|
||||
this.doCorrectValueTypes = doCorrectValueTypes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes)
|
||||
{
|
||||
this.doCorrectValueTypes = doCorrectValueTypes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -82,6 +82,7 @@ public class BulkInsertExtractStep extends AbstractExtractStep
|
||||
.withRecordPipe(getRecordPipe())
|
||||
.withLimit(getLimit())
|
||||
.withCsv(new String(bytes))
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(runBackendStepInput.getInstance().getTable(tableName))
|
||||
.withMapping(mapping)
|
||||
.withRecordCustomizer((record) ->
|
||||
|
||||
@ -26,6 +26,7 @@ import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
@ -75,11 +76,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
** extension of ProcessSummaryLine for lines where a UniqueKey was violated,
|
||||
** where we'll collect a sample (or maybe all) of the values that broke the UK.
|
||||
*******************************************************************************/
|
||||
private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine
|
||||
{
|
||||
private List<String> sampleValues = new ArrayList<>();
|
||||
private Set<String> sampleValues = new LinkedHashSet<>();
|
||||
private boolean areThereMoreSampleValues = false;
|
||||
|
||||
|
||||
|
||||
@ -116,8 +119,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
int rowsInThisPage = runBackendStepInput.getRecords().size();
|
||||
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
|
||||
int rowsInThisPage = runBackendStepInput.getRecords().size();
|
||||
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
|
||||
@ -278,6 +281,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
{
|
||||
processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString());
|
||||
}
|
||||
else
|
||||
{
|
||||
processSummaryLineWithUKSampleValues.areThereMoreSampleValues = true;
|
||||
}
|
||||
foundDupe = true;
|
||||
break;
|
||||
}
|
||||
@ -325,13 +332,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue();
|
||||
|
||||
ukErrorSummary
|
||||
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values such as: "
|
||||
+ StringUtils.joinWithCommasAndAnd(ukErrorSummary.sampleValues))
|
||||
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
|
||||
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
|
||||
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
|
||||
|
||||
.withSingularFutureMessage(tableLabel + " record will not be")
|
||||
.withPluralFutureMessage(tableLabel + " records will not be")
|
||||
.withSingularPastMessage(tableLabel + " record was not")
|
||||
.withPluralPastMessage(tableLabel + " records were not");
|
||||
.withSingularFutureMessage(" record will not be")
|
||||
.withPluralFutureMessage(" records will not be")
|
||||
.withSingularPastMessage(" record was not")
|
||||
.withPluralPastMessage(" records were not");
|
||||
|
||||
ukErrorSummary.addSelfToListIfAnyCount(rs);
|
||||
}
|
||||
|
||||
@ -30,18 +30,20 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.AlphaNumericComparator;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper class for process steps that want to roll up error summary and/or
|
||||
** warning summary lines. e.g., if the process might have a handful of different
|
||||
** error messages. Will record up to 50 unique errors, then throw the rest int
|
||||
** error messages. Will record up to 50 (configurable) unique errors, then throw the rest int
|
||||
** an "other" errors summary.
|
||||
*******************************************************************************/
|
||||
public class ProcessSummaryWarningsAndErrorsRollup
|
||||
{
|
||||
private Map<String, ProcessSummaryLine> errorSummaries = new HashMap<>();
|
||||
private Map<String, ProcessSummaryLine> errorSummaries = new HashMap<>();
|
||||
private Map<String, ProcessSummaryLine> warningSummaries = new HashMap<>();
|
||||
|
||||
private ProcessSummaryLine otherErrorsSummary;
|
||||
@ -49,6 +51,8 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
private ProcessSummaryLine errorTemplate;
|
||||
private ProcessSummaryLine warningTemplate;
|
||||
|
||||
private int uniqueErrorsToShow = 50;
|
||||
private boolean doReplaceSingletonCountLinesWithSuffixOnly = true;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -167,7 +171,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
ProcessSummaryLine processSummaryLine = summaryLineMap.get(message);
|
||||
if(processSummaryLine == null)
|
||||
{
|
||||
if(summaryLineMap.size() < 50)
|
||||
if(summaryLineMap.size() < uniqueErrorsToShow)
|
||||
{
|
||||
processSummaryLine = new ProcessSummaryLine(status)
|
||||
.withMessageSuffix(message)
|
||||
@ -210,17 +214,80 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
|
||||
** extracts string messages out.
|
||||
**
|
||||
** Makes errors from bulk-insert look better when they report, e.g.
|
||||
** Error parsing line #1: ...
|
||||
** Error parsing line #2: ...
|
||||
** Error parsing line #10: ...
|
||||
*******************************************************************************/
|
||||
private static class PSLAlphaNumericComparator implements Comparator<ProcessSummaryLineInterface>
|
||||
{
|
||||
private static AlphaNumericComparator alphaNumericComparator = new AlphaNumericComparator();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int compare(ProcessSummaryLineInterface psli1, ProcessSummaryLineInterface psli2)
|
||||
{
|
||||
int messageComp = (alphaNumericComparator.compare(Objects.requireNonNullElse(psli1.getMessage(), ""), Objects.requireNonNullElse(psli2.getMessage(), "")));
|
||||
if(messageComp != 0)
|
||||
{
|
||||
return (messageComp);
|
||||
}
|
||||
|
||||
if(psli1 instanceof ProcessSummaryLine psl1 && psli2 instanceof ProcessSummaryLine psl2)
|
||||
{
|
||||
return (alphaNumericComparator.compare(Objects.requireNonNullElse(psl1.getMessageSuffix(), ""), Objects.requireNonNullElse(psl2.getMessageSuffix(), "")));
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** sort the process summary lines by count desc
|
||||
*******************************************************************************/
|
||||
private static void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
|
||||
private void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
|
||||
{
|
||||
summaryMap.values().stream()
|
||||
.sorted(Comparator.comparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getCount(), 0)).reversed()
|
||||
.thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessage(), ""))
|
||||
.thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessageSuffix(), ""))
|
||||
.thenComparing(new PSLAlphaNumericComparator())
|
||||
)
|
||||
.forEach(psl -> psl.addSelfToListIfAnyCount(rs));
|
||||
.map(psl ->
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// this is to make lines that are like "1 record had an error: Error parsing line #1: blah" look better, by //
|
||||
// removing the redundant "1 record..." bit. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
if(psl.getCount() == 1)
|
||||
{
|
||||
return (new ProcessSummaryRecordLink().withStatus(Status.ERROR).withLinkPreText(psl.getMessageSuffix()));
|
||||
}
|
||||
}
|
||||
|
||||
return (psl);
|
||||
})
|
||||
.forEach(psli ->
|
||||
{
|
||||
if(psli instanceof ProcessSummaryLine psl)
|
||||
{
|
||||
psl.addSelfToListIfAnyCount(rs);
|
||||
}
|
||||
else
|
||||
{
|
||||
rs.add(psli);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -347,4 +414,67 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public int getUniqueErrorsToShow()
|
||||
{
|
||||
return (this.uniqueErrorsToShow);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public void setUniqueErrorsToShow(int uniqueErrorsToShow)
|
||||
{
|
||||
this.uniqueErrorsToShow = uniqueErrorsToShow;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public ProcessSummaryWarningsAndErrorsRollup withUniqueErrorsToShow(int uniqueErrorsToShow)
|
||||
{
|
||||
this.uniqueErrorsToShow = uniqueErrorsToShow;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public boolean getDoReplaceSingletonCountLinesWithSuffixOnly()
|
||||
{
|
||||
return (this.doReplaceSingletonCountLinesWithSuffixOnly);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public void setDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public ProcessSummaryWarningsAndErrorsRollup withDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -95,7 +95,7 @@ public class ValueUtils
|
||||
}
|
||||
else if(value instanceof String s)
|
||||
{
|
||||
return (Boolean.parseBoolean(s));
|
||||
return "true".equalsIgnoreCase(s) || "yes".equalsIgnoreCase(s);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -496,6 +496,9 @@ public class ValueUtils
|
||||
*******************************************************************************/
|
||||
private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e)
|
||||
{
|
||||
//////////////////////
|
||||
// 1999-12-31T12:59 //
|
||||
//////////////////////
|
||||
if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$"))
|
||||
{
|
||||
//////////////////////////
|
||||
@ -503,11 +506,34 @@ public class ValueUtils
|
||||
//////////////////////////
|
||||
return Instant.parse(s + ":00Z");
|
||||
}
|
||||
|
||||
///////////////////////////
|
||||
// 1999-12-31 12:59:59.0 //
|
||||
///////////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z");
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
/////////////////////////
|
||||
// 1999-12-31 12:59:59 //
|
||||
/////////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T") + "Z";
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
//////////////////////
|
||||
// 1999-12-31 12:59 //
|
||||
//////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T") + ":00Z";
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
try
|
||||
|
||||
@ -0,0 +1,199 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.collections;
|
||||
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Comparator for strings that are a mix of alpha + numeric, where we want to
|
||||
** sort the numeric substrings like numbers.
|
||||
**
|
||||
** e.g., A1, A2, A10 won't come out as A1, A10, A2
|
||||
*******************************************************************************/
|
||||
public class AlphaNumericComparator implements Comparator<String>
|
||||
{
|
||||
private static final int A_FIRST = -1;
|
||||
private static final int B_FIRST = 1;
|
||||
private static final int TIE = 0;
|
||||
|
||||
private static final Pattern INT_PATTERN = Pattern.compile("^\\d+$");
|
||||
private static final Pattern LEADING_INT_PATTERN = Pattern.compile("^\\d+");
|
||||
private static final Pattern ALPHA_THEN_INT_PATTERN = Pattern.compile("^(\\D+)\\d+");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** compare 2 Strings
|
||||
**
|
||||
*******************************************************************************/
|
||||
public int compare(String a, String b)
|
||||
{
|
||||
try
|
||||
{
|
||||
//////////////////////////////////////
|
||||
// eliminate degenerate cases first //
|
||||
//////////////////////////////////////
|
||||
if(a == null && b == null)
|
||||
{
|
||||
return (TIE);
|
||||
}
|
||||
else if(a == null)
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b == null)
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
else if(a.equals(b))
|
||||
{
|
||||
return (TIE);
|
||||
} // also covers a == "" and b == ""
|
||||
else if(a.equals(""))
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b.equals(""))
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// if both strings are pure numeric, parse as int and compare //
|
||||
////////////////////////////////////////////////////////////////
|
||||
if(INT_PATTERN.matcher(a).matches() && INT_PATTERN.matcher(b).matches())
|
||||
{
|
||||
int intsCompared = new Integer(a).compareTo(new Integer(b));
|
||||
if(intsCompared == TIE)
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// in case the integers are the same (ie, "0001" vs "1"), compare as strings //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
else
|
||||
{
|
||||
///////////////////////////////////////////////////////////////
|
||||
// else, if the ints were different, return their comparison //
|
||||
///////////////////////////////////////////////////////////////
|
||||
return (intsCompared);
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// if both start as numbers, extract those numbers //
|
||||
/////////////////////////////////////////////////////
|
||||
Matcher aLeadingIntMatcher = LEADING_INT_PATTERN.matcher(a);
|
||||
Matcher bLeadingIntMatcher = LEADING_INT_PATTERN.matcher(b);
|
||||
if(aLeadingIntMatcher.lookingAt() && bLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
///////////////////////////
|
||||
// extract the int parts //
|
||||
///////////////////////////
|
||||
String aIntPart = a.substring(0, aLeadingIntMatcher.end());
|
||||
String bIntPart = b.substring(0, bLeadingIntMatcher.end());
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// if the ints compare as non-zero, return that comparison //
|
||||
/////////////////////////////////////////////////////////////
|
||||
int intPartCompared = new Integer(aIntPart).compareTo(new Integer(bIntPart));
|
||||
if(intPartCompared != TIE)
|
||||
{
|
||||
return (intPartCompared);
|
||||
}
|
||||
else
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// otherwise, make recursive call to compare the rest of the string //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
String aRest = a.substring(aLeadingIntMatcher.end());
|
||||
String bRest = b.substring(bLeadingIntMatcher.end());
|
||||
return (compare(aRest, bRest));
|
||||
}
|
||||
}
|
||||
//////////////////////////////////////////////////////
|
||||
// if one starts as numeric, but other doesn't //
|
||||
// return the one that starts with the number first //
|
||||
//////////////////////////////////////////////////////
|
||||
else if(aLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(bLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// now, if both parts have an alpha part, followed by digit parts, and //
|
||||
// the alpha parts are the same, then discard the alpha parts and recur //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
Matcher aAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(a);
|
||||
Matcher bAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(b);
|
||||
if(aAlphaThenIntMatcher.lookingAt() && bAlphaThenIntMatcher.lookingAt())
|
||||
{
|
||||
String aAlphaPart = aAlphaThenIntMatcher.group(1);
|
||||
String bAlphaPart = bAlphaThenIntMatcher.group(1);
|
||||
|
||||
if(aAlphaPart.equals(bAlphaPart))
|
||||
{
|
||||
String aRest = a.substring(aAlphaPart.length());
|
||||
String bRest = b.substring(bAlphaPart.length());
|
||||
return (compare(aRest, bRest));
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// as last resort, just do pure string compare //
|
||||
/////////////////////////////////////////////////
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
//////////////////////////////////////////////////////////
|
||||
// on exception, don't allow caller to catch -- rather, //
|
||||
// always return something sensible (and null-safe) //
|
||||
//////////////////////////////////////////////////////////
|
||||
if(a == null && b == null)
|
||||
{
|
||||
return (TIE);
|
||||
}
|
||||
else if(a == null)
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b == null)
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
else
|
||||
{
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -22,15 +22,19 @@
|
||||
package com.kingsrook.qqq.backend.core.adapters;
|
||||
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
@ -395,4 +399,70 @@ class CsvToQRecordAdapterTest extends BaseTest
|
||||
assertEquals("john@doe.com", records.get(0).getValueString("email"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN)))
|
||||
.withCsv("""
|
||||
firstName,birthDate,isEmployed
|
||||
John,1/1/1980,true
|
||||
Paul,1970-06-15,Yes
|
||||
George,,anything-else
|
||||
"""));
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
assertEquals("John", qRecord.getValue("firstName"));
|
||||
assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate"));
|
||||
assertEquals(true, qRecord.getValue("isEmployed"));
|
||||
|
||||
qRecord = qRecords.get(1);
|
||||
assertEquals("Paul", qRecord.getValue("firstName"));
|
||||
assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate"));
|
||||
assertEquals(true, qRecord.getValue("isEmployed"));
|
||||
|
||||
qRecord = qRecords.get(2);
|
||||
assertEquals("George", qRecord.getValue("firstName"));
|
||||
assertNull(qRecord.getValue("birthDate"));
|
||||
assertEquals(false, qRecord.getValue("isEmployed"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(TestUtils.defineTablePerson())
|
||||
.withCsv("""
|
||||
firstName,birthDate,favoriteShapeId
|
||||
John,1980,1
|
||||
Paul,1970-06-15,green
|
||||
"""));
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
assertEquals("John", qRecord.getValue("firstName"));
|
||||
assertThat(qRecord.getErrors()).hasSize(1);
|
||||
assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date");
|
||||
|
||||
qRecord = qRecords.get(1);
|
||||
assertEquals("Paul", qRecord.getValue("firstName"));
|
||||
assertThat(qRecord.getErrors()).hasSize(1);
|
||||
assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -62,6 +62,42 @@ class BulkInsertTest extends BaseTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow1()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow2()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvHeaderUsingLabels()
|
||||
{
|
||||
return ("""
|
||||
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -77,7 +113,7 @@ class BulkInsertTest extends BaseTest
|
||||
// create an uploaded file, similar to how an http server may //
|
||||
////////////////////////////////////////////////////////////////
|
||||
QUploadedFile qUploadedFile = new QUploadedFile();
|
||||
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
|
||||
qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
|
||||
qUploadedFile.setFilename("test.csv");
|
||||
UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
|
||||
TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);
|
||||
|
||||
@ -1252,41 +1252,6 @@ public class TestUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvHeaderUsingLabels()
|
||||
{
|
||||
return ("""
|
||||
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow1()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow2()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
|
||||
@ -0,0 +1,149 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.collections;
|
||||
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for AlphaNumericComparator
|
||||
*******************************************************************************/
|
||||
class AlphaNumericComparatorTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** test odd-balls
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testFringeCases()
|
||||
{
|
||||
test(sort("", null, "foo", " ", "", "1", null),
|
||||
null, null, "", "", "1", " ", "foo");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test alpha-strings only
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testAlphasOnly()
|
||||
{
|
||||
test(sort("F", "G", "A", "AB", "BB", "BA", "BD"),
|
||||
"A", "AB", "BA", "BB", "BD", "F", "G");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test numbers only
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testNumbersOnly()
|
||||
{
|
||||
test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"),
|
||||
"1", "2", "10", "13", "22", "101", "102", "271", "273", "273");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test mixed
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMixed1()
|
||||
{
|
||||
test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"),
|
||||
"001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test mixed
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMixed2()
|
||||
{
|
||||
test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"),
|
||||
"0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void test(List<String> a, String... b)
|
||||
{
|
||||
System.out.println("Expecting: " + Arrays.asList(b));
|
||||
|
||||
assertEquals(a.size(), b.length);
|
||||
|
||||
for(int i = 0; i < a.size(); i++)
|
||||
{
|
||||
String aString = a.get(i);
|
||||
String bString = b[i];
|
||||
|
||||
assertEquals(aString, bString);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> sort(String... input)
|
||||
{
|
||||
List<String> inputList = Arrays.asList(input);
|
||||
System.out.println("Sorting: " + inputList);
|
||||
|
||||
try
|
||||
{
|
||||
List<String> naturalSortList = Arrays.asList(input);
|
||||
Collections.sort(naturalSortList);
|
||||
System.out.println("Natural: " + naturalSortList);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
System.out.println("Natural: FAILED");
|
||||
}
|
||||
|
||||
inputList.sort(new AlphaNumericComparator());
|
||||
System.out.println("Produced: " + inputList);
|
||||
return (inputList);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user