From a0d217ed449985fad0268f38a055be26eb033a28 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 20 Oct 2023 12:18:32 -0500 Subject: [PATCH] CE-604 Bulk load improvements - type handling (via csv adapter) and better errors --- .../core/adapters/CsvToQRecordAdapter.java | 90 +++++++- .../bulk/insert/BulkInsertExtractStep.java | 1 + .../bulk/insert/BulkInsertTransformStep.java | 28 ++- ...ProcessSummaryWarningsAndErrorsRollup.java | 144 ++++++++++++- .../qqq/backend/core/utils/ValueUtils.java | 28 ++- .../collections/AlphaNumericComparator.java | 199 ++++++++++++++++++ .../adapters/CsvToQRecordAdapterTest.java | 70 ++++++ .../bulk/insert/BulkInsertTest.java | 38 +++- .../qqq/backend/core/utils/TestUtils.java | 35 --- .../AlphaNumericComparatorTest.java | 149 +++++++++++++ 10 files changed, 717 insertions(+), 65 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 46f51d58..925abf88 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -156,14 +158,21 @@ public class CsvToQRecordAdapter // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QRecord qRecord = new QRecord(); - for(QFieldMetaData field : table.getFields().values()) + try { - String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); - fieldSource = adjustHeaderCase(fieldSource, inputWrapper); - qRecord.setValue(field.getName(), csvValues.get(fieldSource)); - } + for(QFieldMetaData field : table.getFields().values()) + { + String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); + fieldSource = adjustHeaderCase(fieldSource, inputWrapper); + setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource)); + } - runRecordCustomizer(recordCustomizer, qRecord); + runRecordCustomizer(recordCustomizer, qRecord); + } + catch(Exception e) + { + qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage())); + } addRecord(qRecord); recordCount++; @@ -202,13 +211,20 @@ public class CsvToQRecordAdapter // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QRecord qRecord = new QRecord(); - for(QFieldMetaData field : table.getFields().values()) + try { - Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName()); - qRecord.setValue(field.getName(), csvValues.get(fieldIndex)); - } + for(QFieldMetaData field : table.getFields().values()) + { + Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName()); + setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex)); + } - runRecordCustomizer(recordCustomizer, qRecord); + runRecordCustomizer(recordCustomizer, qRecord); + } + catch(Exception e) + { + qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage())); + } addRecord(qRecord); recordCount++; @@ -231,6 +247,23 @@ public class CsvToQRecordAdapter + /******************************************************************************* + ** + *******************************************************************************/ + private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString) + { + if(inputWrapper.doCorrectValueTypes) + { + qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString)); + } + else + { + qRecord.setValue(field.getName(), valueString); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -341,6 +374,7 @@ public class CsvToQRecordAdapter private AbstractQFieldMapping mapping; private Consumer recordCustomizer; private Integer limit; + private boolean doCorrectValueTypes = false; private boolean caseSensitiveHeaders = false; @@ -582,6 +616,40 @@ public class CsvToQRecordAdapter return (this); } + + + /******************************************************************************* + ** Getter for doCorrectValueTypes + ** + *******************************************************************************/ + public boolean getDoCorrectValueTypes() + { + return doCorrectValueTypes; + } + + + + /******************************************************************************* + ** Setter for doCorrectValueTypes + ** + *******************************************************************************/ + public void setDoCorrectValueTypes(boolean doCorrectValueTypes) + { + this.doCorrectValueTypes = doCorrectValueTypes; + } + + + + /******************************************************************************* + ** Fluent setter for doCorrectValueTypes + ** + *******************************************************************************/ + public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes) + { + this.doCorrectValueTypes = doCorrectValueTypes; + return (this); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index d8850d40..3a83c955 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -82,6 +82,7 @@ public class BulkInsertExtractStep extends AbstractExtractStep .withRecordPipe(getRecordPipe()) .withLimit(getLimit()) .withCsv(new String(bytes)) + .withDoCorrectValueTypes(true) .withTable(runBackendStepInput.getInstance().getTable(tableName)) .withMapping(mapping) .withRecordCustomizer((record) -> diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 9344fdf1..7f33a624 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -75,11 +76,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep /******************************************************************************* - ** + ** extension of ProcessSummaryLine for lines where a UniqueKey was violated, + ** where we'll collect a sample (or maybe all) of the values that broke the UK. *******************************************************************************/ private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine { - private List sampleValues = new ArrayList<>(); + private Set sampleValues = new LinkedHashSet<>(); + private boolean areThereMoreSampleValues = false; @@ -116,8 +119,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int rowsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + int rowsInThisPage = runBackendStepInput.getRecords().size(); + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -278,6 +281,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep { processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString()); } + else + { + processSummaryLineWithUKSampleValues.areThereMoreSampleValues = true; + } foundDupe = true; break; } @@ -325,13 +332,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue(); ukErrorSummary - .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values such as: " - + StringUtils.joinWithCommasAndAnd(ukErrorSummary.sampleValues)) + .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) - .withSingularFutureMessage(tableLabel + " record will not be") - .withPluralFutureMessage(tableLabel + " records will not be") - .withSingularPastMessage(tableLabel + " record was not") - .withPluralPastMessage(tableLabel + " records were not"); + .withSingularFutureMessage(" record will not be") + .withPluralFutureMessage(" records will not be") + .withSingularPastMessage(" record was not") + .withPluralPastMessage(" records were not"); ukErrorSummary.addSelfToListIfAnyCount(rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index ce0cb2ed..231d9113 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -30,18 +30,20 @@ import java.util.Map; import java.util.Objects; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.utils.collections.AlphaNumericComparator; /******************************************************************************* ** Helper class for process steps that want to roll up error summary and/or ** warning summary lines. e.g., if the process might have a handful of different - ** error messages. Will record up to 50 unique errors, then throw the rest int + ** error messages. Will record up to 50 (configurable) unique errors, then throw the rest int ** an "other" errors summary. *******************************************************************************/ public class ProcessSummaryWarningsAndErrorsRollup { - private Map errorSummaries = new HashMap<>(); + private Map errorSummaries = new HashMap<>(); private Map warningSummaries = new HashMap<>(); private ProcessSummaryLine otherErrorsSummary; @@ -49,6 +51,8 @@ public class ProcessSummaryWarningsAndErrorsRollup private ProcessSummaryLine errorTemplate; private ProcessSummaryLine warningTemplate; + private int uniqueErrorsToShow = 50; + private boolean doReplaceSingletonCountLinesWithSuffixOnly = true; /******************************************************************************* @@ -167,7 +171,7 @@ public class ProcessSummaryWarningsAndErrorsRollup ProcessSummaryLine processSummaryLine = summaryLineMap.get(message); if(processSummaryLine == null) { - if(summaryLineMap.size() < 50) + if(summaryLineMap.size() < uniqueErrorsToShow) { processSummaryLine = new ProcessSummaryLine(status) .withMessageSuffix(message) @@ -210,17 +214,80 @@ public class ProcessSummaryWarningsAndErrorsRollup + /******************************************************************************* + ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that + ** extracts string messages out. + ** + ** Makes errors from bulk-insert look better when they report, e.g. + ** Error parsing line #1: ... + ** Error parsing line #2: ... + ** Error parsing line #10: ... + *******************************************************************************/ + private static class PSLAlphaNumericComparator implements Comparator + { + private static AlphaNumericComparator alphaNumericComparator = new AlphaNumericComparator(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int compare(ProcessSummaryLineInterface psli1, ProcessSummaryLineInterface psli2) + { + int messageComp = (alphaNumericComparator.compare(Objects.requireNonNullElse(psli1.getMessage(), ""), Objects.requireNonNullElse(psli2.getMessage(), ""))); + if(messageComp != 0) + { + return (messageComp); + } + + if(psli1 instanceof ProcessSummaryLine psl1 && psli2 instanceof ProcessSummaryLine psl2) + { + return (alphaNumericComparator.compare(Objects.requireNonNullElse(psl1.getMessageSuffix(), ""), Objects.requireNonNullElse(psl2.getMessageSuffix(), ""))); + } + + return (0); + } + } + + + /******************************************************************************* ** sort the process summary lines by count desc *******************************************************************************/ - private static void addProcessSummaryLinesFromMap(ArrayList rs, Map summaryMap) + private void addProcessSummaryLinesFromMap(ArrayList rs, Map summaryMap) { summaryMap.values().stream() .sorted(Comparator.comparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getCount(), 0)).reversed() - .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessage(), "")) - .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessageSuffix(), "")) + .thenComparing(new PSLAlphaNumericComparator()) ) - .forEach(psl -> psl.addSelfToListIfAnyCount(rs)); + .map(psl -> + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this is to make lines that are like "1 record had an error: Error parsing line #1: blah" look better, by // + // removing the redundant "1 record..." bit. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(doReplaceSingletonCountLinesWithSuffixOnly) + { + if(psl.getCount() == 1) + { + return (new ProcessSummaryRecordLink().withStatus(Status.ERROR).withLinkPreText(psl.getMessageSuffix())); + } + } + + return (psl); + }) + .forEach(psli -> + { + if(psli instanceof ProcessSummaryLine psl) + { + psl.addSelfToListIfAnyCount(rs); + } + else + { + rs.add(psli); + } + }); } @@ -347,4 +414,67 @@ public class ProcessSummaryWarningsAndErrorsRollup return (this); } + + + /******************************************************************************* + ** Getter for uniqueErrorsToShow + *******************************************************************************/ + public int getUniqueErrorsToShow() + { + return (this.uniqueErrorsToShow); + } + + + + /******************************************************************************* + ** Setter for uniqueErrorsToShow + *******************************************************************************/ + public void setUniqueErrorsToShow(int uniqueErrorsToShow) + { + this.uniqueErrorsToShow = uniqueErrorsToShow; + } + + + + /******************************************************************************* + ** Fluent setter for uniqueErrorsToShow + *******************************************************************************/ + public ProcessSummaryWarningsAndErrorsRollup withUniqueErrorsToShow(int uniqueErrorsToShow) + { + this.uniqueErrorsToShow = uniqueErrorsToShow; + return (this); + } + + + + /******************************************************************************* + ** Getter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public boolean getDoReplaceSingletonCountLinesWithSuffixOnly() + { + return (this.doReplaceSingletonCountLinesWithSuffixOnly); + } + + + + /******************************************************************************* + ** Setter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public void setDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly) + { + this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly; + } + + + + /******************************************************************************* + ** Fluent setter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public ProcessSummaryWarningsAndErrorsRollup withDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly) + { + this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 6ecc2dc8..8adee919 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -95,7 +95,7 @@ public class ValueUtils } else if(value instanceof String s) { - return (Boolean.parseBoolean(s)); + return "true".equalsIgnoreCase(s) || "yes".equalsIgnoreCase(s); } else { @@ -496,6 +496,9 @@ public class ValueUtils *******************************************************************************/ private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) { + ////////////////////// + // 1999-12-31T12:59 // + ////////////////////// if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$")) { ////////////////////////// @@ -503,11 +506,34 @@ public class ValueUtils ////////////////////////// return Instant.parse(s + ":00Z"); } + + /////////////////////////// + // 1999-12-31 12:59:59.0 // + /////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$")) { s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z"); return Instant.parse(s); } + + ///////////////////////// + // 1999-12-31 12:59:59 // + ///////////////////////// + else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$")) + { + s = s.replaceAll(" ", "T") + "Z"; + return Instant.parse(s); + } + + ////////////////////// + // 1999-12-31 12:59 // + ////////////////////// + else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$")) + { + s = s.replaceAll(" ", "T") + ":00Z"; + return Instant.parse(s); + } + else { try diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java new file mode 100644 index 00000000..2eb32327 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java @@ -0,0 +1,199 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Comparator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/******************************************************************************* + ** Comparator for strings that are a mix of alpha + numeric, where we want to + ** sort the numeric substrings like numbers. + ** + ** e.g., A1, A2, A10 won't come out as A1, A10, A2 + *******************************************************************************/ +public class AlphaNumericComparator implements Comparator +{ + private static final int A_FIRST = -1; + private static final int B_FIRST = 1; + private static final int TIE = 0; + + private static final Pattern INT_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern LEADING_INT_PATTERN = Pattern.compile("^\\d+"); + private static final Pattern ALPHA_THEN_INT_PATTERN = Pattern.compile("^(\\D+)\\d+"); + + + + /******************************************************************************* + ** compare 2 Strings + ** + *******************************************************************************/ + public int compare(String a, String b) + { + try + { + ////////////////////////////////////// + // eliminate degenerate cases first // + ////////////////////////////////////// + if(a == null && b == null) + { + return (TIE); + } + else if(a == null) + { + return (A_FIRST); + } + else if(b == null) + { + return (B_FIRST); + } + else if(a.equals(b)) + { + return (TIE); + } // also covers a == "" and b == "" + else if(a.equals("")) + { + return (A_FIRST); + } + else if(b.equals("")) + { + return (B_FIRST); + } + + //////////////////////////////////////////////////////////////// + // if both strings are pure numeric, parse as int and compare // + //////////////////////////////////////////////////////////////// + if(INT_PATTERN.matcher(a).matches() && INT_PATTERN.matcher(b).matches()) + { + int intsCompared = new Integer(a).compareTo(new Integer(b)); + if(intsCompared == TIE) + { + /////////////////////////////////////////////////////////////////////////////// + // in case the integers are the same (ie, "0001" vs "1"), compare as strings // + /////////////////////////////////////////////////////////////////////////////// + return (a.compareTo(b)); + } + else + { + /////////////////////////////////////////////////////////////// + // else, if the ints were different, return their comparison // + /////////////////////////////////////////////////////////////// + return (intsCompared); + } + } + + ///////////////////////////////////////////////////// + // if both start as numbers, extract those numbers // + ///////////////////////////////////////////////////// + Matcher aLeadingIntMatcher = LEADING_INT_PATTERN.matcher(a); + Matcher bLeadingIntMatcher = LEADING_INT_PATTERN.matcher(b); + if(aLeadingIntMatcher.lookingAt() && bLeadingIntMatcher.lookingAt()) + { + /////////////////////////// + // extract the int parts // + /////////////////////////// + String aIntPart = a.substring(0, aLeadingIntMatcher.end()); + String bIntPart = b.substring(0, bLeadingIntMatcher.end()); + + ///////////////////////////////////////////////////////////// + // if the ints compare as non-zero, return that comparison // + ///////////////////////////////////////////////////////////// + int intPartCompared = new Integer(aIntPart).compareTo(new Integer(bIntPart)); + if(intPartCompared != TIE) + { + return (intPartCompared); + } + else + { + ////////////////////////////////////////////////////////////////////// + // otherwise, make recursive call to compare the rest of the string // + ////////////////////////////////////////////////////////////////////// + String aRest = a.substring(aLeadingIntMatcher.end()); + String bRest = b.substring(bLeadingIntMatcher.end()); + return (compare(aRest, bRest)); + } + } + ////////////////////////////////////////////////////// + // if one starts as numeric, but other doesn't // + // return the one that starts with the number first // + ////////////////////////////////////////////////////// + else if(aLeadingIntMatcher.lookingAt()) + { + return (A_FIRST); + } + else if(bLeadingIntMatcher.lookingAt()) + { + return (B_FIRST); + } + + ////////////////////////////////////////////////////////////////////////// + // now, if both parts have an alpha part, followed by digit parts, and // + // the alpha parts are the same, then discard the alpha parts and recur // + ////////////////////////////////////////////////////////////////////////// + Matcher aAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(a); + Matcher bAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(b); + if(aAlphaThenIntMatcher.lookingAt() && bAlphaThenIntMatcher.lookingAt()) + { + String aAlphaPart = aAlphaThenIntMatcher.group(1); + String bAlphaPart = bAlphaThenIntMatcher.group(1); + + if(aAlphaPart.equals(bAlphaPart)) + { + String aRest = a.substring(aAlphaPart.length()); + String bRest = b.substring(bAlphaPart.length()); + return (compare(aRest, bRest)); + } + } + + ///////////////////////////////////////////////// + // as last resort, just do pure string compare // + ///////////////////////////////////////////////// + return (a.compareTo(b)); + } + catch(Exception e) + { + ////////////////////////////////////////////////////////// + // on exception, don't allow caller to catch -- rather, // + // always return something sensible (and null-safe) // + ////////////////////////////////////////////////////////// + if(a == null && b == null) + { + return (TIE); + } + else if(a == null) + { + return (A_FIRST); + } + else if(b == null) + { + return (B_FIRST); + } + else + { + return (a.compareTo(b)); + } + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java index e1fba082..cb71a4fe 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -22,15 +22,19 @@ package com.kingsrook.qqq.backend.core.adapters; +import java.time.LocalDate; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -395,4 +399,70 @@ class CsvToQRecordAdapterTest extends BaseTest assertEquals("john@doe.com", records.get(0).getValueString("email")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withDoCorrectValueTypes(true) + .withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN))) + .withCsv(""" + firstName,birthDate,isEmployed + John,1/1/1980,true + Paul,1970-06-15,Yes + George,,anything-else + """)); + List qRecords = csvToQRecordAdapter.getRecordList(); + + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate")); + assertEquals(true, qRecord.getValue("isEmployed")); + + qRecord = qRecords.get(1); + assertEquals("Paul", qRecord.getValue("firstName")); + assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate")); + assertEquals(true, qRecord.getValue("isEmployed")); + + qRecord = qRecords.get(2); + assertEquals("George", qRecord.getValue("firstName")); + assertNull(qRecord.getValue("birthDate")); + assertEquals(false, qRecord.getValue("isEmployed")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withDoCorrectValueTypes(true) + .withTable(TestUtils.defineTablePerson()) + .withCsv(""" + firstName,birthDate,favoriteShapeId + John,1980,1 + Paul,1970-06-15,green + """)); + List qRecords = csvToQRecordAdapter.getRecordList(); + + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertThat(qRecord.getErrors()).hasSize(1); + assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date"); + + qRecord = qRecords.get(1); + assertEquals("Paul", qRecord.getValue("firstName")); + assertThat(qRecord.getErrors()).hasSize(1); + assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer."); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java index c9c1603d..496ad0be 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java @@ -62,6 +62,42 @@ class BulkInsertTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" + """); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -77,7 +113,7 @@ class BulkInsertTest extends BaseTest // create an uploaded file, similar to how an http server may // //////////////////////////////////////////////////////////////// QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); + qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); qUploadedFile.setFilename("test.csv"); UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 12512dfb..bf1d0825 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -1252,41 +1252,6 @@ public class TestUtils - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvHeaderUsingLabels() - { - return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow1() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow2() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" - """); - } - - /******************************************************************************* ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java new file mode 100644 index 00000000..a61f2abd --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java @@ -0,0 +1,149 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for AlphaNumericComparator + *******************************************************************************/ +class AlphaNumericComparatorTest extends BaseTest +{ + + /******************************************************************************* + ** test odd-balls + ** + *******************************************************************************/ + @Test + public void testFringeCases() + { + test(sort("", null, "foo", " ", "", "1", null), + null, null, "", "", "1", " ", "foo"); + } + + + + /******************************************************************************* + ** test alpha-strings only + ** + *******************************************************************************/ + @Test + public void testAlphasOnly() + { + test(sort("F", "G", "A", "AB", "BB", "BA", "BD"), + "A", "AB", "BA", "BB", "BD", "F", "G"); + } + + + + /******************************************************************************* + ** test numbers only + ** + *******************************************************************************/ + @Test + public void testNumbersOnly() + { + test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"), + "1", "2", "10", "13", "22", "101", "102", "271", "273", "273"); + } + + + + /******************************************************************************* + ** test mixed + ** + *******************************************************************************/ + @Test + public void testMixed1() + { + test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"), + "001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10"); + } + + + + /******************************************************************************* + ** test mixed + ** + *******************************************************************************/ + @Test + public void testMixed2() + { + test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"), + "0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1"); + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + private void test(List a, String... b) + { + System.out.println("Expecting: " + Arrays.asList(b)); + + assertEquals(a.size(), b.length); + + for(int i = 0; i < a.size(); i++) + { + String aString = a.get(i); + String bString = b[i]; + + assertEquals(aString, bString); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + private List sort(String... input) + { + List inputList = Arrays.asList(input); + System.out.println("Sorting: " + inputList); + + try + { + List naturalSortList = Arrays.asList(input); + Collections.sort(naturalSortList); + System.out.println("Natural: " + naturalSortList); + } + catch(Exception e) + { + System.out.println("Natural: FAILED"); + } + + inputList.sort(new AlphaNumericComparator()); + System.out.println("Produced: " + inputList); + return (inputList); + } +} \ No newline at end of file