diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index d6fc2da1..4bb8ca06 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -84,7 +84,7 @@ org.json json - 20230227 + 20230618 org.apache.commons diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java index d1e21dc4..196ea4b8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/AbstractPreInsertCustomizer.java @@ -55,6 +55,21 @@ public abstract class AbstractPreInsertCustomizer + ///////////////////////////////////////////////////////////////////////////////// + // allow the customizer to specify when it should be executed as part of the // + // insert action. default (per method in this class) is AFTER_ALL_VALIDATIONS // + ///////////////////////////////////////////////////////////////////////////////// + public enum WhenToRun + { + BEFORE_ALL_VALIDATIONS, + BEFORE_UNIQUE_KEY_CHECKS, + BEFORE_REQUIRED_FIELD_CHECKS, + BEFORE_SECURITY_CHECKS, + AFTER_ALL_VALIDATIONS + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -62,6 +77,16 @@ public abstract class AbstractPreInsertCustomizer + /******************************************************************************* + ** + *******************************************************************************/ + public WhenToRun getWhenToRun() + { + return (WhenToRun.AFTER_ALL_VALIDATIONS); + } + + + /******************************************************************************* ** Getter for insertInput ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java index bbd18e47..f69596af 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/DateTimeGroupBy.java @@ -99,7 +99,17 @@ public enum DateTimeGroupBy public String getSqlExpression() { ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId(); - String targetTimezone = sessionOrInstanceZoneId.toString(); + return (getSqlExpression(sessionOrInstanceZoneId)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getSqlExpression(ZoneId targetZoneId) + { + String targetTimezone = targetZoneId.toString(); if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone)) { @@ -158,7 +168,18 @@ public enum DateTimeGroupBy *******************************************************************************/ public String makeSelectedString(Instant time) { - ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId()); + return (makeSelectedString(time, ValueUtils.getSessionOrInstanceZoneId())); + } + + + + /******************************************************************************* + ** Make an Instant into a string that will match what came out of the database's + ** DATE_FORMAT() function + *******************************************************************************/ + public String makeSelectedString(Instant time, ZoneId zoneId) + { + ZonedDateTime zoned = time.atZone(zoneId); if(this == WEEK) { @@ -182,7 +203,17 @@ public enum DateTimeGroupBy *******************************************************************************/ public String makeHumanString(Instant instant) { - ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId()); + return (makeHumanString(instant, ValueUtils.getSessionOrInstanceZoneId())); + } + + + + /******************************************************************************* + ** Make a string to show to a user + *******************************************************************************/ + public String makeHumanString(Instant instant, ZoneId zoneId) + { + ZonedDateTime zoned = instant.atZone(zoneId); if(this.equals(WEEK)) { DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d"); @@ -215,25 +246,35 @@ public enum DateTimeGroupBy /******************************************************************************* ** *******************************************************************************/ - @SuppressWarnings("checkstyle:indentation") public Instant roundDown(Instant instant) { - ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId()); + return roundDown(instant, ValueUtils.getSessionOrInstanceZoneId()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("checkstyle:indentation") + public Instant roundDown(Instant instant, ZoneId zoneId) + { + ZonedDateTime zoned = instant.atZone(zoneId); return switch(this) + { + case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant(); + case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant(); + case WEEK -> { - case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant(); - case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant(); - case WEEK -> + while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue()) { - while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue()) - { - zoned = zoned.minusDays(1); - } - yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant()); + zoned = zoned.minusDays(1); } - case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant(); - case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant(); - }; + yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant()); + } + case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant(); + case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant(); + }; } @@ -243,7 +284,17 @@ public enum DateTimeGroupBy *******************************************************************************/ public Instant increment(Instant instant) { - ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId()); + return (increment(instant, ValueUtils.getSessionOrInstanceZoneId())); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Instant increment(Instant instant, ZoneId zoneId) + { + ZonedDateTime zoned = instant.atZone(zoneId); return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant()); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java index 6feb2105..f3937a15 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/ParentWidgetRenderer.java @@ -60,6 +60,8 @@ public class ParentWidgetRenderer extends AbstractWidgetRenderer widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList()); } + widgetData.setLayoutType(metaData.getLayoutType()); + return (new RenderWidgetOutput(widgetData)); } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 0b5adcb0..a537c883 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -193,25 +193,76 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { preInsertCustomizer.get().setInsertInput(insertInput); preInsertCustomizer.get().setIsPreview(isPreview); - insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); + } + + setDefaultValuesInRecords(table, insertInput.getRecords()); + + ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords()); + + runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS); + setErrorsIfUniqueKeyErrors(insertInput, table); + + runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS); + if(insertInput.getInputSource().shouldValidateRequiredFields()) + { + validateRequiredFields(insertInput); + } + + runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS); + ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT); + + runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void setDefaultValuesInRecords(QTableMetaData table, List records) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // for all fields in the table - if any have a default value, then look at all input records, // + // and if they have null value, then apply the default // + //////////////////////////////////////////////////////////////////////////////////////////////// + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getDefaultValue() != null) + { + for(QRecord record : records) + { + if(record.getValue(field.getName()) == null) + { + record.setValue(field.getName(), field.getDefaultValue()); + } + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException + { + if(preInsertCustomizer.isPresent()) + { + if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun())) + { + insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords())); + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 7dc51ca4..860a4279 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -274,6 +274,14 @@ public class QValueFormatter *******************************************************************************/ private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record) { + ////////////////////////////////////////////////////////////////////////////////////// + // if the record already has a label (say, from a query-customizer), then return it // + ////////////////////////////////////////////////////////////////////////////////////// + if(record.getRecordLabel() != null) + { + return (record.getRecordLabel()); + } + /////////////////////////////////////////////////////////////////////////////////////// // if there's no record label format, then just return the primary key display value // /////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index fcfd2fa1..7816dba4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -208,37 +210,50 @@ public class SearchPossibleValueSourceAction } else { - if(StringUtils.hasContent(input.getSearchTerm())) + String searchTerm = input.getSearchTerm(); + if(StringUtils.hasContent(searchTerm)) { for(String valueField : possibleValueSource.getSearchFields()) { - QFieldMetaData field = table.getField(valueField); - if(field.getType().equals(QFieldType.STRING)) + try { - queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm()))); - } - else if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME)) - { - LOG.debug("Not querying PVS [" + possibleValueSource.getName() + "] on date field [" + field.getName() + "]"); - // todo - what? queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm()))); - } - else - { - try + QFieldMetaData field = table.getField(valueField); + if(field.getType().equals(QFieldType.STRING)) { - Integer valueAsInteger = ValueUtils.getValueAsInteger(input.getSearchTerm()); + queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(searchTerm))); + } + else if(field.getType().equals(QFieldType.DATE)) + { + LocalDate searchDate = ValueUtils.getValueAsLocalDate(searchTerm); + if(searchDate != null) + { + queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate)); + } + } + else if(field.getType().equals(QFieldType.DATE_TIME)) + { + Instant searchDate = ValueUtils.getValueAsInstant(searchTerm); + if(searchDate != null) + { + queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate)); + } + } + else + { + Integer valueAsInteger = ValueUtils.getValueAsInteger(searchTerm); if(valueAsInteger != null) { queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, List.of(valueAsInteger))); } } - catch(Exception e) - { - //////////////////////////////////////////////////////// - // write a FALSE criteria if the value isn't a number // - //////////////////////////////////////////////////////// - queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of())); - } + } + catch(Exception e) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // write a FALSE criteria upon exceptions (e.g., type conversion fails) // + // Why are we doing this? so a single-field query finds nothing instead of everything. // + ////////////////////////////////////////////////////////////////////////////////////////// + queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of())); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java index 46f51d58..925abf88 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapter.java @@ -36,7 +36,9 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -156,14 +158,21 @@ public class CsvToQRecordAdapter // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QRecord qRecord = new QRecord(); - for(QFieldMetaData field : table.getFields().values()) + try { - String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); - fieldSource = adjustHeaderCase(fieldSource, inputWrapper); - qRecord.setValue(field.getName(), csvValues.get(fieldSource)); - } + for(QFieldMetaData field : table.getFields().values()) + { + String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName())); + fieldSource = adjustHeaderCase(fieldSource, inputWrapper); + setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource)); + } - runRecordCustomizer(recordCustomizer, qRecord); + runRecordCustomizer(recordCustomizer, qRecord); + } + catch(Exception e) + { + qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage())); + } addRecord(qRecord); recordCount++; @@ -202,13 +211,20 @@ public class CsvToQRecordAdapter // now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QRecord qRecord = new QRecord(); - for(QFieldMetaData field : table.getFields().values()) + try { - Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName()); - qRecord.setValue(field.getName(), csvValues.get(fieldIndex)); - } + for(QFieldMetaData field : table.getFields().values()) + { + Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName()); + setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex)); + } - runRecordCustomizer(recordCustomizer, qRecord); + runRecordCustomizer(recordCustomizer, qRecord); + } + catch(Exception e) + { + qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage())); + } addRecord(qRecord); recordCount++; @@ -231,6 +247,23 @@ public class CsvToQRecordAdapter + /******************************************************************************* + ** + *******************************************************************************/ + private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString) + { + if(inputWrapper.doCorrectValueTypes) + { + qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString)); + } + else + { + qRecord.setValue(field.getName(), valueString); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -341,6 +374,7 @@ public class CsvToQRecordAdapter private AbstractQFieldMapping mapping; private Consumer recordCustomizer; private Integer limit; + private boolean doCorrectValueTypes = false; private boolean caseSensitiveHeaders = false; @@ -582,6 +616,40 @@ public class CsvToQRecordAdapter return (this); } + + + /******************************************************************************* + ** Getter for doCorrectValueTypes + ** + *******************************************************************************/ + public boolean getDoCorrectValueTypes() + { + return doCorrectValueTypes; + } + + + + /******************************************************************************* + ** Setter for doCorrectValueTypes + ** + *******************************************************************************/ + public void setDoCorrectValueTypes(boolean doCorrectValueTypes) + { + this.doCorrectValueTypes = doCorrectValueTypes; + } + + + + /******************************************************************************* + ** Fluent setter for doCorrectValueTypes + ** + *******************************************************************************/ + public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes) + { + this.doCorrectValueTypes = doCorrectValueTypes; + return (this); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java index 55d58086..3683c62a 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/logging/QLogger.java @@ -124,7 +124,7 @@ public class QLogger *******************************************************************************/ public void log(Level level, String message) { - logger.log(level, makeJsonString(message)); + logger.log(level, () -> makeJsonString(message)); } @@ -134,7 +134,7 @@ public class QLogger *******************************************************************************/ public void log(Level level, String message, Throwable t) { - logger.log(level, makeJsonString(message, t)); + logger.log(level, () -> makeJsonString(message, t)); } @@ -144,7 +144,7 @@ public class QLogger *******************************************************************************/ public void log(Level level, String message, Throwable t, LogPair... logPairs) { - logger.log(level, makeJsonString(message, t, logPairs)); + logger.log(level, () -> makeJsonString(message, t, logPairs)); } @@ -154,7 +154,7 @@ public class QLogger *******************************************************************************/ public void log(Level level, Throwable t) { - logger.log(level, makeJsonString(null, t)); + logger.log(level, () -> makeJsonString(null, t)); } @@ -164,7 +164,7 @@ public class QLogger *******************************************************************************/ public void trace(String message) { - logger.trace(makeJsonString(message)); + logger.trace(() -> makeJsonString(message)); } @@ -174,7 +174,7 @@ public class QLogger *******************************************************************************/ public void trace(String message, LogPair... logPairs) { - logger.trace(makeJsonString(message, null, logPairs)); + logger.trace(() -> makeJsonString(message, null, logPairs)); } @@ -194,7 +194,7 @@ public class QLogger *******************************************************************************/ public void trace(String message, Throwable t) { - logger.trace(makeJsonString(message, t)); + logger.trace(() -> makeJsonString(message, t)); } @@ -204,7 +204,7 @@ public class QLogger *******************************************************************************/ public void trace(String message, Throwable t, LogPair... logPairs) { - logger.trace(makeJsonString(message, t, logPairs)); + logger.trace(() -> makeJsonString(message, t, logPairs)); } @@ -214,7 +214,7 @@ public class QLogger *******************************************************************************/ public void trace(Throwable t) { - logger.trace(makeJsonString(null, t)); + logger.trace(() -> makeJsonString(null, t)); } @@ -224,7 +224,7 @@ public class QLogger *******************************************************************************/ public void debug(String message) { - logger.debug(makeJsonString(message)); + logger.debug(() -> makeJsonString(message)); } @@ -234,7 +234,7 @@ public class QLogger *******************************************************************************/ public void debug(String message, LogPair... logPairs) { - logger.debug(makeJsonString(message, null, logPairs)); + logger.debug(() -> makeJsonString(message, null, logPairs)); } @@ -254,7 +254,7 @@ public class QLogger *******************************************************************************/ public void debug(String message, Throwable t) { - logger.debug(makeJsonString(message, t)); + logger.debug(() -> makeJsonString(message, t)); } @@ -264,7 +264,7 @@ public class QLogger *******************************************************************************/ public void debug(String message, Throwable t, LogPair... logPairs) { - logger.debug(makeJsonString(message, t, logPairs)); + logger.debug(() -> makeJsonString(message, t, logPairs)); } @@ -274,7 +274,7 @@ public class QLogger *******************************************************************************/ public void debug(Throwable t) { - logger.debug(makeJsonString(null, t)); + logger.debug(() -> makeJsonString(null, t)); } @@ -284,7 +284,7 @@ public class QLogger *******************************************************************************/ public void info(String message) { - logger.info(makeJsonString(message)); + logger.info(() -> makeJsonString(message)); } @@ -294,7 +294,7 @@ public class QLogger *******************************************************************************/ public void info(LogPair... logPairs) { - logger.info(makeJsonString(null, null, logPairs)); + logger.info(() -> makeJsonString(null, null, logPairs)); } @@ -304,7 +304,7 @@ public class QLogger *******************************************************************************/ public void info(List logPairList) { - logger.info(makeJsonString(null, null, logPairList)); + logger.info(() -> makeJsonString(null, null, logPairList)); } @@ -314,7 +314,7 @@ public class QLogger *******************************************************************************/ public void info(String message, LogPair... logPairs) { - logger.info(makeJsonString(message, null, logPairs)); + logger.info(() -> makeJsonString(message, null, logPairs)); } @@ -334,7 +334,7 @@ public class QLogger *******************************************************************************/ public void info(String message, Throwable t) { - logger.info(makeJsonString(message, t)); + logger.info(() -> makeJsonString(message, t)); } @@ -344,7 +344,7 @@ public class QLogger *******************************************************************************/ public void info(String message, Throwable t, LogPair... logPairs) { - logger.info(makeJsonString(message, t, logPairs)); + logger.info(() -> makeJsonString(message, t, logPairs)); } @@ -354,7 +354,7 @@ public class QLogger *******************************************************************************/ public void info(Throwable t) { - logger.info(makeJsonString(null, t)); + logger.info(() -> makeJsonString(null, t)); } @@ -364,7 +364,7 @@ public class QLogger *******************************************************************************/ public void warn(String message) { - logger.warn(makeJsonString(message)); + logger.warn(() -> makeJsonString(message)); } @@ -374,7 +374,7 @@ public class QLogger *******************************************************************************/ public void warn(String message, LogPair... logPairs) { - logger.warn(makeJsonString(message, null, logPairs)); + logger.warn(() -> makeJsonString(message, null, logPairs)); } @@ -394,7 +394,7 @@ public class QLogger *******************************************************************************/ public void warn(String message, Throwable t) { - logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t)); + logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t)); } @@ -404,7 +404,7 @@ public class QLogger *******************************************************************************/ public void warn(String message, Throwable t, LogPair... logPairs) { - logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t, logPairs)); + logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t, logPairs)); } @@ -414,7 +414,7 @@ public class QLogger *******************************************************************************/ public void warn(Throwable t) { - logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(null, t)); + logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(null, t)); } @@ -424,7 +424,7 @@ public class QLogger *******************************************************************************/ public void error(String message) { - logger.error(makeJsonString(message)); + logger.error(() -> makeJsonString(message)); } @@ -434,7 +434,7 @@ public class QLogger *******************************************************************************/ public void error(String message, LogPair... logPairs) { - logger.error(makeJsonString(message, null, logPairs)); + logger.error(() -> makeJsonString(message, null, logPairs)); } @@ -454,7 +454,7 @@ public class QLogger *******************************************************************************/ public void error(String message, Throwable t) { - logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t)); + logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t)); } @@ -464,7 +464,7 @@ public class QLogger *******************************************************************************/ public void error(String message, Throwable t, LogPair... logPairs) { - logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t, logPairs)); + logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t, logPairs)); } @@ -474,7 +474,7 @@ public class QLogger *******************************************************************************/ public void error(Throwable t) { - logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(null, t)); + logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(null, t)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartData.java index a3866948..ac131938 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartData.java @@ -48,6 +48,8 @@ public class ChartData extends QWidgetData private boolean isCurrency = false; private int height; + private ChartSubheaderData chartSubheaderData; + /******************************************************************************* @@ -387,6 +389,7 @@ public class ChartData extends QWidgetData private String color; private String backgroundColor; private List urls; + private List backgroundColors; @@ -423,6 +426,17 @@ public class ChartData extends QWidgetData + /******************************************************************************* + ** Getter for backgroundColors + ** + *******************************************************************************/ + public List getBackgroundColors() + { + return backgroundColors; + } + + + /******************************************************************************* ** Setter for backgroundColor ** @@ -434,6 +448,17 @@ public class ChartData extends QWidgetData + /******************************************************************************* + ** Setter for backgroundColor + ** + *******************************************************************************/ + public void setBackgroundColors(List backgroundColors) + { + this.backgroundColors = backgroundColors; + } + + + /******************************************************************************* ** Fluent setter for backgroundColor ** @@ -446,6 +471,18 @@ public class ChartData extends QWidgetData + /******************************************************************************* + ** Fluent setter for backgroundColor + ** + *******************************************************************************/ + public Dataset withBackgroundColors(List backgroundColors) + { + this.backgroundColors = backgroundColors; + return (this); + } + + + /******************************************************************************* ** Getter for color ** @@ -559,4 +596,36 @@ public class ChartData extends QWidgetData } } } + + + + /******************************************************************************* + ** Getter for chartSubheaderData + *******************************************************************************/ + public ChartSubheaderData getChartSubheaderData() + { + return (this.chartSubheaderData); + } + + + + /******************************************************************************* + ** Setter for chartSubheaderData + *******************************************************************************/ + public void setChartSubheaderData(ChartSubheaderData chartSubheaderData) + { + this.chartSubheaderData = chartSubheaderData; + } + + + + /******************************************************************************* + ** Fluent setter for chartSubheaderData + *******************************************************************************/ + public ChartData withChartSubheaderData(ChartSubheaderData chartSubheaderData) + { + this.chartSubheaderData = chartSubheaderData; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartSubheaderData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartSubheaderData.java new file mode 100644 index 00000000..6e52dfb4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ChartSubheaderData.java @@ -0,0 +1,331 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.dashboard.widgets; + + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ChartSubheaderData +{ + private Number mainNumber; + private Number vsPreviousPercent; + private Number vsPreviousNumber; + private Boolean isUpVsPrevious; + private Boolean isGoodVsPrevious; + private String vsDescription = "vs prev period"; + + private String mainNumberUrl; + private String previousNumberUrl; + + + + /******************************************************************************* + ** Getter for mainNumber + *******************************************************************************/ + public Number getMainNumber() + { + return (this.mainNumber); + } + + + + /******************************************************************************* + ** Setter for mainNumber + *******************************************************************************/ + public void setMainNumber(Number mainNumber) + { + this.mainNumber = mainNumber; + } + + + + /******************************************************************************* + ** Fluent setter for mainNumber + *******************************************************************************/ + public ChartSubheaderData withMainNumber(Number mainNumber) + { + this.mainNumber = mainNumber; + return (this); + } + + + + /******************************************************************************* + ** Getter for vsPreviousNumber + *******************************************************************************/ + public Number getVsPreviousNumber() + { + return (this.vsPreviousNumber); + } + + + + /******************************************************************************* + ** Setter for vsPreviousNumber + *******************************************************************************/ + public void setVsPreviousNumber(Number vsPreviousNumber) + { + this.vsPreviousNumber = vsPreviousNumber; + } + + + + /******************************************************************************* + ** Fluent setter for vsPreviousNumber + *******************************************************************************/ + public ChartSubheaderData withVsPreviousNumber(Number vsPreviousNumber) + { + this.vsPreviousNumber = vsPreviousNumber; + return (this); + } + + + + /******************************************************************************* + ** Getter for vsDescription + *******************************************************************************/ + public String getVsDescription() + { + return (this.vsDescription); + } + + + + /******************************************************************************* + ** Setter for vsDescription + *******************************************************************************/ + public void setVsDescription(String vsDescription) + { + this.vsDescription = vsDescription; + } + + + + /******************************************************************************* + ** Fluent setter for vsDescription + *******************************************************************************/ + public ChartSubheaderData withVsDescription(String vsDescription) + { + this.vsDescription = vsDescription; + return (this); + } + + + + /******************************************************************************* + ** Getter for vsPreviousPercent + *******************************************************************************/ + public Number getVsPreviousPercent() + { + return (this.vsPreviousPercent); + } + + + + /******************************************************************************* + ** Setter for vsPreviousPercent + *******************************************************************************/ + public void setVsPreviousPercent(Number vsPreviousPercent) + { + this.vsPreviousPercent = vsPreviousPercent; + } + + + + /******************************************************************************* + ** Fluent setter for vsPreviousPercent + *******************************************************************************/ + public ChartSubheaderData withVsPreviousPercent(Number vsPreviousPercent) + { + this.vsPreviousPercent = vsPreviousPercent; + return (this); + } + + + + /******************************************************************************* + ** Getter for isUpVsPrevious + *******************************************************************************/ + public Boolean getIsUpVsPrevious() + { + return (this.isUpVsPrevious); + } + + + + /******************************************************************************* + ** Setter for isUpVsPrevious + *******************************************************************************/ + public void setIsUpVsPrevious(Boolean isUpVsPrevious) + { + this.isUpVsPrevious = isUpVsPrevious; + } + + + + /******************************************************************************* + ** Fluent setter for isUpVsPrevious + *******************************************************************************/ + public ChartSubheaderData withIsUpVsPrevious(Boolean isUpVsPrevious) + { + this.isUpVsPrevious = isUpVsPrevious; + return (this); + } + + + + /******************************************************************************* + ** Getter for isGoodVsPrevious + *******************************************************************************/ + public Boolean getIsGoodVsPrevious() + { + return (this.isGoodVsPrevious); + } + + + + /******************************************************************************* + ** Setter for isGoodVsPrevious + *******************************************************************************/ + public void setIsGoodVsPrevious(Boolean isGoodVsPrevious) + { + this.isGoodVsPrevious = isGoodVsPrevious; + } + + + + /******************************************************************************* + ** Fluent setter for isGoodVsPrevious + *******************************************************************************/ + public ChartSubheaderData withIsGoodVsPrevious(Boolean isGoodVsPrevious) + { + this.isGoodVsPrevious = isGoodVsPrevious; + return (this); + } + + + + /******************************************************************************* + ** Getter for mainNumberUrl + *******************************************************************************/ + public String getMainNumberUrl() + { + return (this.mainNumberUrl); + } + + + + /******************************************************************************* + ** Setter for mainNumberUrl + *******************************************************************************/ + public void setMainNumberUrl(String mainNumberUrl) + { + this.mainNumberUrl = mainNumberUrl; + } + + + + /******************************************************************************* + ** Fluent setter for mainNumberUrl + *******************************************************************************/ + public ChartSubheaderData withMainNumberUrl(String mainNumberUrl) + { + this.mainNumberUrl = mainNumberUrl; + return (this); + } + + + + /******************************************************************************* + ** Getter for previousNumberUrl + *******************************************************************************/ + public String getPreviousNumberUrl() + { + return (this.previousNumberUrl); + } + + + + /******************************************************************************* + ** Setter for previousNumberUrl + *******************************************************************************/ + public void setPreviousNumberUrl(String previousNumberUrl) + { + this.previousNumberUrl = previousNumberUrl; + } + + + + /******************************************************************************* + ** Fluent setter for previousNumberUrl + *******************************************************************************/ + public ChartSubheaderData withPreviousNumberUrl(String previousNumberUrl) + { + this.previousNumberUrl = previousNumberUrl; + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void calculatePercentsEtc(boolean isUpGood) + { + if(mainNumber != null && vsPreviousNumber != null && vsPreviousNumber.doubleValue() > 0) + { + ///////////////////////////////////////////////////////////////// + // these are the results we're going for: // + // current: 10, previous: 20 = -50% // + // current: 15, previous: 20 = -25% // + // current: 20, previous: 10 = +100% // + // current: 15, previous: 10 = +50% // + // this formula gets us that: (current - previous) / previous // + // (with a *100 in there to make it a percent-looking value) // + ///////////////////////////////////////////////////////////////// + BigDecimal current = new BigDecimal(String.valueOf(mainNumber)); + BigDecimal previous = new BigDecimal(String.valueOf(vsPreviousNumber)); + BigDecimal difference = current.subtract(previous); + BigDecimal ratio = difference.divide(previous, new MathContext(2, RoundingMode.HALF_UP)); + BigDecimal percentBD = ratio.multiply(new BigDecimal(100)); + Integer percent = Math.abs(percentBD.intValue()); + if(mainNumber.doubleValue() < vsPreviousNumber.doubleValue()) + { + setIsUpVsPrevious(false); + setIsGoodVsPrevious(isUpGood ? false : true); + setVsPreviousPercent(percent); + } + else // note - equal is being considered here in the good. + { + setIsUpVsPrevious(true); + setIsGoodVsPrevious(isUpGood ? true : false); + setVsPreviousPercent(percent); + } + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ParentWidgetData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ParentWidgetData.java index f75f032e..87338413 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ParentWidgetData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/ParentWidgetData.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData; /******************************************************************************* @@ -32,6 +33,7 @@ import java.util.List; public class ParentWidgetData extends QWidgetData { private List childWidgetNameList; + private ParentWidgetMetaData.LayoutType layoutType = ParentWidgetMetaData.LayoutType.GRID; @@ -87,4 +89,36 @@ public class ParentWidgetData extends QWidgetData return (this); } + + + /******************************************************************************* + ** Getter for layoutType + *******************************************************************************/ + public ParentWidgetMetaData.LayoutType getLayoutType() + { + return (this.layoutType); + } + + + + /******************************************************************************* + ** Setter for layoutType + *******************************************************************************/ + public void setLayoutType(ParentWidgetMetaData.LayoutType layoutType) + { + this.layoutType = layoutType; + } + + + + /******************************************************************************* + ** Fluent setter for layoutType + *******************************************************************************/ + public ParentWidgetData withLayoutType(ParentWidgetMetaData.LayoutType layoutType) + { + this.layoutType = layoutType; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/TableData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/TableData.java index d8330878..630e1cc5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/TableData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/TableData.java @@ -39,6 +39,8 @@ public class TableData extends QWidgetData private List> rows; private Integer rowsPerPage; private Boolean hidePaginationDropdown; + private Boolean fixedStickyLastRow = false; + private Integer fixedHeight; @@ -543,4 +545,67 @@ public class TableData extends QWidgetData } } + + + + /******************************************************************************* + ** Getter for fixedStickyLastRow + *******************************************************************************/ + public Boolean getFixedStickyLastRow() + { + return (this.fixedStickyLastRow); + } + + + + /******************************************************************************* + ** Setter for fixedStickyLastRow + *******************************************************************************/ + public void setFixedStickyLastRow(Boolean fixedStickyLastRow) + { + this.fixedStickyLastRow = fixedStickyLastRow; + } + + + + /******************************************************************************* + ** Fluent setter for fixedStickyLastRow + *******************************************************************************/ + public TableData withFixedStickyLastRow(Boolean fixedStickyLastRow) + { + this.fixedStickyLastRow = fixedStickyLastRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for fixedHeight + *******************************************************************************/ + public Integer getFixedHeight() + { + return (this.fixedHeight); + } + + + + /******************************************************************************* + ** Setter for fixedHeight + *******************************************************************************/ + public void setFixedHeight(Integer fixedHeight) + { + this.fixedHeight = fixedHeight; + } + + + + /******************************************************************************* + ** Fluent setter for fixedHeight + *******************************************************************************/ + public TableData withFixedHeight(Integer fixedHeight) + { + this.fixedHeight = fixedHeight; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java index 6cf505c0..1bcf60b6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java @@ -27,18 +27,22 @@ import java.math.BigDecimal; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.time.temporal.Temporal; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.utils.ValueUtils; -import org.apache.commons.lang.SerializationUtils; +import org.apache.commons.lang3.SerializationUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -61,6 +65,8 @@ import org.apache.commons.lang.SerializationUtils; *******************************************************************************/ public class QRecord implements Serializable { + private static final QLogger LOG = QLogger.getLogger(QRecord.class); + private String tableName; private String recordLabel; @@ -110,12 +116,14 @@ public class QRecord implements Serializable this.tableName = record.tableName; this.recordLabel = record.recordLabel; - this.values = doDeepCopy(record.values); - this.displayValues = doDeepCopy(record.displayValues); - this.backendDetails = doDeepCopy(record.backendDetails); - this.errors = doDeepCopy(record.errors); - this.warnings = doDeepCopy(record.warnings); - this.associatedRecords = doDeepCopy(record.associatedRecords); + this.values = deepCopySimpleMap(record.values); + this.displayValues = deepCopySimpleMap(record.displayValues); + this.backendDetails = deepCopySimpleMap(record.backendDetails); + + this.associatedRecords = deepCopyAssociatedRecords(record.associatedRecords); + + this.errors = record.errors == null ? null : new ArrayList<>(record.errors); + this.warnings = record.warnings == null ? null : new ArrayList<>(record.warnings); } @@ -135,40 +143,57 @@ public class QRecord implements Serializable ** todo - move to a cloning utils maybe? *******************************************************************************/ @SuppressWarnings({ "unchecked" }) - private Map doDeepCopy(Map map) + private Map deepCopySimpleMap(Map map) { if(map == null) { return (null); } - if(map instanceof Serializable serializableMap) + Map clone = new LinkedHashMap<>(); + for(Map.Entry entry : map.entrySet()) { - return (Map) SerializationUtils.clone(serializableMap); - } + V value = entry.getValue(); - return (new LinkedHashMap<>(map)); + ////////////////////////////////////////////////////////////////////////// + // not sure from where/how java.sql.Date objects are getting in here... // + ////////////////////////////////////////////////////////////////////////// + if(value == null || value instanceof String || value instanceof Number || value instanceof Boolean || value instanceof Temporal || value instanceof Date) + { + clone.put(entry.getKey(), entry.getValue()); + } + else if(entry.getValue() instanceof Serializable serializableValue) + { + LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass())); + clone.put(entry.getKey(), (V) SerializationUtils.clone(serializableValue)); + } + else + { + LOG.warn("Non-serializable value in QRecord...", logPair("key", entry.getKey()), logPair("type", value.getClass())); + clone.put(entry.getKey(), entry.getValue()); + } + } + return (clone); } /******************************************************************************* - ** todo - move to a cloning utils maybe? + ** *******************************************************************************/ - @SuppressWarnings({ "unchecked" }) - private List doDeepCopy(List list) + private Map> deepCopyAssociatedRecords(Map> input) { - if(list == null) + if(input == null) { return (null); } - if(list instanceof Serializable serializableList) + Map> clone = new HashMap<>(); + for(Map.Entry> entry : input.entrySet()) { - return (List) SerializationUtils.clone(serializableList); + clone.put(entry.getKey(), new ArrayList<>(entry.getValue())); } - - return (new ArrayList<>(list)); + return (clone); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index d8e678eb..d980f7f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -79,7 +79,7 @@ public class MetaDataProducerHelper } catch(Exception e) { - LOG.info("Error adding metaData from producer", logPair("producer", aClass.getSimpleName()), e); + LOG.info("Error adding metaData from producer", e, logPair("producer", aClass.getSimpleName())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java index bfedd7d1..77657937 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/ParentWidgetMetaData.java @@ -35,6 +35,16 @@ public class ParentWidgetMetaData extends QWidgetMetaData private List childWidgetNameList; private List childProcessNameList; + private LayoutType layoutType = LayoutType.GRID; + + + + public enum LayoutType + { + GRID, + TABS + } + /******************************************************************************* @@ -137,4 +147,36 @@ public class ParentWidgetMetaData extends QWidgetMetaData return (this); } + + + /******************************************************************************* + ** Getter for layoutType + *******************************************************************************/ + public LayoutType getLayoutType() + { + return (this.layoutType); + } + + + + /******************************************************************************* + ** Setter for layoutType + *******************************************************************************/ + public void setLayoutType(LayoutType layoutType) + { + this.layoutType = layoutType; + } + + + + /******************************************************************************* + ** Fluent setter for layoutType + *******************************************************************************/ + public ParentWidgetMetaData withLayoutType(LayoutType layoutType) + { + this.layoutType = layoutType; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java index fc528db8..c9453798 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaData.java @@ -28,6 +28,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; @@ -40,6 +41,7 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface protected String name; protected String icon; protected String label; + protected String tooltip; protected String type; protected String minHeight; protected String footerHTML; @@ -55,6 +57,8 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface private boolean showReloadButton = true; private boolean showExportButton = true; + protected Map icons; + protected Map defaultValues = new LinkedHashMap<>(); @@ -594,4 +598,81 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface return (this); } + + + /******************************************************************************* + ** Getter for icons + *******************************************************************************/ + public Map getIcons() + { + return (this.icons); + } + + + + /******************************************************************************* + ** Setter for icons + *******************************************************************************/ + public void setIcons(Map icons) + { + this.icons = icons; + } + + + + /******************************************************************************* + ** Fluent setter for icons + *******************************************************************************/ + public QWidgetMetaData withIcon(String role, QIcon icon) + { + if(this.icons == null) + { + this.icons = new LinkedHashMap<>(); + } + this.icons.put(role, icon); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for icons + *******************************************************************************/ + public QWidgetMetaData withIcons(Map icons) + { + this.icons = icons; + return (this); + } + + + + /******************************************************************************* + ** Getter for tooltip + *******************************************************************************/ + public String getTooltip() + { + return (this.tooltip); + } + + + + /******************************************************************************* + ** Setter for tooltip + *******************************************************************************/ + public void setTooltip(String tooltip) + { + this.tooltip = tooltip; + } + + + + /******************************************************************************* + ** Fluent setter for tooltip + *******************************************************************************/ + public QWidgetMetaData withTooltip(String tooltip) + { + this.tooltip = tooltip; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java index 0e4df559..1c3ad3db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QWidgetMetaDataInterface.java @@ -216,5 +216,15 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules ** Fluent setter for dropdowns *******************************************************************************/ QWidgetMetaData withDropdown(WidgetDropdownData dropdown); + + + /******************************************************************************* + ** Getter for tooltip + *******************************************************************************/ + default String getTooltip() + { + return (null); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java index a70c64d6..11845237 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/dashboard/QuickSightChartMetaData.java @@ -32,7 +32,7 @@ import java.util.Map; ** AWS Quicksite specific meta data for frontend dashboard widget ** *******************************************************************************/ -public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface +public class QuickSightChartMetaData extends QWidgetMetaData { private String accessKey; private String secretKey; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java index b33f4314..3d710251 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendWidgetMetaData.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend; import java.util.List; +import java.util.Map; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; @@ -30,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; /******************************************************************************* @@ -42,6 +44,7 @@ public class QFrontendWidgetMetaData { private final String name; private final String label; + private final String tooltip; private final String type; private final String icon; private final Integer gridColumns; @@ -54,10 +57,13 @@ public class QFrontendWidgetMetaData private boolean showReloadButton = false; private boolean showExportButton = false; + protected Map icons; + private final boolean hasPermission; ////////////////////////////////////////////////////////////////////////////////// - // do not add setters. take values from the source-object in the constructor!! // + // DO add getters for all fields - this tells Jackson to include them in JSON. // + // do NOT add setters. take values from the source-object in the constructor!! // ////////////////////////////////////////////////////////////////////////////////// @@ -69,6 +75,7 @@ public class QFrontendWidgetMetaData { this.name = widgetMetaData.getName(); this.label = widgetMetaData.getLabel(); + this.tooltip = widgetMetaData.getTooltip(); this.type = widgetMetaData.getType(); this.icon = widgetMetaData.getIcon(); this.gridColumns = widgetMetaData.getGridColumns(); @@ -82,6 +89,7 @@ public class QFrontendWidgetMetaData { this.showExportButton = qWidgetMetaData.getShowExportButton(); this.showReloadButton = qWidgetMetaData.getShowReloadButton(); + this.icons = qWidgetMetaData.getIcons(); } hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name); @@ -229,4 +237,26 @@ public class QFrontendWidgetMetaData { return showExportButton; } + + + + /******************************************************************************* + ** Getter for icons + ** + *******************************************************************************/ + public Map getIcons() + { + return icons; + } + + + + /******************************************************************************* + ** Getter for tooltip + ** + *******************************************************************************/ + public String getTooltip() + { + return tooltip; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java index cfc2aa40..72117679 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/layout/QIcon.java @@ -35,6 +35,7 @@ public class QIcon { private String name; private String path; + private String color; @@ -123,4 +124,36 @@ public class QIcon return (this); } + + + /******************************************************************************* + ** Getter for color + *******************************************************************************/ + public String getColor() + { + return (this.color); + } + + + + /******************************************************************************* + ** Setter for color + *******************************************************************************/ + public void setColor(String color) + { + this.color = color; + } + + + + /******************************************************************************* + ** Fluent setter for color + *******************************************************************************/ + public QIcon withColor(String color) + { + this.color = color; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java index c0d0076b..60463273 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java @@ -92,6 +92,21 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface + /******************************************************************************* + ** Create a new possible value source, for an enum, with default settings. + ** e.g., type=ENUM; name from param values from the param; LABEL_ONLY format + *******************************************************************************/ + public static > QPossibleValueSource newForEnum(String name, T[] values) + { + return new QPossibleValueSource() + .withName(name) + .withType(QPossibleValueSourceType.ENUM) + .withValuesFromEnum(values) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index d8850d40..3a83c955 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -82,6 +82,7 @@ public class BulkInsertExtractStep extends AbstractExtractStep .withRecordPipe(getRecordPipe()) .withLimit(getLimit()) .withCsv(new String(bytes)) + .withDoCorrectValueTypes(true) .withTable(runBackendStepInput.getInstance().getTable(tableName)) .withMapping(mapping) .withRecordCustomizer((record) -> diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 36b29c04..7f33a624 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -26,10 +26,15 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -48,6 +53,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -59,12 +65,36 @@ public class BulkInsertTransformStep extends AbstractTransformStep private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); - private Map ukErrorSummaries = new HashMap<>(); + private Map ukErrorSummaries = new HashMap<>(); private QTableMetaData table; private Map>> keysInThisFile = new HashMap<>(); + private int rowsProcessed = 0; + + + + /******************************************************************************* + ** extension of ProcessSummaryLine for lines where a UniqueKey was violated, + ** where we'll collect a sample (or maybe all) of the values that broke the UK. + *******************************************************************************/ + private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine + { + private Set sampleValues = new LinkedHashSet<>(); + private boolean areThereMoreSampleValues = false; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineWithUKSampleValues(Status status) + { + super(status); + } + } + /******************************************************************************* @@ -89,14 +119,48 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + int rowsInThisPage = runBackendStepInput.getRecords().size(); + QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName()); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + InsertInput insertInput = new InsertInput(); + insertInput.setInputSource(QInputSource.USER); + insertInput.setTableName(runBackendStepInput.getTableName()); + insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setSkipUniqueKeyCheck(true); + + ////////////////////////////////////////////////////////////////////// + // load the pre-insert customizer and set it up, if there is one // + // then we'll run it based on its WhenToRun value // + // we do this, in case it needs to, for example, adjust values that // + // are part of a unique key // + ////////////////////////////////////////////////////////////////////// + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + if(preInsertCustomizer.isPresent()) + { + preInsertCustomizer.get().setInsertInput(insertInput); + preInsertCustomizer.get().setIsPreview(true); + AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun(); + if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) + { + List recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords()); + runBackendStepInput.setRecords(recordsAfterCustomizer); + + /////////////////////////////////////////////////////////////////////////////////////// + // todo - do we care if the customizer runs both now, and in the validation below? // + // right now we'll let it run both times, but maybe that should be protected against // + /////////////////////////////////////////////////////////////////////////////////////// + } + } Map>> existingKeys = new HashMap<>(); List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); - ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR)); + ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -105,7 +169,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { - runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount())); + runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(rowsProcessed + 1)); } else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE)) { @@ -123,70 +187,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////// - // if there are no UK's, proceed with all records // - //////////////////////////////////////////////////// - List recordsWithoutUkErrors = new ArrayList<>(); - if(existingKeys.isEmpty()) - { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); - } - else - { - ///////////////////////////////////////////////////////////// - // else, only proceed with records that don't violate a UK // - ///////////////////////////////////////////////////////////// - for(UniqueKey uniqueKey : uniqueKeys) - { - keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); - } - - /////////////////////////////////////////////////////////////////////////// - // else, get each records keys and see if it already exists or not // - // also, build a set of keys we've seen (within this page (or overall?)) // - /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) - { - ////////////////////////////////////////////////////////// - // check if this record violates any of the unique keys // - ////////////////////////////////////////////////////////// - boolean foundDupe = false; - for(UniqueKey uniqueKey : uniqueKeys) - { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); - if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get()))) - { - ukErrorSummaries.get(uniqueKey).incrementCount(); - foundDupe = true; - break; - } - } - - /////////////////////////////////////////////////////////////////////////////// - // if this record doesn't violate any uk's, then we can add it to the output // - /////////////////////////////////////////////////////////////////////////////// - if(!foundDupe) - { - for(UniqueKey uniqueKey : uniqueKeys) - { - Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); - keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); - } - recordsWithoutUkErrors.add(record); - } - } - } + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - InsertAction insertAction = new InsertAction(); - InsertInput insertInput = new InsertInput(); - insertInput.setInputSource(QInputSource.USER); - insertInput.setTableName(runBackendStepInput.getTableName()); insertInput.setRecords(recordsWithoutUkErrors); - insertInput.setSkipUniqueKeyCheck(true); + InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); @@ -215,6 +222,89 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); + + this.rowsProcessed += rowsInThisPage; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + { + //////////////////////////////////////////////////// + // if there are no UK's, proceed with all records // + //////////////////////////////////////////////////// + List recordsWithoutUkErrors = new ArrayList<>(); + if(existingKeys.isEmpty()) + { + recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + } + else + { + ///////////////////////////////////////////////////////////// + // else, only proceed with records that don't violate a UK // + ///////////////////////////////////////////////////////////// + for(UniqueKey uniqueKey : uniqueKeys) + { + keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>()); + } + + /////////////////////////////////////////////////////////////////////////// + // else, get each records keys and see if it already exists or not // + // also, build a set of keys we've seen (within this page (or overall?)) // + /////////////////////////////////////////////////////////////////////////// + for(QRecord record : runBackendStepInput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + /////////////////////////////////////////////////// + // skip any records that may already be in error // + /////////////////////////////////////////////////// + recordsWithoutUkErrors.add(record); + continue; + } + + ////////////////////////////////////////////////////////// + // check if this record violates any of the unique keys // + ////////////////////////////////////////////////////////// + boolean foundDupe = false; + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get()))) + { + ProcessSummaryLineWithUKSampleValues processSummaryLineWithUKSampleValues = ukErrorSummaries.get(uniqueKey); + processSummaryLineWithUKSampleValues.incrementCount(); + if(processSummaryLineWithUKSampleValues.sampleValues.size() < 3) + { + processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString()); + } + else + { + processSummaryLineWithUKSampleValues.areThereMoreSampleValues = true; + } + foundDupe = true; + break; + } + } + + /////////////////////////////////////////////////////////////////////////////// + // if this record doesn't violate any uk's, then we can add it to the output // + /////////////////////////////////////////////////////////////////////////////// + if(!foundDupe) + { + for(UniqueKey uniqueKey : uniqueKeys) + { + Optional> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record); + keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv)); + } + recordsWithoutUkErrors.add(record); + } + } + } + return recordsWithoutUkErrors; } @@ -236,17 +326,20 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); - for(Map.Entry entry : ukErrorSummaries.entrySet()) + for(Map.Entry entry : ukErrorSummaries.entrySet()) { - UniqueKey uniqueKey = entry.getKey(); - ProcessSummaryLine ukErrorSummary = entry.getValue(); - String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")"; + UniqueKey uniqueKey = entry.getKey(); + ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue(); ukErrorSummary - .withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix) - .withPluralFutureMessage(tableLabel + " records will not be" + ukErrorSuffix) - .withSingularPastMessage(tableLabel + " record was not" + ukErrorSuffix) - .withPluralPastMessage(tableLabel + " records were not" + ukErrorSuffix); + .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + .withSingularFutureMessage(" record will not be") + .withPluralFutureMessage(" records will not be") + .withSingularPastMessage(" record was not") + .withPluralPastMessage(" records were not"); ukErrorSummary.addSelfToListIfAnyCount(rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/LoadInitialRecordsStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/LoadInitialRecordsStep.java index 164221b5..7055a6b7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/LoadInitialRecordsStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/LoadInitialRecordsStep.java @@ -49,14 +49,17 @@ public class LoadInitialRecordsStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - ///////////////////////////////////////////////////////////////////////////////////////////////// - // basically this is a no-op... we Just need a backendStep to be the first step in the process // - // but, while we're here, go ahead and put the query filter in the payload as a value, in case // - // someone else wants it (see BulkDelete) // - ///////////////////////////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // basically this is a no-op... sometimes we just need a backendStep to be the first step in a process. // + // While we're here, go ahead and put the query filter in the payload as a value - this is needed for // + // processes that have a screen before their first backend step (why is this needed? not sure, but is) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepInput.getAsyncJobCallback().updateStatus("Loading records"); - QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter(); - runBackendStepOutput.addValue("queryFilterJSON", JsonUtils.toJson(queryFilter)); + if(runBackendStepInput.getCallback() != null) + { + QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter(); + runBackendStepOutput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter)); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index ce0cb2ed..606a026b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -30,18 +30,20 @@ import java.util.Map; import java.util.Objects; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.utils.collections.AlphaNumericComparator; /******************************************************************************* ** Helper class for process steps that want to roll up error summary and/or ** warning summary lines. e.g., if the process might have a handful of different - ** error messages. Will record up to 50 unique errors, then throw the rest int + ** error messages. Will record up to 50 (configurable) unique errors, then throw the rest int ** an "other" errors summary. *******************************************************************************/ public class ProcessSummaryWarningsAndErrorsRollup { - private Map errorSummaries = new HashMap<>(); + private Map errorSummaries = new HashMap<>(); private Map warningSummaries = new HashMap<>(); private ProcessSummaryLine otherErrorsSummary; @@ -49,6 +51,8 @@ public class ProcessSummaryWarningsAndErrorsRollup private ProcessSummaryLine errorTemplate; private ProcessSummaryLine warningTemplate; + private int uniqueErrorsToShow = 50; + private boolean doReplaceSingletonCountLinesWithSuffixOnly = true; /******************************************************************************* @@ -167,7 +171,7 @@ public class ProcessSummaryWarningsAndErrorsRollup ProcessSummaryLine processSummaryLine = summaryLineMap.get(message); if(processSummaryLine == null) { - if(summaryLineMap.size() < 50) + if(summaryLineMap.size() < uniqueErrorsToShow) { processSummaryLine = new ProcessSummaryLine(status) .withMessageSuffix(message) @@ -210,17 +214,80 @@ public class ProcessSummaryWarningsAndErrorsRollup + /******************************************************************************* + ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that + ** extracts string messages out. + ** + ** Makes errors from bulk-insert look better when they report, e.g. + ** Error parsing line #1: ... + ** Error parsing line #2: ... + ** Error parsing line #10: ... + *******************************************************************************/ + private static class PSLAlphaNumericComparator implements Comparator + { + private static AlphaNumericComparator alphaNumericComparator = new AlphaNumericComparator(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public int compare(ProcessSummaryLineInterface psli1, ProcessSummaryLineInterface psli2) + { + int messageComp = (alphaNumericComparator.compare(Objects.requireNonNullElse(psli1.getMessage(), ""), Objects.requireNonNullElse(psli2.getMessage(), ""))); + if(messageComp != 0) + { + return (messageComp); + } + + if(psli1 instanceof ProcessSummaryLine psl1 && psli2 instanceof ProcessSummaryLine psl2) + { + return (alphaNumericComparator.compare(Objects.requireNonNullElse(psl1.getMessageSuffix(), ""), Objects.requireNonNullElse(psl2.getMessageSuffix(), ""))); + } + + return (0); + } + } + + + /******************************************************************************* ** sort the process summary lines by count desc *******************************************************************************/ - private static void addProcessSummaryLinesFromMap(ArrayList rs, Map summaryMap) + private void addProcessSummaryLinesFromMap(ArrayList rs, Map summaryMap) { summaryMap.values().stream() .sorted(Comparator.comparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getCount(), 0)).reversed() - .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessage(), "")) - .thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessageSuffix(), "")) + .thenComparing(new PSLAlphaNumericComparator()) ) - .forEach(psl -> psl.addSelfToListIfAnyCount(rs)); + .map(psl -> + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this is to make lines that are like "1 record had an error: Error parsing line #1: blah" look better, by // + // removing the redundant "1 record..." bit. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(doReplaceSingletonCountLinesWithSuffixOnly) + { + if(psl.getCount() == 1) + { + return (new ProcessSummaryRecordLink().withStatus(psl.getStatus()).withLinkPreText(psl.getMessageSuffix())); + } + } + + return (psl); + }) + .forEach(psli -> + { + if(psli instanceof ProcessSummaryLine psl) + { + psl.addSelfToListIfAnyCount(rs); + } + else + { + rs.add(psli); + } + }); } @@ -347,4 +414,67 @@ public class ProcessSummaryWarningsAndErrorsRollup return (this); } + + + /******************************************************************************* + ** Getter for uniqueErrorsToShow + *******************************************************************************/ + public int getUniqueErrorsToShow() + { + return (this.uniqueErrorsToShow); + } + + + + /******************************************************************************* + ** Setter for uniqueErrorsToShow + *******************************************************************************/ + public void setUniqueErrorsToShow(int uniqueErrorsToShow) + { + this.uniqueErrorsToShow = uniqueErrorsToShow; + } + + + + /******************************************************************************* + ** Fluent setter for uniqueErrorsToShow + *******************************************************************************/ + public ProcessSummaryWarningsAndErrorsRollup withUniqueErrorsToShow(int uniqueErrorsToShow) + { + this.uniqueErrorsToShow = uniqueErrorsToShow; + return (this); + } + + + + /******************************************************************************* + ** Getter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public boolean getDoReplaceSingletonCountLinesWithSuffixOnly() + { + return (this.doReplaceSingletonCountLinesWithSuffixOnly); + } + + + + /******************************************************************************* + ** Setter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public void setDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly) + { + this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly; + } + + + + /******************************************************************************* + ** Fluent setter for doReplaceSingletonCountLinesWithSuffixOnly + *******************************************************************************/ + public ProcessSummaryWarningsAndErrorsRollup withDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly) + { + this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index 6ecc2dc8..8adee919 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -95,7 +95,7 @@ public class ValueUtils } else if(value instanceof String s) { - return (Boolean.parseBoolean(s)); + return "true".equalsIgnoreCase(s) || "yes".equalsIgnoreCase(s); } else { @@ -496,6 +496,9 @@ public class ValueUtils *******************************************************************************/ private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) { + ////////////////////// + // 1999-12-31T12:59 // + ////////////////////// if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$")) { ////////////////////////// @@ -503,11 +506,34 @@ public class ValueUtils ////////////////////////// return Instant.parse(s + ":00Z"); } + + /////////////////////////// + // 1999-12-31 12:59:59.0 // + /////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$")) { s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z"); return Instant.parse(s); } + + ///////////////////////// + // 1999-12-31 12:59:59 // + ///////////////////////// + else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$")) + { + s = s.replaceAll(" ", "T") + "Z"; + return Instant.parse(s); + } + + ////////////////////// + // 1999-12-31 12:59 // + ////////////////////// + else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$")) + { + s = s.replaceAll(" ", "T") + ":00Z"; + return Instant.parse(s); + } + else { try diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java new file mode 100644 index 00000000..2eb32327 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparator.java @@ -0,0 +1,199 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Comparator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/******************************************************************************* + ** Comparator for strings that are a mix of alpha + numeric, where we want to + ** sort the numeric substrings like numbers. + ** + ** e.g., A1, A2, A10 won't come out as A1, A10, A2 + *******************************************************************************/ +public class AlphaNumericComparator implements Comparator +{ + private static final int A_FIRST = -1; + private static final int B_FIRST = 1; + private static final int TIE = 0; + + private static final Pattern INT_PATTERN = Pattern.compile("^\\d+$"); + private static final Pattern LEADING_INT_PATTERN = Pattern.compile("^\\d+"); + private static final Pattern ALPHA_THEN_INT_PATTERN = Pattern.compile("^(\\D+)\\d+"); + + + + /******************************************************************************* + ** compare 2 Strings + ** + *******************************************************************************/ + public int compare(String a, String b) + { + try + { + ////////////////////////////////////// + // eliminate degenerate cases first // + ////////////////////////////////////// + if(a == null && b == null) + { + return (TIE); + } + else if(a == null) + { + return (A_FIRST); + } + else if(b == null) + { + return (B_FIRST); + } + else if(a.equals(b)) + { + return (TIE); + } // also covers a == "" and b == "" + else if(a.equals("")) + { + return (A_FIRST); + } + else if(b.equals("")) + { + return (B_FIRST); + } + + //////////////////////////////////////////////////////////////// + // if both strings are pure numeric, parse as int and compare // + //////////////////////////////////////////////////////////////// + if(INT_PATTERN.matcher(a).matches() && INT_PATTERN.matcher(b).matches()) + { + int intsCompared = new Integer(a).compareTo(new Integer(b)); + if(intsCompared == TIE) + { + /////////////////////////////////////////////////////////////////////////////// + // in case the integers are the same (ie, "0001" vs "1"), compare as strings // + /////////////////////////////////////////////////////////////////////////////// + return (a.compareTo(b)); + } + else + { + /////////////////////////////////////////////////////////////// + // else, if the ints were different, return their comparison // + /////////////////////////////////////////////////////////////// + return (intsCompared); + } + } + + ///////////////////////////////////////////////////// + // if both start as numbers, extract those numbers // + ///////////////////////////////////////////////////// + Matcher aLeadingIntMatcher = LEADING_INT_PATTERN.matcher(a); + Matcher bLeadingIntMatcher = LEADING_INT_PATTERN.matcher(b); + if(aLeadingIntMatcher.lookingAt() && bLeadingIntMatcher.lookingAt()) + { + /////////////////////////// + // extract the int parts // + /////////////////////////// + String aIntPart = a.substring(0, aLeadingIntMatcher.end()); + String bIntPart = b.substring(0, bLeadingIntMatcher.end()); + + ///////////////////////////////////////////////////////////// + // if the ints compare as non-zero, return that comparison // + ///////////////////////////////////////////////////////////// + int intPartCompared = new Integer(aIntPart).compareTo(new Integer(bIntPart)); + if(intPartCompared != TIE) + { + return (intPartCompared); + } + else + { + ////////////////////////////////////////////////////////////////////// + // otherwise, make recursive call to compare the rest of the string // + ////////////////////////////////////////////////////////////////////// + String aRest = a.substring(aLeadingIntMatcher.end()); + String bRest = b.substring(bLeadingIntMatcher.end()); + return (compare(aRest, bRest)); + } + } + ////////////////////////////////////////////////////// + // if one starts as numeric, but other doesn't // + // return the one that starts with the number first // + ////////////////////////////////////////////////////// + else if(aLeadingIntMatcher.lookingAt()) + { + return (A_FIRST); + } + else if(bLeadingIntMatcher.lookingAt()) + { + return (B_FIRST); + } + + ////////////////////////////////////////////////////////////////////////// + // now, if both parts have an alpha part, followed by digit parts, and // + // the alpha parts are the same, then discard the alpha parts and recur // + ////////////////////////////////////////////////////////////////////////// + Matcher aAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(a); + Matcher bAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(b); + if(aAlphaThenIntMatcher.lookingAt() && bAlphaThenIntMatcher.lookingAt()) + { + String aAlphaPart = aAlphaThenIntMatcher.group(1); + String bAlphaPart = bAlphaThenIntMatcher.group(1); + + if(aAlphaPart.equals(bAlphaPart)) + { + String aRest = a.substring(aAlphaPart.length()); + String bRest = b.substring(bAlphaPart.length()); + return (compare(aRest, bRest)); + } + } + + ///////////////////////////////////////////////// + // as last resort, just do pure string compare // + ///////////////////////////////////////////////// + return (a.compareTo(b)); + } + catch(Exception e) + { + ////////////////////////////////////////////////////////// + // on exception, don't allow caller to catch -- rather, // + // always return something sensible (and null-safe) // + ////////////////////////////////////////////////////////// + if(a == null && b == null) + { + return (TIE); + } + else if(a == null) + { + return (A_FIRST); + } + else if(b == null) + { + return (B_FIRST); + } + else + { + return (a.compareTo(b)); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java new file mode 100644 index 00000000..e874c5e5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java @@ -0,0 +1,184 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.memoization; + + +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.logging.QLogger; + + +/******************************************************************************* + ** Basic memoization functionality - with result timeouts (only when doing a get - + ** there's no cleanup thread), and max-size. + *******************************************************************************/ +public class Memoization +{ + private static final QLogger LOG = QLogger.getLogger(Memoization.class); + + private final Map> map = Collections.synchronizedMap(new LinkedHashMap<>()); + + private Duration timeout = Duration.ofSeconds(600); + private Integer maxSize = 1000; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Memoization() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Optional getResult(K key) + { + MemoizedResult result = map.get(key); + if(result != null) + { + if(result.getTime().isAfter(Instant.now().minus(timeout))) + { + return (Optional.ofNullable(result.getResult())); + } + } + + return (Optional.empty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Optional> getMemoizedResult(K key) + { + MemoizedResult result = map.get(key); + if(result != null) + { + if(result.getTime().isAfter(Instant.now().minus(timeout))) + { + return (Optional.ofNullable(result)); + } + } + + return (Optional.empty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void storeResult(K key, V value) + { + map.put(key, new MemoizedResult<>(value)); + + ////////////////////////////////////// + // make sure map didn't get too big // + // do this thread safely, please // + ////////////////////////////////////// + try + { + if(map.size() > maxSize) + { + synchronized(map) + { + Iterator>> iterator = null; + while(map.size() > maxSize) + { + if(iterator == null) + { + iterator = map.entrySet().iterator(); + } + + if(iterator.hasNext()) + { + iterator.next(); + iterator.remove(); + } + else + { + break; + } + } + } + } + } + catch(Exception e) + { + LOG.error("Error managing size of a Memoization", e); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clear() + { + this.map.clear(); + } + + + + /******************************************************************************* + ** Setter for timeoutSeconds + ** + *******************************************************************************/ + public void setTimeout(Duration timeout) + { + this.timeout = timeout; + } + + + + /******************************************************************************* + ** Setter for maxSize + ** + *******************************************************************************/ + public void setMaxSize(Integer maxSize) + { + this.maxSize = maxSize; + } + + + + /******************************************************************************* + ** package-private - for tests to look at the map. + ** + *******************************************************************************/ + Map> getMap() + { + return map; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizedResult.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizedResult.java new file mode 100644 index 00000000..281e1425 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizedResult.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.memoization; + + +import java.time.Instant; + + +/******************************************************************************* + ** Object stored in the Memoization class. Shouldn't need to be visible outside + ** its package. + *******************************************************************************/ +public class MemoizedResult +{ + private T result; + private Instant time; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public MemoizedResult(T result) + { + this.result = result; + this.time = Instant.now(); + } + + + + /******************************************************************************* + ** Getter for result + ** + *******************************************************************************/ + public T getResult() + { + return result; + } + + + + /******************************************************************************* + ** Getter for time + ** + *******************************************************************************/ + public Instant getTime() + { + return time; + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java index 2fa44957..8ac80a30 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/InsertActionTest.java @@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; @@ -751,4 +753,28 @@ class InsertActionTest extends BaseTest } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDefaultValues() throws QException + { + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .getField("noOfShoes").withDefaultValue(2); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("noOfShoes", 4), + new QRecord().withValue("firstName", "Tim").withValue("lastName", "Chamberlain") + )); + new InsertAction().execute(insertInput); + + List records = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())).getRecords(); + assertEquals(4, records.get(0).getValueInteger("noOfShoes")); + assertEquals(2, records.get(1).getValueInteger("noOfShoes")); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index 52b9db39..893c2a41 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -121,6 +121,12 @@ class QValueFormatterTest extends BaseTest table = new QTableMetaData().withPrimaryKeyField("id"); assertEquals("42", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 42))); + /////////////////////////////////////////////////////////////////////////////////////// + // exceptional flow: no recordLabelFormat specified, and record already had a label // + /////////////////////////////////////////////////////////////////////////////////////// + table = new QTableMetaData().withPrimaryKeyField("id"); + assertEquals("my label", QValueFormatter.formatRecordLabel(table, new QRecord().withRecordLabel("my label").withValue("id", 42))); + ///////////////////////////////////////////////// // exceptional flow: no fields for the format // ///////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java index 8a15a6d8..ee2bb104 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java @@ -23,12 +23,18 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; +import java.time.LocalDate; +import java.time.Month; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; @@ -224,6 +230,74 @@ class SearchPossibleValueSourceActionTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_tableByIdOnlyNonNumeric() throws QException + { + QContext.getQInstance().getPossibleValueSource(TestUtils.TABLE_NAME_SHAPE) + .withSearchFields(List.of("id")); + + ///////////////////////////////////////////////////////////////////////////////////////////// + // a non-integer input should find nothing // + // the catch { (IN, empty) } code makes this happen - without that, all records are found. // + // (furthermore, i think that's only exposed if there's only 1 search field, maybe) // + ///////////////////////////////////////////////////////////////////////////////////////////// + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutput("A", TestUtils.TABLE_NAME_SHAPE); + assertEquals(0, output.getResults().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_tableByLocalDate() throws QException + { + MemoryRecordStore.getInstance().reset(); + + //////////////////////////////////////////// + // make a PVS for the person-memory table // + //////////////////////////////////////////// + QContext.getQInstance().addPossibleValueSource(QPossibleValueSource.newForTable(TestUtils.TABLE_NAME_PERSON_MEMORY) + .withSearchFields(List.of("id", "firstName", "birthDate")) + ); + + List shapeRecords = List.of( + new QRecord().withValue("id", 1).withValue("firstName", "Homer").withValue("birthDate", LocalDate.of(1960, Month.JANUARY, 1)), + new QRecord().withValue("id", 2).withValue("firstName", "Marge").withValue("birthDate", LocalDate.of(1961, Month.FEBRUARY, 2)), + new QRecord().withValue("id", 3).withValue("firstName", "Bart").withValue("birthDate", LocalDate.of(1980, Month.MARCH, 3))); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY); + insertInput.setRecords(shapeRecords); + new InsertAction().execute(insertInput); + + ///////////////////////////////////// + // a parseable date yields a match // + ///////////////////////////////////// + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutput("1960-01-01", TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(1, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1)); + + /////////////////////////////////////////////////////////////////////// + // alternative date format also works (thanks to ValueUtils parsing) // + /////////////////////////////////////////////////////////////////////// + output = getSearchPossibleValueSourceOutput("1/1/1960", TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(1, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1)); + + /////////////////////////////////// + // incomplete date finds nothing // + /////////////////////////////////// + output = getSearchPossibleValueSourceOutput("1960-01", TestUtils.TABLE_NAME_PERSON_MEMORY); + assertEquals(0, output.getResults().size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java index e1fba082..cb71a4fe 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/adapters/CsvToQRecordAdapterTest.java @@ -22,15 +22,19 @@ package com.kingsrook.qqq.backend.core.adapters; +import java.time.LocalDate; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -395,4 +399,70 @@ class CsvToQRecordAdapterTest extends BaseTest assertEquals("john@doe.com", records.get(0).getValueString("email")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withDoCorrectValueTypes(true) + .withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN))) + .withCsv(""" + firstName,birthDate,isEmployed + John,1/1/1980,true + Paul,1970-06-15,Yes + George,,anything-else + """)); + List qRecords = csvToQRecordAdapter.getRecordList(); + + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate")); + assertEquals(true, qRecord.getValue("isEmployed")); + + qRecord = qRecords.get(1); + assertEquals("Paul", qRecord.getValue("firstName")); + assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate")); + assertEquals(true, qRecord.getValue("isEmployed")); + + qRecord = qRecords.get(2); + assertEquals("George", qRecord.getValue("firstName")); + assertNull(qRecord.getValue("birthDate")); + assertEquals(false, qRecord.getValue("isEmployed")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException + { + CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter(); + csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() + .withDoCorrectValueTypes(true) + .withTable(TestUtils.defineTablePerson()) + .withCsv(""" + firstName,birthDate,favoriteShapeId + John,1980,1 + Paul,1970-06-15,green + """)); + List qRecords = csvToQRecordAdapter.getRecordList(); + + QRecord qRecord = qRecords.get(0); + assertEquals("John", qRecord.getValue("firstName")); + assertThat(qRecord.getErrors()).hasSize(1); + assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date"); + + qRecord = qRecords.get(1); + assertEquals("Paul", qRecord.getValue("firstName")); + assertThat(qRecord.getErrors()).hasSize(1); + assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer."); + } + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java new file mode 100644 index 00000000..fffd70b9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordTest.java @@ -0,0 +1,155 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.data; + + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS; +import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + + +/******************************************************************************* + ** Unit test for QRecord + *******************************************************************************/ +class QRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCopyConstructor() + { + String jsonValue = """ + {"key": [1,2]} + """; + Map fieldLengths = MapBuilder.of("a", 1, "b", 2); + + QRecord original = new QRecord() + .withTableName("myTable") + .withRecordLabel("My Record") + .withValue("one", 1) + .withValue("two", "two") + .withValue("three", new BigDecimal("3")) + .withValue("false", false) + .withValue("empty", null) + .withDisplayValue("three", "3.00") + .withBackendDetail(BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT, jsonValue) + .withBackendDetail(BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>(fieldLengths)) + .withError(new BadInputStatusMessage("Bad Input")) + .withAssociatedRecord("child", new QRecord().withValue("id", "child1")) + .withAssociatedRecord("child", new QRecord().withValue("id", "child2")) + .withAssociatedRecord("nephew", new QRecord().withValue("id", "nephew1")); + + QRecord clone = new QRecord(original); + + ////////////////////////////////////////////////////////////// + // assert equality on all the members values in the records // + ////////////////////////////////////////////////////////////// + assertEquals("myTable", clone.getTableName()); + assertEquals("My Record", clone.getRecordLabel()); + assertEquals(1, clone.getValue("one")); + assertEquals("two", clone.getValue("two")); + assertEquals(new BigDecimal("3"), clone.getValue("three")); + assertEquals(false, clone.getValue("false")); + assertNull(clone.getValue("empty")); + assertEquals("3.00", clone.getDisplayValue("three")); + assertEquals(jsonValue, clone.getBackendDetail(BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT)); + assertEquals(fieldLengths, clone.getBackendDetail(BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)); + assertEquals(1, clone.getErrors().size()); + assertEquals(BadInputStatusMessage.class, clone.getErrors().get(0).getClass()); + assertEquals("Bad Input", clone.getErrors().get(0).getMessage()); + assertEquals(0, clone.getWarnings().size()); + assertEquals(2, clone.getAssociatedRecords().size()); + assertEquals(2, clone.getAssociatedRecords().get("child").size()); + assertEquals("child1", clone.getAssociatedRecords().get("child").get(0).getValue("id")); + assertEquals("child2", clone.getAssociatedRecords().get("child").get(1).getValue("id")); + assertEquals(1, clone.getAssociatedRecords().get("nephew").size()); + assertEquals("nephew1", clone.getAssociatedRecords().get("nephew").get(0).getValue("id")); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure the associated record data structures are not the same (e.g., not the same map & lists) // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + assertNotSame(clone.getAssociatedRecords(), original.getAssociatedRecords()); + assertNotSame(clone.getAssociatedRecords().get("child"), original.getAssociatedRecords().get("child")); + + ///////////////////////////////////////////////////////////////////////////////////// + // but we'll be okay with the same records inside the associated records structure // + ///////////////////////////////////////////////////////////////////////////////////// + assertSame(clone.getAssociatedRecords().get("child").get(0), original.getAssociatedRecords().get("child").get(0)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCopyConstructorEdgeCases() + { + QRecord nullValuesRecord = new QRecord(); + nullValuesRecord.setValues(null); + assertNull(new QRecord(nullValuesRecord).getValues()); + + QRecord nullDisplayValuesRecord = new QRecord(); + nullDisplayValuesRecord.setDisplayValues(null); + assertNull(new QRecord(nullDisplayValuesRecord).getDisplayValues()); + + QRecord nullBackendDetailsRecord = new QRecord(); + nullBackendDetailsRecord.setBackendDetails(null); + assertNull(new QRecord(nullBackendDetailsRecord).getBackendDetails()); + + QRecord nullAssociations = new QRecord(); + nullAssociations.setAssociatedRecords(null); + assertNull(new QRecord(nullAssociations).getAssociatedRecords()); + + QRecord nullErrors = new QRecord(); + nullErrors.setErrors(null); + assertNull(new QRecord(nullErrors).getErrors()); + + QRecord nullWarnings = new QRecord(); + nullWarnings.setWarnings(null); + assertNull(new QRecord(nullWarnings).getWarnings()); + + QRecord emptyRecord = new QRecord(); + QRecord emptyClone = new QRecord(emptyRecord); + assertNull(emptyClone.getTableName()); + assertNull(emptyClone.getRecordLabel()); + assertEquals(0, emptyClone.getValues().size()); + assertEquals(0, emptyClone.getDisplayValues().size()); + assertEquals(0, emptyClone.getBackendDetails().size()); + assertEquals(0, emptyClone.getErrors().size()); + assertEquals(0, emptyClone.getWarnings().size()); + assertEquals(0, emptyClone.getAssociatedRecords().size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java index 35e81866..f8133223 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/edit/BulkEditTest.java @@ -32,6 +32,8 @@ import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.processes.Status; @@ -310,7 +312,7 @@ class BulkEditTest extends BaseTest runProcessOutput = new RunProcessAction().execute(runProcessInput); @SuppressWarnings("unchecked") - List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + List processSummaryLines = (List) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); assertThat(processSummaryLines).hasSize(1 + 50 + 1 + 30 + 1); int index = 0; @@ -323,7 +325,7 @@ class BulkEditTest extends BaseTest { assertThat(processSummaryLines.get(index++)) .hasFieldOrPropertyWithValue("status", Status.ERROR) - .hasFieldOrPropertyWithValue("count", 1) + .isInstanceOf(ProcessSummaryRecordLink.class) // this is because it's a singleton, so we get rid of the "1 had an error" thing (doReplaceSingletonCountLinesWithSuffixOnly) .matches(psl -> psl.getMessage().contains("less than 60 is error"), "expected message"); } @@ -336,7 +338,7 @@ class BulkEditTest extends BaseTest { assertThat(processSummaryLines.get(index++)) .hasFieldOrPropertyWithValue("status", Status.WARNING) - .hasFieldOrPropertyWithValue("count", 1) + .isInstanceOf(ProcessSummaryRecordLink.class) // this is because it's a singleton, so we get rid of the "1 had an error" thing (doReplaceSingletonCountLinesWithSuffixOnly) .matches(psl -> psl.getMessage().contains("less than 90 is warning"), "expected message"); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java index c9c1603d..496ad0be 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java @@ -62,6 +62,42 @@ class BulkInsertTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" + """); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -77,7 +113,7 @@ class BulkInsertTest extends BaseTest // create an uploaded file, similar to how an http server may // //////////////////////////////////////////////////////////////// QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); + qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); qUploadedFile.setFilename("test.csv"); UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 12512dfb..bf1d0825 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -1252,41 +1252,6 @@ public class TestUtils - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvHeaderUsingLabels() - { - return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow1() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow2() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" - """); - } - - /******************************************************************************* ** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index cf7f857f..5508a0ee 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -83,7 +83,15 @@ class ValueUtilsTest extends BaseTest assertTrue(ValueUtils.getValueAsBoolean("True")); assertTrue(ValueUtils.getValueAsBoolean("TRUE")); assertFalse(ValueUtils.getValueAsBoolean("false")); - assertFalse(ValueUtils.getValueAsBoolean("yes")); + + /////////////////////////////////////////////////////////////////////// + // time used to be, that "yes" was false... changing that 2023-10-20 // + /////////////////////////////////////////////////////////////////////// + assertTrue(ValueUtils.getValueAsBoolean("yes")); + assertTrue(ValueUtils.getValueAsBoolean("Yes")); + assertTrue(ValueUtils.getValueAsBoolean("YES")); + assertTrue(ValueUtils.getValueAsBoolean("yES")); + assertFalse(ValueUtils.getValueAsBoolean("t")); assertFalse(ValueUtils.getValueAsBoolean(new Object())); assertFalse(ValueUtils.getValueAsBoolean(1)); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java new file mode 100644 index 00000000..a61f2abd --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/AlphaNumericComparatorTest.java @@ -0,0 +1,149 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for AlphaNumericComparator + *******************************************************************************/ +class AlphaNumericComparatorTest extends BaseTest +{ + + /******************************************************************************* + ** test odd-balls + ** + *******************************************************************************/ + @Test + public void testFringeCases() + { + test(sort("", null, "foo", " ", "", "1", null), + null, null, "", "", "1", " ", "foo"); + } + + + + /******************************************************************************* + ** test alpha-strings only + ** + *******************************************************************************/ + @Test + public void testAlphasOnly() + { + test(sort("F", "G", "A", "AB", "BB", "BA", "BD"), + "A", "AB", "BA", "BB", "BD", "F", "G"); + } + + + + /******************************************************************************* + ** test numbers only + ** + *******************************************************************************/ + @Test + public void testNumbersOnly() + { + test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"), + "1", "2", "10", "13", "22", "101", "102", "271", "273", "273"); + } + + + + /******************************************************************************* + ** test mixed + ** + *******************************************************************************/ + @Test + public void testMixed1() + { + test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"), + "001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10"); + } + + + + /******************************************************************************* + ** test mixed + ** + *******************************************************************************/ + @Test + public void testMixed2() + { + test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"), + "0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1"); + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + private void test(List a, String... b) + { + System.out.println("Expecting: " + Arrays.asList(b)); + + assertEquals(a.size(), b.length); + + for(int i = 0; i < a.size(); i++) + { + String aString = a.get(i); + String bString = b[i]; + + assertEquals(aString, bString); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + private List sort(String... input) + { + List inputList = Arrays.asList(input); + System.out.println("Sorting: " + inputList); + + try + { + List naturalSortList = Arrays.asList(input); + Collections.sort(naturalSortList); + System.out.println("Natural: " + naturalSortList); + } + catch(Exception e) + { + System.out.println("Natural: FAILED"); + } + + inputList.sort(new AlphaNumericComparator()); + System.out.println("Produced: " + inputList); + return (inputList); + } +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java new file mode 100644 index 00000000..85683482 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/memoization/MemoizationTest.java @@ -0,0 +1,174 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.memoization; + + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for Memoization + *******************************************************************************/ +class MemoizationTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + Memoization memoization = new Memoization<>(); + memoization.setMaxSize(3); + memoization.setTimeout(Duration.ofMillis(100)); + + assertThat(memoization.getResult("one")).isEmpty(); + memoization.storeResult("one", 1); + assertThat(memoization.getResult("one")).isPresent().get().isEqualTo(1); + + //////////////////////////////////////////////////// + // store 3 more results - this should force 1 out // + //////////////////////////////////////////////////// + memoization.storeResult("two", 2); + memoization.storeResult("three", 3); + memoization.storeResult("four", 4); + assertThat(memoization.getResult("one")).isEmpty(); + + ////////////////////////////////// + // make sure others are present // + ////////////////////////////////// + assertThat(memoization.getResult("two")).isPresent().get().isEqualTo(2); + assertThat(memoization.getResult("three")).isPresent().get().isEqualTo(3); + assertThat(memoization.getResult("four")).isPresent().get().isEqualTo(4); + + ///////////////////////////////////////////////////////////// + // wait more than the timeout, then make sure all are gone // + ///////////////////////////////////////////////////////////// + SleepUtils.sleep(150, TimeUnit.MILLISECONDS); + assertThat(memoization.getResult("two")).isEmpty(); + assertThat(memoization.getResult("three")).isEmpty(); + assertThat(memoization.getResult("four")).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCanStoreNull() + { + Memoization memoization = new Memoization<>(); + memoization.storeResult("null", null); + + /////////////////////////////////////////////////////////////////////////////////////////// + // note - we can't tell a stored null apart from a non-stored value by calling getResult // + /////////////////////////////////////////////////////////////////////////////////////////// + Optional optionalNull = memoization.getResult("null"); + assertNotNull(optionalNull); + assertTrue(optionalNull.isEmpty()); + + //////////////////////////////////////////// + // instead, we must use getMemoizedResult // + //////////////////////////////////////////// + Optional> optionalMemoizedResult = memoization.getMemoizedResult("null"); + assertNotNull(optionalMemoizedResult); + assertTrue(optionalMemoizedResult.isPresent()); + assertNull(optionalMemoizedResult.get().getResult()); + + ///////////////////////////////////////////////////////////////// + // make sure getMemoizedResult returns empty for an un-set key // + ///////////////////////////////////////////////////////////////// + optionalMemoizedResult = memoization.getMemoizedResult("never-stored"); + assertNotNull(optionalMemoizedResult); + assertTrue(optionalMemoizedResult.isEmpty()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @Disabled("Slow, so not for CI - but good to demonstrate thread-safety during dev") + void testMultiThread() throws InterruptedException, ExecutionException + { + Memoization memoization = new Memoization<>(); + ExecutorService executorService = Executors.newFixedThreadPool(20); + + List> futures = new ArrayList<>(); + + for(int i = 0; i < 20; i++) + { + int finalI = i; + futures.add(executorService.submit(() -> + { + System.out.println("Start " + finalI); + for(int n = 0; n < 1_000_000; n++) + { + memoization.storeResult(String.valueOf(n), n); + memoization.getResult(String.valueOf(n)); + + if(n % 100_000 == 0) + { + System.out.format("Thread %d at %,d\n", finalI, +n); + } + } + System.out.println("End " + finalI); + })); + } + + while(!futures.isEmpty()) + { + Iterator> iterator = futures.iterator(); + while(iterator.hasNext()) + { + Future next = iterator.next(); + if(next.isDone()) + { + Object o = next.get(); + iterator.remove(); + } + } + } + + System.out.println("All Done"); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 75078c8a..4aed4e14 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -149,7 +149,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface ** and type conversions that we can do "better" than jdbc... ** *******************************************************************************/ - protected Serializable scrubValue(QFieldMetaData field, Serializable value, boolean isInsert) + protected Serializable scrubValue(QFieldMetaData field, Serializable value) { if("".equals(value)) { @@ -724,9 +724,10 @@ public abstract class AbstractRDBMSAction implements QActionInterface throw new IllegalArgumentException("Incorrect number of values given for criteria [" + field.getName() + "]"); } - ////////////////////////////////////////////////////////////// - // replace any expression-type values with their evaluation // - ////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // replace any expression-type values with their evaluation // + // also, "scrub" non-expression values, which type-converts them (e.g., strings in various supported date formats become LocalDate) // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ListIterator valueListIterator = values.listIterator(); while(valueListIterator.hasNext()) { @@ -735,6 +736,11 @@ public abstract class AbstractRDBMSAction implements QActionInterface { valueListIterator.set(expression.evaluate()); } + else + { + Serializable scrubbedValue = scrubValue(field, value); + valueListIterator.set(scrubbedValue); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 5074bfdc..9dfc16ea 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -135,7 +135,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QFieldMetaData field : insertableFields) { Serializable value = record.getValue(field.getName()); - value = scrubValue(field, value, true); + value = scrubValue(field, value); params.add(value); } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 1783f419..e53e5b8b 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -214,7 +214,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte for(String fieldName : fieldsBeingUpdated) { Serializable value = record.getValue(fieldName); - value = scrubValue(table.getField(fieldName), value, false); + value = scrubValue(table.getField(fieldName), value); rowValues.add(value); } rowValues.add(record.getValue(table.getPrimaryKeyField())); @@ -286,7 +286,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte for(String fieldName : fieldsBeingUpdated) { Serializable value = record0.getValue(fieldName); - value = scrubValue(table.getField(fieldName), value, false); + value = scrubValue(table.getField(fieldName), value); params.add(value); }