mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-20 14:10:44 +00:00
Compare commits
3 Commits
snapshot-f
...
snapshot-f
| Author | SHA1 | Date | |
|---|---|---|---|
| d977abfc3d | |||
| c559cc21eb | |||
| 8ebe776847 |
@ -102,14 +102,14 @@ jobs:
|
||||
mvn_test:
|
||||
executor: localstack/default
|
||||
steps:
|
||||
## - localstack/startup
|
||||
- localstack/startup
|
||||
- install_java17
|
||||
- mvn_verify
|
||||
|
||||
mvn_deploy:
|
||||
executor: localstack/default
|
||||
steps:
|
||||
## - localstack/startup
|
||||
- localstack/startup
|
||||
- install_java17
|
||||
- mvn_verify
|
||||
- mvn_jar_deploy
|
||||
|
||||
2
pom.xml
2
pom.xml
@ -44,7 +44,7 @@
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<revision>0.20.0-SNAPSHOT</revision>
|
||||
<revision>0.19.0-SNAPSHOT</revision>
|
||||
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
|
||||
@ -84,7 +84,7 @@
|
||||
<dependency>
|
||||
<groupId>org.json</groupId>
|
||||
<artifactId>json</artifactId>
|
||||
<version>20230618</version>
|
||||
<version>20230227</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
||||
@ -319,10 +319,8 @@ public class DMLAuditAction extends AbstractQActionFunction<DMLAuditInput, DMLAu
|
||||
|
||||
if(detailRecord != null)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// useful if doing dev in here - but overkill for any other time. //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
|
||||
LOG.debug("Returning with message: " + detailRecord.getValueString("message"));
|
||||
|
||||
detailRecord.withValue("fieldName", fieldName);
|
||||
return (Optional.of(detailRecord));
|
||||
}
|
||||
|
||||
@ -55,21 +55,6 @@ public abstract class AbstractPreInsertCustomizer
|
||||
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// allow the customizer to specify when it should be executed as part of the //
|
||||
// insert action. default (per method in this class) is AFTER_ALL_VALIDATIONS //
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
public enum WhenToRun
|
||||
{
|
||||
BEFORE_ALL_VALIDATIONS,
|
||||
BEFORE_UNIQUE_KEY_CHECKS,
|
||||
BEFORE_REQUIRED_FIELD_CHECKS,
|
||||
BEFORE_SECURITY_CHECKS,
|
||||
AFTER_ALL_VALIDATIONS
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -77,16 +62,6 @@ public abstract class AbstractPreInsertCustomizer
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public WhenToRun getWhenToRun()
|
||||
{
|
||||
return (WhenToRun.AFTER_ALL_VALIDATIONS);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for insertInput
|
||||
**
|
||||
|
||||
@ -99,17 +99,7 @@ public enum DateTimeGroupBy
|
||||
public String getSqlExpression()
|
||||
{
|
||||
ZoneId sessionOrInstanceZoneId = ValueUtils.getSessionOrInstanceZoneId();
|
||||
return (getSqlExpression(sessionOrInstanceZoneId));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getSqlExpression(ZoneId targetZoneId)
|
||||
{
|
||||
String targetTimezone = targetZoneId.toString();
|
||||
String targetTimezone = sessionOrInstanceZoneId.toString();
|
||||
|
||||
if("Z".equals(targetTimezone) || !StringUtils.hasContent(targetTimezone))
|
||||
{
|
||||
@ -168,18 +158,7 @@ public enum DateTimeGroupBy
|
||||
*******************************************************************************/
|
||||
public String makeSelectedString(Instant time)
|
||||
{
|
||||
return (makeSelectedString(time, ValueUtils.getSessionOrInstanceZoneId()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make an Instant into a string that will match what came out of the database's
|
||||
** DATE_FORMAT() function
|
||||
*******************************************************************************/
|
||||
public String makeSelectedString(Instant time, ZoneId zoneId)
|
||||
{
|
||||
ZonedDateTime zoned = time.atZone(zoneId);
|
||||
ZonedDateTime zoned = time.atZone(ValueUtils.getSessionOrInstanceZoneId());
|
||||
|
||||
if(this == WEEK)
|
||||
{
|
||||
@ -203,17 +182,7 @@ public enum DateTimeGroupBy
|
||||
*******************************************************************************/
|
||||
public String makeHumanString(Instant instant)
|
||||
{
|
||||
return (makeHumanString(instant, ValueUtils.getSessionOrInstanceZoneId()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Make a string to show to a user
|
||||
*******************************************************************************/
|
||||
public String makeHumanString(Instant instant, ZoneId zoneId)
|
||||
{
|
||||
ZonedDateTime zoned = instant.atZone(zoneId);
|
||||
ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
|
||||
if(this.equals(WEEK))
|
||||
{
|
||||
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("M'/'d");
|
||||
@ -246,35 +215,25 @@ public enum DateTimeGroupBy
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("checkstyle:indentation")
|
||||
public Instant roundDown(Instant instant)
|
||||
{
|
||||
return roundDown(instant, ValueUtils.getSessionOrInstanceZoneId());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("checkstyle:indentation")
|
||||
public Instant roundDown(Instant instant, ZoneId zoneId)
|
||||
{
|
||||
ZonedDateTime zoned = instant.atZone(zoneId);
|
||||
ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
|
||||
return switch(this)
|
||||
{
|
||||
case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case WEEK ->
|
||||
{
|
||||
while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
|
||||
case YEAR -> zoned.with(TemporalAdjusters.firstDayOfYear()).truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case MONTH -> zoned.with(TemporalAdjusters.firstDayOfMonth()).truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case WEEK ->
|
||||
{
|
||||
zoned = zoned.minusDays(1);
|
||||
while(zoned.get(ChronoField.DAY_OF_WEEK) != DayOfWeek.SUNDAY.getValue())
|
||||
{
|
||||
zoned = zoned.minusDays(1);
|
||||
}
|
||||
yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
|
||||
}
|
||||
yield (zoned.truncatedTo(ChronoUnit.DAYS).toInstant());
|
||||
}
|
||||
case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
|
||||
};
|
||||
case DAY -> zoned.truncatedTo(ChronoUnit.DAYS).toInstant();
|
||||
case HOUR -> zoned.truncatedTo(ChronoUnit.HOURS).toInstant();
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -284,17 +243,7 @@ public enum DateTimeGroupBy
|
||||
*******************************************************************************/
|
||||
public Instant increment(Instant instant)
|
||||
{
|
||||
return (increment(instant, ValueUtils.getSessionOrInstanceZoneId()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Instant increment(Instant instant, ZoneId zoneId)
|
||||
{
|
||||
ZonedDateTime zoned = instant.atZone(zoneId);
|
||||
ZonedDateTime zoned = instant.atZone(ValueUtils.getSessionOrInstanceZoneId());
|
||||
return (zoned.plus(noOfChronoUnitsToAdd, chronoUnitToAdd).toInstant());
|
||||
}
|
||||
}
|
||||
|
||||
@ -60,8 +60,6 @@ public class ParentWidgetRenderer extends AbstractWidgetRenderer
|
||||
widgetData.setChildWidgetNameList(metaData.getChildWidgetNameList());
|
||||
}
|
||||
|
||||
widgetData.setLayoutType(metaData.getLayoutType());
|
||||
|
||||
return (new RenderWidgetOutput(widgetData));
|
||||
}
|
||||
catch(Exception e)
|
||||
|
||||
@ -28,7 +28,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -56,7 +55,7 @@ public interface GetInterface
|
||||
{
|
||||
QTableMetaData table = getInput.getTable();
|
||||
boolean foundMatch = false;
|
||||
for(UniqueKey uniqueKey : CollectionUtils.nonNullList(table.getUniqueKeys()))
|
||||
for(UniqueKey uniqueKey : table.getUniqueKeys())
|
||||
{
|
||||
if(new HashSet<>(uniqueKey.getFieldNames()).equals(getInput.getUniqueKey().keySet()))
|
||||
{
|
||||
|
||||
@ -193,76 +193,25 @@ public class InsertAction extends AbstractQActionFunction<InsertInput, InsertOut
|
||||
{
|
||||
QTableMetaData table = insertInput.getTable();
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// load the pre-insert customizer and set it up, if there is one //
|
||||
// then we'll run it based on its WhenToRun value //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
|
||||
if(preInsertCustomizer.isPresent())
|
||||
{
|
||||
preInsertCustomizer.get().setInsertInput(insertInput);
|
||||
preInsertCustomizer.get().setIsPreview(isPreview);
|
||||
runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS);
|
||||
}
|
||||
|
||||
setDefaultValuesInRecords(table, insertInput.getRecords());
|
||||
|
||||
ValueBehaviorApplier.applyFieldBehaviors(insertInput.getInstance(), table, insertInput.getRecords());
|
||||
|
||||
runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_UNIQUE_KEY_CHECKS);
|
||||
setErrorsIfUniqueKeyErrors(insertInput, table);
|
||||
|
||||
runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_REQUIRED_FIELD_CHECKS);
|
||||
if(insertInput.getInputSource().shouldValidateRequiredFields())
|
||||
{
|
||||
validateRequiredFields(insertInput);
|
||||
}
|
||||
|
||||
runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_SECURITY_CHECKS);
|
||||
ValidateRecordSecurityLockHelper.validateSecurityFields(insertInput.getTable(), insertInput.getRecords(), ValidateRecordSecurityLockHelper.Action.INSERT);
|
||||
|
||||
runPreInsertCustomizerIfItIsTime(insertInput, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.AFTER_ALL_VALIDATIONS);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setDefaultValuesInRecords(QTableMetaData table, List<QRecord> records)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// for all fields in the table - if any have a default value, then look at all input records, //
|
||||
// and if they have null value, then apply the default //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
if(field.getDefaultValue() != null)
|
||||
{
|
||||
for(QRecord record : records)
|
||||
{
|
||||
if(record.getValue(field.getName()) == null)
|
||||
{
|
||||
record.setValue(field.getName(), field.getDefaultValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void runPreInsertCustomizerIfItIsTime(InsertInput insertInput, Optional<AbstractPreInsertCustomizer> preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun whenToRun) throws QException
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// after all validations, run the pre-insert customizer, if there is one //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
|
||||
if(preInsertCustomizer.isPresent())
|
||||
{
|
||||
if(whenToRun.equals(preInsertCustomizer.get().getWhenToRun()))
|
||||
{
|
||||
insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
|
||||
}
|
||||
preInsertCustomizer.get().setInsertInput(insertInput);
|
||||
preInsertCustomizer.get().setIsPreview(isPreview);
|
||||
insertInput.setRecords(preInsertCustomizer.get().apply(insertInput.getRecords()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -32,14 +32,11 @@ import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
@ -77,47 +74,9 @@ public class QPossibleValueTranslator
|
||||
///////////////////////////////////////////////////////
|
||||
private Map<String, Map<Serializable, String>> possibleValueCache = new HashMap<>();
|
||||
|
||||
private int maxSizePerPvsCache = 50_000;
|
||||
|
||||
private Map<String, QBackendTransaction> transactionsPerTable = new HashMap<>();
|
||||
|
||||
// todo not commit - remove instance & session - use Context
|
||||
|
||||
|
||||
boolean useTransactionsAsConnectionPool = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QBackendTransaction getTransaction(String tableName)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////
|
||||
// mmm, this does cut down on connections used - //
|
||||
// especially seems helpful in big exports. //
|
||||
// but, let's just start using connection pools instead... //
|
||||
/////////////////////////////////////////////////////////////
|
||||
if(useTransactionsAsConnectionPool)
|
||||
{
|
||||
try
|
||||
{
|
||||
if(!transactionsPerTable.containsKey(tableName))
|
||||
{
|
||||
transactionsPerTable.put(tableName, new InsertAction().openTransaction(new InsertInput(tableName)));
|
||||
}
|
||||
|
||||
return (transactionsPerTable.get(tableName));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Error opening transaction for table", logPair("tableName", tableName));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
@ -466,10 +425,9 @@ public class QPossibleValueTranslator
|
||||
for(Map.Entry<String, Map<Serializable, String>> entry : possibleValueCache.entrySet())
|
||||
{
|
||||
int size = entry.getValue().size();
|
||||
if(size > maxSizePerPvsCache)
|
||||
if(size > 50_000)
|
||||
{
|
||||
LOG.info("Found a big PVS cache - clearing it.", logPair("name", entry.getKey()), logPair("size", size));
|
||||
entry.getValue().clear();
|
||||
}
|
||||
}
|
||||
|
||||
@ -563,7 +521,6 @@ public class QPossibleValueTranslator
|
||||
QueryInput queryInput = new QueryInput();
|
||||
queryInput.setTableName(tableName);
|
||||
queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria(primaryKeyField, QCriteriaOperator.IN, page)));
|
||||
queryInput.setTransaction(getTransaction(tableName));
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// when querying for possible values, we do want to generate their display values, which makes record labels, which are usually used as PVS labels //
|
||||
@ -656,24 +613,4 @@ public class QPossibleValueTranslator
|
||||
return (count < 5);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for maxSizePerPvsCache
|
||||
*******************************************************************************/
|
||||
public int getMaxSizePerPvsCache()
|
||||
{
|
||||
return (this.maxSizePerPvsCache);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for maxSizePerPvsCache
|
||||
*******************************************************************************/
|
||||
public void setMaxSizePerPvsCache(int maxSizePerPvsCache)
|
||||
{
|
||||
this.maxSizePerPvsCache = maxSizePerPvsCache;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -112,12 +112,6 @@ public class QValueFormatter
|
||||
{
|
||||
return formatLocalTime(lt);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// else, just return the value as a string, rather than going through String.formatted //
|
||||
// this saves some overhead incurred by String.formatted when called millions of times. //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
return (ValueUtils.getValueAsString(value));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////
|
||||
@ -274,14 +268,6 @@ public class QValueFormatter
|
||||
*******************************************************************************/
|
||||
private static String formatRecordLabelExceptionalCases(QTableMetaData table, QRecord record)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
// if the record already has a label (say, from a query-customizer), then return it //
|
||||
//////////////////////////////////////////////////////////////////////////////////////
|
||||
if(record.getRecordLabel() != null)
|
||||
{
|
||||
return (record.getRecordLabel());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// if there's no record label format, then just return the primary key display value //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.values;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
@ -210,50 +208,37 @@ public class SearchPossibleValueSourceAction
|
||||
}
|
||||
else
|
||||
{
|
||||
String searchTerm = input.getSearchTerm();
|
||||
if(StringUtils.hasContent(searchTerm))
|
||||
if(StringUtils.hasContent(input.getSearchTerm()))
|
||||
{
|
||||
for(String valueField : possibleValueSource.getSearchFields())
|
||||
{
|
||||
try
|
||||
QFieldMetaData field = table.getField(valueField);
|
||||
if(field.getType().equals(QFieldType.STRING))
|
||||
{
|
||||
QFieldMetaData field = table.getField(valueField);
|
||||
if(field.getType().equals(QFieldType.STRING))
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
|
||||
}
|
||||
else if(field.getType().equals(QFieldType.DATE) || field.getType().equals(QFieldType.DATE_TIME))
|
||||
{
|
||||
LOG.debug("Not querying PVS [" + possibleValueSource.getName() + "] on date field [" + field.getName() + "]");
|
||||
// todo - what? queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(input.getSearchTerm())));
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.STARTS_WITH, List.of(searchTerm)));
|
||||
}
|
||||
else if(field.getType().equals(QFieldType.DATE))
|
||||
{
|
||||
LocalDate searchDate = ValueUtils.getValueAsLocalDate(searchTerm);
|
||||
if(searchDate != null)
|
||||
{
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
|
||||
}
|
||||
}
|
||||
else if(field.getType().equals(QFieldType.DATE_TIME))
|
||||
{
|
||||
Instant searchDate = ValueUtils.getValueAsInstant(searchTerm);
|
||||
if(searchDate != null)
|
||||
{
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, searchDate));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Integer valueAsInteger = ValueUtils.getValueAsInteger(searchTerm);
|
||||
Integer valueAsInteger = ValueUtils.getValueAsInteger(input.getSearchTerm());
|
||||
if(valueAsInteger != null)
|
||||
{
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.EQUALS, List.of(valueAsInteger)));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
// write a FALSE criteria upon exceptions (e.g., type conversion fails) //
|
||||
// Why are we doing this? so a single-field query finds nothing instead of everything. //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
|
||||
catch(Exception e)
|
||||
{
|
||||
////////////////////////////////////////////////////////
|
||||
// write a FALSE criteria if the value isn't a number //
|
||||
////////////////////////////////////////////////////////
|
||||
queryFilter.addCriteria(new QFilterCriteria(valueField, QCriteriaOperator.IN, List.of()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,9 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.AbstractQFiel
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import org.apache.commons.csv.CSVFormat;
|
||||
import org.apache.commons.csv.CSVParser;
|
||||
import org.apache.commons.csv.CSVRecord;
|
||||
@ -158,21 +156,14 @@ public class CsvToQRecordAdapter
|
||||
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
QRecord qRecord = new QRecord();
|
||||
try
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
|
||||
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
|
||||
setValue(inputWrapper, qRecord, field, csvValues.get(fieldSource));
|
||||
}
|
||||
String fieldSource = mapping == null ? field.getName() : String.valueOf(mapping.getFieldSource(field.getName()));
|
||||
fieldSource = adjustHeaderCase(fieldSource, inputWrapper);
|
||||
qRecord.setValue(field.getName(), csvValues.get(fieldSource));
|
||||
}
|
||||
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
|
||||
}
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
addRecord(qRecord);
|
||||
|
||||
recordCount++;
|
||||
@ -211,20 +202,13 @@ public class CsvToQRecordAdapter
|
||||
// now move values into the QRecord, using the mapping to get the 'header' corresponding to each QField //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
QRecord qRecord = new QRecord();
|
||||
try
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
|
||||
setValue(inputWrapper, qRecord, field, csvValues.get(fieldIndex));
|
||||
}
|
||||
Integer fieldIndex = (Integer) mapping.getFieldSource(field.getName());
|
||||
qRecord.setValue(field.getName(), csvValues.get(fieldIndex));
|
||||
}
|
||||
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
qRecord.addError(new BadInputStatusMessage("Error parsing line #" + (recordCount + 1) + ": " + e.getMessage()));
|
||||
}
|
||||
runRecordCustomizer(recordCustomizer, qRecord);
|
||||
addRecord(qRecord);
|
||||
|
||||
recordCount++;
|
||||
@ -247,23 +231,6 @@ public class CsvToQRecordAdapter
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void setValue(InputWrapper inputWrapper, QRecord qRecord, QFieldMetaData field, String valueString)
|
||||
{
|
||||
if(inputWrapper.doCorrectValueTypes)
|
||||
{
|
||||
qRecord.setValue(field.getName(), ValueUtils.getValueAsFieldType(field.getType(), valueString));
|
||||
}
|
||||
else
|
||||
{
|
||||
qRecord.setValue(field.getName(), valueString);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -374,7 +341,6 @@ public class CsvToQRecordAdapter
|
||||
private AbstractQFieldMapping<?> mapping;
|
||||
private Consumer<QRecord> recordCustomizer;
|
||||
private Integer limit;
|
||||
private boolean doCorrectValueTypes = false;
|
||||
|
||||
private boolean caseSensitiveHeaders = false;
|
||||
|
||||
@ -616,40 +582,6 @@ public class CsvToQRecordAdapter
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public boolean getDoCorrectValueTypes()
|
||||
{
|
||||
return doCorrectValueTypes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setDoCorrectValueTypes(boolean doCorrectValueTypes)
|
||||
{
|
||||
this.doCorrectValueTypes = doCorrectValueTypes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for doCorrectValueTypes
|
||||
**
|
||||
*******************************************************************************/
|
||||
public InputWrapper withDoCorrectValueTypes(boolean doCorrectValueTypes)
|
||||
{
|
||||
this.doCorrectValueTypes = doCorrectValueTypes;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -80,23 +80,8 @@ public class QRecordToCsvAdapter
|
||||
/*******************************************************************************
|
||||
** todo - kinda weak... can we find this in a CSV lib??
|
||||
*******************************************************************************/
|
||||
static String sanitize(String value)
|
||||
private String sanitize(String value)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////////
|
||||
// especially in big exports, we see a TON of memory allocated and CPU spent here, //
|
||||
// if we just blindly replaceAll. So, only do it if needed. //
|
||||
/////////////////////////////////////////////////////////////////////////////////////
|
||||
if(value.contains("\""))
|
||||
{
|
||||
value = value.replaceAll("\"", "\"\"");
|
||||
}
|
||||
|
||||
if(value.contains("\n"))
|
||||
{
|
||||
value = value.replaceAll("\n", " ");
|
||||
}
|
||||
|
||||
return (value);
|
||||
return (value.replaceAll("\"", "\"\"").replaceAll("\n", " "));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -124,7 +124,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void log(Level level, String message)
|
||||
{
|
||||
logger.log(level, () -> makeJsonString(message));
|
||||
logger.log(level, makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -134,7 +134,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void log(Level level, String message, Throwable t)
|
||||
{
|
||||
logger.log(level, () -> makeJsonString(message, t));
|
||||
logger.log(level, makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -144,7 +144,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void log(Level level, String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.log(level, () -> makeJsonString(message, t, logPairs));
|
||||
logger.log(level, makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -154,7 +154,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void log(Level level, Throwable t)
|
||||
{
|
||||
logger.log(level, () -> makeJsonString(null, t));
|
||||
logger.log(level, makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
@ -164,7 +164,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void trace(String message)
|
||||
{
|
||||
logger.trace(() -> makeJsonString(message));
|
||||
logger.trace(makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -174,7 +174,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void trace(String message, LogPair... logPairs)
|
||||
{
|
||||
logger.trace(() -> makeJsonString(message, null, logPairs));
|
||||
logger.trace(makeJsonString(message, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -194,7 +194,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void trace(String message, Throwable t)
|
||||
{
|
||||
logger.trace(() -> makeJsonString(message, t));
|
||||
logger.trace(makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -204,7 +204,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void trace(String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.trace(() -> makeJsonString(message, t, logPairs));
|
||||
logger.trace(makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -214,7 +214,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void trace(Throwable t)
|
||||
{
|
||||
logger.trace(() -> makeJsonString(null, t));
|
||||
logger.trace(makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
@ -224,7 +224,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void debug(String message)
|
||||
{
|
||||
logger.debug(() -> makeJsonString(message));
|
||||
logger.debug(makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -234,7 +234,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void debug(String message, LogPair... logPairs)
|
||||
{
|
||||
logger.debug(() -> makeJsonString(message, null, logPairs));
|
||||
logger.debug(makeJsonString(message, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -254,7 +254,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void debug(String message, Throwable t)
|
||||
{
|
||||
logger.debug(() -> makeJsonString(message, t));
|
||||
logger.debug(makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -264,7 +264,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void debug(String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.debug(() -> makeJsonString(message, t, logPairs));
|
||||
logger.debug(makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -274,7 +274,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void debug(Throwable t)
|
||||
{
|
||||
logger.debug(() -> makeJsonString(null, t));
|
||||
logger.debug(makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
@ -284,7 +284,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(String message)
|
||||
{
|
||||
logger.info(() -> makeJsonString(message));
|
||||
logger.info(makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -294,7 +294,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(LogPair... logPairs)
|
||||
{
|
||||
logger.info(() -> makeJsonString(null, null, logPairs));
|
||||
logger.info(makeJsonString(null, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -304,7 +304,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(List<LogPair> logPairList)
|
||||
{
|
||||
logger.info(() -> makeJsonString(null, null, logPairList));
|
||||
logger.info(makeJsonString(null, null, logPairList));
|
||||
}
|
||||
|
||||
|
||||
@ -314,7 +314,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(String message, LogPair... logPairs)
|
||||
{
|
||||
logger.info(() -> makeJsonString(message, null, logPairs));
|
||||
logger.info(makeJsonString(message, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -334,7 +334,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(String message, Throwable t)
|
||||
{
|
||||
logger.info(() -> makeJsonString(message, t));
|
||||
logger.info(makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -344,7 +344,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.info(() -> makeJsonString(message, t, logPairs));
|
||||
logger.info(makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -354,7 +354,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void info(Throwable t)
|
||||
{
|
||||
logger.info(() -> makeJsonString(null, t));
|
||||
logger.info(makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
@ -364,7 +364,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void warn(String message)
|
||||
{
|
||||
logger.warn(() -> makeJsonString(message));
|
||||
logger.warn(makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -374,7 +374,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void warn(String message, LogPair... logPairs)
|
||||
{
|
||||
logger.warn(() -> makeJsonString(message, null, logPairs));
|
||||
logger.warn(makeJsonString(message, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -394,7 +394,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void warn(String message, Throwable t)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -404,7 +404,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void warn(String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(message, t, logPairs));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -414,7 +414,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void warn(Throwable t)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), () -> makeJsonString(null, t));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.WARN), makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
@ -424,7 +424,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void error(String message)
|
||||
{
|
||||
logger.error(() -> makeJsonString(message));
|
||||
logger.error(makeJsonString(message));
|
||||
}
|
||||
|
||||
|
||||
@ -434,7 +434,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void error(String message, LogPair... logPairs)
|
||||
{
|
||||
logger.error(() -> makeJsonString(message, null, logPairs));
|
||||
logger.error(makeJsonString(message, null, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -454,7 +454,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void error(String message, Throwable t)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t));
|
||||
}
|
||||
|
||||
|
||||
@ -464,7 +464,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void error(String message, Throwable t, LogPair... logPairs)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(message, t, logPairs));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(message, t, logPairs));
|
||||
}
|
||||
|
||||
|
||||
@ -474,7 +474,7 @@ public class QLogger
|
||||
*******************************************************************************/
|
||||
public void error(Throwable t)
|
||||
{
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), () -> makeJsonString(null, t));
|
||||
logger.log(determineIfShouldDowngrade(t, Level.ERROR), makeJsonString(null, t));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -28,9 +28,6 @@ import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.actions.audits.AuditAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLockFilters;
|
||||
@ -55,41 +52,6 @@ public class AuditSingleInput
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public AuditSingleInput()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public AuditSingleInput(QTableMetaData table, QRecord record, String auditMessage)
|
||||
{
|
||||
setAuditTableName(table.getName());
|
||||
setRecordId(record.getValueInteger(table.getPrimaryKeyField()));
|
||||
setSecurityKeyValues(AuditAction.getRecordSecurityKeyValues(table, record, Optional.empty()));
|
||||
setMessage(auditMessage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public AuditSingleInput(String tableName, QRecord record, String auditMessage)
|
||||
{
|
||||
this(QContext.getQInstance().getTable(tableName), record, auditMessage);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for auditTableName
|
||||
*******************************************************************************/
|
||||
|
||||
@ -131,17 +131,6 @@ public class ProcessSummaryFilterLink implements ProcessSummaryLineInterface
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getMessage()
|
||||
{
|
||||
return getFullText();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for status
|
||||
**
|
||||
|
||||
@ -182,7 +182,6 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface
|
||||
** Getter for message
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getMessage()
|
||||
{
|
||||
return message;
|
||||
|
||||
@ -38,10 +38,6 @@ public interface ProcessSummaryLineInterface extends Serializable
|
||||
*******************************************************************************/
|
||||
Status getStatus();
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
String getMessage();
|
||||
|
||||
/*******************************************************************************
|
||||
** meant to be called by framework, after process is complete, give the
|
||||
|
||||
@ -95,17 +95,6 @@ public class ProcessSummaryRecordLink implements ProcessSummaryLineInterface
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getMessage()
|
||||
{
|
||||
return getFullText();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -48,8 +48,6 @@ public class ChartData extends QWidgetData
|
||||
private boolean isCurrency = false;
|
||||
private int height;
|
||||
|
||||
private ChartSubheaderData chartSubheaderData;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -389,7 +387,6 @@ public class ChartData extends QWidgetData
|
||||
private String color;
|
||||
private String backgroundColor;
|
||||
private List<String> urls;
|
||||
private List<String> backgroundColors;
|
||||
|
||||
|
||||
|
||||
@ -426,17 +423,6 @@ public class ChartData extends QWidgetData
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for backgroundColors
|
||||
**
|
||||
*******************************************************************************/
|
||||
public List<String> getBackgroundColors()
|
||||
{
|
||||
return backgroundColors;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for backgroundColor
|
||||
**
|
||||
@ -448,17 +434,6 @@ public class ChartData extends QWidgetData
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for backgroundColor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setBackgroundColors(List<String> backgroundColors)
|
||||
{
|
||||
this.backgroundColors = backgroundColors;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for backgroundColor
|
||||
**
|
||||
@ -471,18 +446,6 @@ public class ChartData extends QWidgetData
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for backgroundColor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Dataset withBackgroundColors(List<String> backgroundColors)
|
||||
{
|
||||
this.backgroundColors = backgroundColors;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for color
|
||||
**
|
||||
@ -596,36 +559,4 @@ public class ChartData extends QWidgetData
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for chartSubheaderData
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData getChartSubheaderData()
|
||||
{
|
||||
return (this.chartSubheaderData);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for chartSubheaderData
|
||||
*******************************************************************************/
|
||||
public void setChartSubheaderData(ChartSubheaderData chartSubheaderData)
|
||||
{
|
||||
this.chartSubheaderData = chartSubheaderData;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for chartSubheaderData
|
||||
*******************************************************************************/
|
||||
public ChartData withChartSubheaderData(ChartSubheaderData chartSubheaderData)
|
||||
{
|
||||
this.chartSubheaderData = chartSubheaderData;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,331 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
|
||||
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.MathContext;
|
||||
import java.math.RoundingMode;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class ChartSubheaderData
|
||||
{
|
||||
private Number mainNumber;
|
||||
private Number vsPreviousPercent;
|
||||
private Number vsPreviousNumber;
|
||||
private Boolean isUpVsPrevious;
|
||||
private Boolean isGoodVsPrevious;
|
||||
private String vsDescription = "vs prev period";
|
||||
|
||||
private String mainNumberUrl;
|
||||
private String previousNumberUrl;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for mainNumber
|
||||
*******************************************************************************/
|
||||
public Number getMainNumber()
|
||||
{
|
||||
return (this.mainNumber);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for mainNumber
|
||||
*******************************************************************************/
|
||||
public void setMainNumber(Number mainNumber)
|
||||
{
|
||||
this.mainNumber = mainNumber;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for mainNumber
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withMainNumber(Number mainNumber)
|
||||
{
|
||||
this.mainNumber = mainNumber;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for vsPreviousNumber
|
||||
*******************************************************************************/
|
||||
public Number getVsPreviousNumber()
|
||||
{
|
||||
return (this.vsPreviousNumber);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for vsPreviousNumber
|
||||
*******************************************************************************/
|
||||
public void setVsPreviousNumber(Number vsPreviousNumber)
|
||||
{
|
||||
this.vsPreviousNumber = vsPreviousNumber;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for vsPreviousNumber
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withVsPreviousNumber(Number vsPreviousNumber)
|
||||
{
|
||||
this.vsPreviousNumber = vsPreviousNumber;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for vsDescription
|
||||
*******************************************************************************/
|
||||
public String getVsDescription()
|
||||
{
|
||||
return (this.vsDescription);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for vsDescription
|
||||
*******************************************************************************/
|
||||
public void setVsDescription(String vsDescription)
|
||||
{
|
||||
this.vsDescription = vsDescription;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for vsDescription
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withVsDescription(String vsDescription)
|
||||
{
|
||||
this.vsDescription = vsDescription;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for vsPreviousPercent
|
||||
*******************************************************************************/
|
||||
public Number getVsPreviousPercent()
|
||||
{
|
||||
return (this.vsPreviousPercent);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for vsPreviousPercent
|
||||
*******************************************************************************/
|
||||
public void setVsPreviousPercent(Number vsPreviousPercent)
|
||||
{
|
||||
this.vsPreviousPercent = vsPreviousPercent;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for vsPreviousPercent
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withVsPreviousPercent(Number vsPreviousPercent)
|
||||
{
|
||||
this.vsPreviousPercent = vsPreviousPercent;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for isUpVsPrevious
|
||||
*******************************************************************************/
|
||||
public Boolean getIsUpVsPrevious()
|
||||
{
|
||||
return (this.isUpVsPrevious);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for isUpVsPrevious
|
||||
*******************************************************************************/
|
||||
public void setIsUpVsPrevious(Boolean isUpVsPrevious)
|
||||
{
|
||||
this.isUpVsPrevious = isUpVsPrevious;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for isUpVsPrevious
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withIsUpVsPrevious(Boolean isUpVsPrevious)
|
||||
{
|
||||
this.isUpVsPrevious = isUpVsPrevious;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for isGoodVsPrevious
|
||||
*******************************************************************************/
|
||||
public Boolean getIsGoodVsPrevious()
|
||||
{
|
||||
return (this.isGoodVsPrevious);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for isGoodVsPrevious
|
||||
*******************************************************************************/
|
||||
public void setIsGoodVsPrevious(Boolean isGoodVsPrevious)
|
||||
{
|
||||
this.isGoodVsPrevious = isGoodVsPrevious;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for isGoodVsPrevious
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withIsGoodVsPrevious(Boolean isGoodVsPrevious)
|
||||
{
|
||||
this.isGoodVsPrevious = isGoodVsPrevious;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for mainNumberUrl
|
||||
*******************************************************************************/
|
||||
public String getMainNumberUrl()
|
||||
{
|
||||
return (this.mainNumberUrl);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for mainNumberUrl
|
||||
*******************************************************************************/
|
||||
public void setMainNumberUrl(String mainNumberUrl)
|
||||
{
|
||||
this.mainNumberUrl = mainNumberUrl;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for mainNumberUrl
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withMainNumberUrl(String mainNumberUrl)
|
||||
{
|
||||
this.mainNumberUrl = mainNumberUrl;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for previousNumberUrl
|
||||
*******************************************************************************/
|
||||
public String getPreviousNumberUrl()
|
||||
{
|
||||
return (this.previousNumberUrl);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for previousNumberUrl
|
||||
*******************************************************************************/
|
||||
public void setPreviousNumberUrl(String previousNumberUrl)
|
||||
{
|
||||
this.previousNumberUrl = previousNumberUrl;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for previousNumberUrl
|
||||
*******************************************************************************/
|
||||
public ChartSubheaderData withPreviousNumberUrl(String previousNumberUrl)
|
||||
{
|
||||
this.previousNumberUrl = previousNumberUrl;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void calculatePercentsEtc(boolean isUpGood)
|
||||
{
|
||||
if(mainNumber != null && vsPreviousNumber != null && vsPreviousNumber.doubleValue() > 0)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// these are the results we're going for: //
|
||||
// current: 10, previous: 20 = -50% //
|
||||
// current: 15, previous: 20 = -25% //
|
||||
// current: 20, previous: 10 = +100% //
|
||||
// current: 15, previous: 10 = +50% //
|
||||
// this formula gets us that: (current - previous) / previous //
|
||||
// (with a *100 in there to make it a percent-looking value) //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
BigDecimal current = new BigDecimal(String.valueOf(mainNumber));
|
||||
BigDecimal previous = new BigDecimal(String.valueOf(vsPreviousNumber));
|
||||
BigDecimal difference = current.subtract(previous);
|
||||
BigDecimal ratio = difference.divide(previous, new MathContext(2, RoundingMode.HALF_UP));
|
||||
BigDecimal percentBD = ratio.multiply(new BigDecimal(100));
|
||||
Integer percent = Math.abs(percentBD.intValue());
|
||||
if(mainNumber.doubleValue() < vsPreviousNumber.doubleValue())
|
||||
{
|
||||
setIsUpVsPrevious(false);
|
||||
setIsGoodVsPrevious(isUpGood ? false : true);
|
||||
setVsPreviousPercent(percent);
|
||||
}
|
||||
else // note - equal is being considered here in the good.
|
||||
{
|
||||
setIsUpVsPrevious(true);
|
||||
setIsGoodVsPrevious(isUpGood ? true : false);
|
||||
setVsPreviousPercent(percent);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.model.dashboard.widgets;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -33,7 +32,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.ParentWidgetMetaD
|
||||
public class ParentWidgetData extends QWidgetData
|
||||
{
|
||||
private List<String> childWidgetNameList;
|
||||
private ParentWidgetMetaData.LayoutType layoutType = ParentWidgetMetaData.LayoutType.GRID;
|
||||
|
||||
|
||||
|
||||
@ -89,36 +87,4 @@ public class ParentWidgetData extends QWidgetData
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for layoutType
|
||||
*******************************************************************************/
|
||||
public ParentWidgetMetaData.LayoutType getLayoutType()
|
||||
{
|
||||
return (this.layoutType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for layoutType
|
||||
*******************************************************************************/
|
||||
public void setLayoutType(ParentWidgetMetaData.LayoutType layoutType)
|
||||
{
|
||||
this.layoutType = layoutType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for layoutType
|
||||
*******************************************************************************/
|
||||
public ParentWidgetData withLayoutType(ParentWidgetMetaData.LayoutType layoutType)
|
||||
{
|
||||
this.layoutType = layoutType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -39,8 +39,6 @@ public class TableData extends QWidgetData
|
||||
private List<Map<String, Object>> rows;
|
||||
private Integer rowsPerPage;
|
||||
private Boolean hidePaginationDropdown;
|
||||
private Boolean fixedStickyLastRow = false;
|
||||
private Integer fixedHeight;
|
||||
|
||||
|
||||
|
||||
@ -545,67 +543,4 @@ public class TableData extends QWidgetData
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fixedStickyLastRow
|
||||
*******************************************************************************/
|
||||
public Boolean getFixedStickyLastRow()
|
||||
{
|
||||
return (this.fixedStickyLastRow);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for fixedStickyLastRow
|
||||
*******************************************************************************/
|
||||
public void setFixedStickyLastRow(Boolean fixedStickyLastRow)
|
||||
{
|
||||
this.fixedStickyLastRow = fixedStickyLastRow;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fixedStickyLastRow
|
||||
*******************************************************************************/
|
||||
public TableData withFixedStickyLastRow(Boolean fixedStickyLastRow)
|
||||
{
|
||||
this.fixedStickyLastRow = fixedStickyLastRow;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for fixedHeight
|
||||
*******************************************************************************/
|
||||
public Integer getFixedHeight()
|
||||
{
|
||||
return (this.fixedHeight);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for fixedHeight
|
||||
*******************************************************************************/
|
||||
public void setFixedHeight(Integer fixedHeight)
|
||||
{
|
||||
this.fixedHeight = fixedHeight;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for fixedHeight
|
||||
*******************************************************************************/
|
||||
public TableData withFixedHeight(Integer fixedHeight)
|
||||
{
|
||||
this.fixedHeight = fixedHeight;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -27,22 +27,18 @@ import java.math.BigDecimal;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.LocalTime;
|
||||
import java.time.temporal.Temporal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import org.apache.commons.lang3.SerializationUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
import org.apache.commons.lang.SerializationUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -65,8 +61,6 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
*******************************************************************************/
|
||||
public class QRecord implements Serializable
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(QRecord.class);
|
||||
|
||||
private String tableName;
|
||||
private String recordLabel;
|
||||
|
||||
@ -116,14 +110,12 @@ public class QRecord implements Serializable
|
||||
this.tableName = record.tableName;
|
||||
this.recordLabel = record.recordLabel;
|
||||
|
||||
this.values = deepCopySimpleMap(record.values);
|
||||
this.displayValues = deepCopySimpleMap(record.displayValues);
|
||||
this.backendDetails = deepCopySimpleMap(record.backendDetails);
|
||||
|
||||
this.associatedRecords = deepCopyAssociatedRecords(record.associatedRecords);
|
||||
|
||||
this.errors = record.errors == null ? null : new ArrayList<>(record.errors);
|
||||
this.warnings = record.warnings == null ? null : new ArrayList<>(record.warnings);
|
||||
this.values = doDeepCopy(record.values);
|
||||
this.displayValues = doDeepCopy(record.displayValues);
|
||||
this.backendDetails = doDeepCopy(record.backendDetails);
|
||||
this.errors = doDeepCopy(record.errors);
|
||||
this.warnings = doDeepCopy(record.warnings);
|
||||
this.associatedRecords = doDeepCopy(record.associatedRecords);
|
||||
}
|
||||
|
||||
|
||||
@ -143,57 +135,40 @@ public class QRecord implements Serializable
|
||||
** todo - move to a cloning utils maybe?
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings({ "unchecked" })
|
||||
private <K, V> Map<K, V> deepCopySimpleMap(Map<K, V> map)
|
||||
private <K, V> Map<K, V> doDeepCopy(Map<K, V> map)
|
||||
{
|
||||
if(map == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
Map<K, V> clone = new LinkedHashMap<>();
|
||||
for(Map.Entry<K, V> entry : map.entrySet())
|
||||
if(map instanceof Serializable serializableMap)
|
||||
{
|
||||
V value = entry.getValue();
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// not sure from where/how java.sql.Date objects are getting in here... //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
if(value == null || value instanceof String || value instanceof Number || value instanceof Boolean || value instanceof Temporal || value instanceof Date)
|
||||
{
|
||||
clone.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
else if(entry.getValue() instanceof Serializable serializableValue)
|
||||
{
|
||||
LOG.info("Non-primitive serializable value in QRecord - calling SerializationUtils.clone...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
|
||||
clone.put(entry.getKey(), (V) SerializationUtils.clone(serializableValue));
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.warn("Non-serializable value in QRecord...", logPair("key", entry.getKey()), logPair("type", value.getClass()));
|
||||
clone.put(entry.getKey(), entry.getValue());
|
||||
}
|
||||
return (Map<K, V>) SerializationUtils.clone(serializableMap);
|
||||
}
|
||||
return (clone);
|
||||
|
||||
return (new LinkedHashMap<>(map));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
** todo - move to a cloning utils maybe?
|
||||
*******************************************************************************/
|
||||
private Map<String, List<QRecord>> deepCopyAssociatedRecords(Map<String, List<QRecord>> input)
|
||||
@SuppressWarnings({ "unchecked" })
|
||||
private <T> List<T> doDeepCopy(List<T> list)
|
||||
{
|
||||
if(input == null)
|
||||
if(list == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
Map<String, List<QRecord>> clone = new HashMap<>();
|
||||
for(Map.Entry<String, List<QRecord>> entry : input.entrySet())
|
||||
if(list instanceof Serializable serializableList)
|
||||
{
|
||||
clone.put(entry.getKey(), new ArrayList<>(entry.getValue()));
|
||||
return (List<T>) SerializationUtils.clone(serializableList);
|
||||
}
|
||||
return (clone);
|
||||
|
||||
return (new ArrayList<>(list));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -79,7 +79,7 @@ public class MetaDataProducerHelper
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Error adding metaData from producer", e, logPair("producer", aClass.getSimpleName()));
|
||||
LOG.info("Error adding metaData from producer", logPair("producer", aClass.getSimpleName()), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -35,16 +35,6 @@ public class ParentWidgetMetaData extends QWidgetMetaData
|
||||
private List<String> childWidgetNameList;
|
||||
private List<String> childProcessNameList;
|
||||
|
||||
private LayoutType layoutType = LayoutType.GRID;
|
||||
|
||||
|
||||
|
||||
public enum LayoutType
|
||||
{
|
||||
GRID,
|
||||
TABS
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -147,36 +137,4 @@ public class ParentWidgetMetaData extends QWidgetMetaData
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for layoutType
|
||||
*******************************************************************************/
|
||||
public LayoutType getLayoutType()
|
||||
{
|
||||
return (this.layoutType);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for layoutType
|
||||
*******************************************************************************/
|
||||
public void setLayoutType(LayoutType layoutType)
|
||||
{
|
||||
this.layoutType = layoutType;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for layoutType
|
||||
*******************************************************************************/
|
||||
public ParentWidgetMetaData withLayoutType(LayoutType layoutType)
|
||||
{
|
||||
this.layoutType = layoutType;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -28,7 +28,6 @@ import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
|
||||
|
||||
@ -41,7 +40,6 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
|
||||
protected String name;
|
||||
protected String icon;
|
||||
protected String label;
|
||||
protected String tooltip;
|
||||
protected String type;
|
||||
protected String minHeight;
|
||||
protected String footerHTML;
|
||||
@ -57,8 +55,6 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
|
||||
private boolean showReloadButton = true;
|
||||
private boolean showExportButton = true;
|
||||
|
||||
protected Map<String, QIcon> icons;
|
||||
|
||||
protected Map<String, Serializable> defaultValues = new LinkedHashMap<>();
|
||||
|
||||
|
||||
@ -598,81 +594,4 @@ public class QWidgetMetaData implements QWidgetMetaDataInterface
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for icons
|
||||
*******************************************************************************/
|
||||
public Map<String, QIcon> getIcons()
|
||||
{
|
||||
return (this.icons);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for icons
|
||||
*******************************************************************************/
|
||||
public void setIcons(Map<String, QIcon> icons)
|
||||
{
|
||||
this.icons = icons;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for icons
|
||||
*******************************************************************************/
|
||||
public QWidgetMetaData withIcon(String role, QIcon icon)
|
||||
{
|
||||
if(this.icons == null)
|
||||
{
|
||||
this.icons = new LinkedHashMap<>();
|
||||
}
|
||||
this.icons.put(role, icon);
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for icons
|
||||
*******************************************************************************/
|
||||
public QWidgetMetaData withIcons(Map<String, QIcon> icons)
|
||||
{
|
||||
this.icons = icons;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tooltip
|
||||
*******************************************************************************/
|
||||
public String getTooltip()
|
||||
{
|
||||
return (this.tooltip);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for tooltip
|
||||
*******************************************************************************/
|
||||
public void setTooltip(String tooltip)
|
||||
{
|
||||
this.tooltip = tooltip;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for tooltip
|
||||
*******************************************************************************/
|
||||
public QWidgetMetaData withTooltip(String tooltip)
|
||||
{
|
||||
this.tooltip = tooltip;
|
||||
return (this);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -216,15 +216,5 @@ public interface QWidgetMetaDataInterface extends MetaDataWithPermissionRules
|
||||
** Fluent setter for dropdowns
|
||||
*******************************************************************************/
|
||||
QWidgetMetaData withDropdown(WidgetDropdownData dropdown);
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tooltip
|
||||
*******************************************************************************/
|
||||
default String getTooltip()
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -32,7 +32,7 @@ import java.util.Map;
|
||||
** AWS Quicksite specific meta data for frontend dashboard widget
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class QuickSightChartMetaData extends QWidgetMetaData
|
||||
public class QuickSightChartMetaData extends QWidgetMetaData implements QWidgetMetaDataInterface
|
||||
{
|
||||
private String accessKey;
|
||||
private String secretKey;
|
||||
|
||||
@ -23,7 +23,6 @@ package com.kingsrook.qqq.backend.core.model.metadata.frontend;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude.Include;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
@ -31,7 +30,6 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.WidgetDropdownData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -44,7 +42,6 @@ public class QFrontendWidgetMetaData
|
||||
{
|
||||
private final String name;
|
||||
private final String label;
|
||||
private final String tooltip;
|
||||
private final String type;
|
||||
private final String icon;
|
||||
private final Integer gridColumns;
|
||||
@ -57,13 +54,10 @@ public class QFrontendWidgetMetaData
|
||||
private boolean showReloadButton = false;
|
||||
private boolean showExportButton = false;
|
||||
|
||||
protected Map<String, QIcon> icons;
|
||||
|
||||
private final boolean hasPermission;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
// DO add getters for all fields - this tells Jackson to include them in JSON. //
|
||||
// do NOT add setters. take values from the source-object in the constructor!! //
|
||||
// do not add setters. take values from the source-object in the constructor!! //
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@ -75,7 +69,6 @@ public class QFrontendWidgetMetaData
|
||||
{
|
||||
this.name = widgetMetaData.getName();
|
||||
this.label = widgetMetaData.getLabel();
|
||||
this.tooltip = widgetMetaData.getTooltip();
|
||||
this.type = widgetMetaData.getType();
|
||||
this.icon = widgetMetaData.getIcon();
|
||||
this.gridColumns = widgetMetaData.getGridColumns();
|
||||
@ -89,7 +82,6 @@ public class QFrontendWidgetMetaData
|
||||
{
|
||||
this.showExportButton = qWidgetMetaData.getShowExportButton();
|
||||
this.showReloadButton = qWidgetMetaData.getShowReloadButton();
|
||||
this.icons = qWidgetMetaData.getIcons();
|
||||
}
|
||||
|
||||
hasPermission = PermissionsHelper.hasWidgetPermission(actionInput, name);
|
||||
@ -237,26 +229,4 @@ public class QFrontendWidgetMetaData
|
||||
{
|
||||
return showExportButton;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for icons
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Map<String, QIcon> getIcons()
|
||||
{
|
||||
return icons;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for tooltip
|
||||
**
|
||||
*******************************************************************************/
|
||||
public String getTooltip()
|
||||
{
|
||||
return tooltip;
|
||||
}
|
||||
}
|
||||
|
||||
@ -35,7 +35,6 @@ public class QIcon
|
||||
{
|
||||
private String name;
|
||||
private String path;
|
||||
private String color;
|
||||
|
||||
|
||||
|
||||
@ -124,36 +123,4 @@ public class QIcon
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for color
|
||||
*******************************************************************************/
|
||||
public String getColor()
|
||||
{
|
||||
return (this.color);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for color
|
||||
*******************************************************************************/
|
||||
public void setColor(String color)
|
||||
{
|
||||
this.color = color;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for color
|
||||
*******************************************************************************/
|
||||
public QIcon withColor(String color)
|
||||
{
|
||||
this.color = color;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -92,21 +92,6 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create a new possible value source, for an enum, with default settings.
|
||||
** e.g., type=ENUM; name from param values from the param; LABEL_ONLY format
|
||||
*******************************************************************************/
|
||||
public static <T extends PossibleValueEnum<?>> QPossibleValueSource newForEnum(String name, T[] values)
|
||||
{
|
||||
return new QPossibleValueSource()
|
||||
.withName(name)
|
||||
.withType(QPossibleValueSourceType.ENUM)
|
||||
.withValuesFromEnum(values)
|
||||
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -89,34 +89,6 @@ public class ExtractViaBasepullQueryStep extends ExtractViaQueryStep
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Let a subclass know if getQueryFilter will use the "default filter" (e.g., from
|
||||
** our base class, which would come from values passed in to the process), or if
|
||||
** the BasePull Query would be used (e.g., for a scheduled job).
|
||||
*******************************************************************************/
|
||||
protected boolean willTheBasePullQueryBeUsed(RunBackendStepInput runBackendStepInput)
|
||||
{
|
||||
try
|
||||
{
|
||||
super.getQueryFilter(runBackendStepInput);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if super.getQueryFilter returned - then - there's a default query to use (e.g., a user selecting rows on a screen). //
|
||||
// this means we won't use the BasePull query, so return a false here. //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
return (false);
|
||||
}
|
||||
catch(QException qe)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// if we catch here, assume that is because there was no default filter - in which case - we'll use the BasePull Query //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
return (true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -82,7 +82,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
|
||||
.withRecordPipe(getRecordPipe())
|
||||
.withLimit(getLimit())
|
||||
.withCsv(new String(bytes))
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(runBackendStepInput.getInstance().getTable(tableName))
|
||||
.withMapping(mapping)
|
||||
.withRecordCustomizer((record) ->
|
||||
|
||||
@ -26,15 +26,10 @@ import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@ -53,7 +48,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -65,36 +59,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
|
||||
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted");
|
||||
|
||||
private Map<UniqueKey, ProcessSummaryLineWithUKSampleValues> ukErrorSummaries = new HashMap<>();
|
||||
private Map<UniqueKey, ProcessSummaryLine> ukErrorSummaries = new HashMap<>();
|
||||
|
||||
private QTableMetaData table;
|
||||
|
||||
private Map<UniqueKey, Set<List<Serializable>>> keysInThisFile = new HashMap<>();
|
||||
|
||||
private int rowsProcessed = 0;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** extension of ProcessSummaryLine for lines where a UniqueKey was violated,
|
||||
** where we'll collect a sample (or maybe all) of the values that broke the UK.
|
||||
*******************************************************************************/
|
||||
private static class ProcessSummaryLineWithUKSampleValues extends ProcessSummaryLine
|
||||
{
|
||||
private Set<String> sampleValues = new LinkedHashSet<>();
|
||||
private boolean areThereMoreSampleValues = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public ProcessSummaryLineWithUKSampleValues(Status status)
|
||||
{
|
||||
super(status);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -119,48 +89,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
int rowsInThisPage = runBackendStepInput.getRecords().size();
|
||||
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setInputSource(QInputSource.USER);
|
||||
insertInput.setTableName(runBackendStepInput.getTableName());
|
||||
insertInput.setRecords(runBackendStepInput.getRecords());
|
||||
insertInput.setSkipUniqueKeyCheck(true);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// load the pre-insert customizer and set it up, if there is one //
|
||||
// then we'll run it based on its WhenToRun value //
|
||||
// we do this, in case it needs to, for example, adjust values that //
|
||||
// are part of a unique key //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
Optional<AbstractPreInsertCustomizer> preInsertCustomizer = QCodeLoader.getTableCustomizer(AbstractPreInsertCustomizer.class, table, TableCustomizers.PRE_INSERT_RECORD.getRole());
|
||||
if(preInsertCustomizer.isPresent())
|
||||
{
|
||||
preInsertCustomizer.get().setInsertInput(insertInput);
|
||||
preInsertCustomizer.get().setIsPreview(true);
|
||||
AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().getWhenToRun();
|
||||
if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun))
|
||||
{
|
||||
List<QRecord> recordsAfterCustomizer = preInsertCustomizer.get().apply(runBackendStepInput.getRecords());
|
||||
runBackendStepInput.setRecords(recordsAfterCustomizer);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// todo - do we care if the customizer runs both now, and in the validation below? //
|
||||
// right now we'll let it run both times, but maybe that should be protected against //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
}
|
||||
}
|
||||
QTableMetaData table = runBackendStepInput.getInstance().getTable(runBackendStepInput.getTableName());
|
||||
|
||||
Map<UniqueKey, Set<List<Serializable>>> existingKeys = new HashMap<>();
|
||||
List<UniqueKey> uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys());
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet());
|
||||
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR));
|
||||
ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLine(Status.ERROR));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@ -169,7 +105,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE))
|
||||
{
|
||||
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(rowsProcessed + 1));
|
||||
runBackendStepInput.getAsyncJobCallback().updateStatus("Processing row " + "%,d".formatted(okSummary.getCount()));
|
||||
}
|
||||
else if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE))
|
||||
{
|
||||
@ -187,13 +123,70 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
// Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction //
|
||||
// will only be getting the records in pages, but in here, we'll track UK's across pages!! //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
List<QRecord> recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// if there are no UK's, proceed with all records //
|
||||
////////////////////////////////////////////////////
|
||||
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
|
||||
if(existingKeys.isEmpty())
|
||||
{
|
||||
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
|
||||
}
|
||||
else
|
||||
{
|
||||
/////////////////////////////////////////////////////////////
|
||||
// else, only proceed with records that don't violate a UK //
|
||||
/////////////////////////////////////////////////////////////
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// else, get each records keys and see if it already exists or not //
|
||||
// also, build a set of keys we've seen (within this page (or overall?)) //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
for(QRecord record : runBackendStepInput.getRecords())
|
||||
{
|
||||
//////////////////////////////////////////////////////////
|
||||
// check if this record violates any of the unique keys //
|
||||
//////////////////////////////////////////////////////////
|
||||
boolean foundDupe = false;
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
|
||||
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
|
||||
{
|
||||
ukErrorSummaries.get(uniqueKey).incrementCount();
|
||||
foundDupe = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// if this record doesn't violate any uk's, then we can add it to the output //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
if(!foundDupe)
|
||||
{
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
|
||||
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
|
||||
}
|
||||
recordsWithoutUkErrors.add(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
// run all validation from the insert action - in Preview mode (boolean param) //
|
||||
/////////////////////////////////////////////////////////////////////////////////
|
||||
insertInput.setRecords(recordsWithoutUkErrors);
|
||||
InsertAction insertAction = new InsertAction();
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setInputSource(QInputSource.USER);
|
||||
insertInput.setTableName(runBackendStepInput.getTableName());
|
||||
insertInput.setRecords(recordsWithoutUkErrors);
|
||||
insertInput.setSkipUniqueKeyCheck(true);
|
||||
insertAction.performValidations(insertInput, true);
|
||||
List<QRecord> validationResultRecords = insertInput.getRecords();
|
||||
|
||||
@ -222,89 +215,6 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
}
|
||||
|
||||
runBackendStepOutput.setRecords(outputRecords);
|
||||
|
||||
this.rowsProcessed += rowsInThisPage;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<QRecord> getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map<UniqueKey, Set<List<Serializable>>> existingKeys, List<UniqueKey> uniqueKeys, QTableMetaData table)
|
||||
{
|
||||
////////////////////////////////////////////////////
|
||||
// if there are no UK's, proceed with all records //
|
||||
////////////////////////////////////////////////////
|
||||
List<QRecord> recordsWithoutUkErrors = new ArrayList<>();
|
||||
if(existingKeys.isEmpty())
|
||||
{
|
||||
recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords());
|
||||
}
|
||||
else
|
||||
{
|
||||
/////////////////////////////////////////////////////////////
|
||||
// else, only proceed with records that don't violate a UK //
|
||||
/////////////////////////////////////////////////////////////
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
keysInThisFile.computeIfAbsent(uniqueKey, x -> new HashSet<>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// else, get each records keys and see if it already exists or not //
|
||||
// also, build a set of keys we've seen (within this page (or overall?)) //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
for(QRecord record : runBackendStepInput.getRecords())
|
||||
{
|
||||
if(CollectionUtils.nullSafeHasContents(record.getErrors()))
|
||||
{
|
||||
///////////////////////////////////////////////////
|
||||
// skip any records that may already be in error //
|
||||
///////////////////////////////////////////////////
|
||||
recordsWithoutUkErrors.add(record);
|
||||
continue;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////
|
||||
// check if this record violates any of the unique keys //
|
||||
//////////////////////////////////////////////////////////
|
||||
boolean foundDupe = false;
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
|
||||
if(keyValues.isPresent() && (existingKeys.get(uniqueKey).contains(keyValues.get()) || keysInThisFile.get(uniqueKey).contains(keyValues.get())))
|
||||
{
|
||||
ProcessSummaryLineWithUKSampleValues processSummaryLineWithUKSampleValues = ukErrorSummaries.get(uniqueKey);
|
||||
processSummaryLineWithUKSampleValues.incrementCount();
|
||||
if(processSummaryLineWithUKSampleValues.sampleValues.size() < 3)
|
||||
{
|
||||
processSummaryLineWithUKSampleValues.sampleValues.add(keyValues.get().toString());
|
||||
}
|
||||
else
|
||||
{
|
||||
processSummaryLineWithUKSampleValues.areThereMoreSampleValues = true;
|
||||
}
|
||||
foundDupe = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// if this record doesn't violate any uk's, then we can add it to the output //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
if(!foundDupe)
|
||||
{
|
||||
for(UniqueKey uniqueKey : uniqueKeys)
|
||||
{
|
||||
Optional<List<Serializable>> keyValues = UniqueKeyHelper.getKeyValues(table, uniqueKey, record);
|
||||
keyValues.ifPresent(kv -> keysInThisFile.get(uniqueKey).add(kv));
|
||||
}
|
||||
recordsWithoutUkErrors.add(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
return recordsWithoutUkErrors;
|
||||
}
|
||||
|
||||
|
||||
@ -326,20 +236,17 @@ public class BulkInsertTransformStep extends AbstractTransformStep
|
||||
okSummary.pickMessage(isForResultScreen);
|
||||
okSummary.addSelfToListIfAnyCount(rs);
|
||||
|
||||
for(Map.Entry<UniqueKey, ProcessSummaryLineWithUKSampleValues> entry : ukErrorSummaries.entrySet())
|
||||
for(Map.Entry<UniqueKey, ProcessSummaryLine> entry : ukErrorSummaries.entrySet())
|
||||
{
|
||||
UniqueKey uniqueKey = entry.getKey();
|
||||
ProcessSummaryLineWithUKSampleValues ukErrorSummary = entry.getValue();
|
||||
UniqueKey uniqueKey = entry.getKey();
|
||||
ProcessSummaryLine ukErrorSummary = entry.getValue();
|
||||
String ukErrorSuffix = " inserted, because of duplicate values in a unique key (" + uniqueKey.getDescription(table) + ")";
|
||||
|
||||
ukErrorSummary
|
||||
.withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values"
|
||||
+ (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ")
|
||||
+ StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues)))
|
||||
|
||||
.withSingularFutureMessage(" record will not be")
|
||||
.withPluralFutureMessage(" records will not be")
|
||||
.withSingularPastMessage(" record was not")
|
||||
.withPluralPastMessage(" records were not");
|
||||
.withSingularFutureMessage(tableLabel + " record will not be" + ukErrorSuffix)
|
||||
.withPluralFutureMessage(tableLabel + " records will not be" + ukErrorSuffix)
|
||||
.withSingularPastMessage(tableLabel + " record was not" + ukErrorSuffix)
|
||||
.withPluralPastMessage(tableLabel + " records were not" + ukErrorSuffix);
|
||||
|
||||
ukErrorSummary.addSelfToListIfAnyCount(rs);
|
||||
}
|
||||
|
||||
@ -1,50 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load step that does nothing.
|
||||
**
|
||||
*******************************************************************************/
|
||||
public class NoopLoadStep extends AbstractLoadStep
|
||||
{
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Execute the backend step - using the request as input, and the result as output.
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
///////////
|
||||
// noop. //
|
||||
///////////
|
||||
}
|
||||
|
||||
}
|
||||
@ -49,17 +49,14 @@ public class LoadInitialRecordsStep implements BackendStep
|
||||
@Override
|
||||
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// basically this is a no-op... sometimes we just need a backendStep to be the first step in a process. //
|
||||
// While we're here, go ahead and put the query filter in the payload as a value - this is needed for //
|
||||
// processes that have a screen before their first backend step (why is this needed? not sure, but is) //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// basically this is a no-op... we Just need a backendStep to be the first step in the process //
|
||||
// but, while we're here, go ahead and put the query filter in the payload as a value, in case //
|
||||
// someone else wants it (see BulkDelete) //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
runBackendStepInput.getAsyncJobCallback().updateStatus("Loading records");
|
||||
if(runBackendStepInput.getCallback() != null)
|
||||
{
|
||||
QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter();
|
||||
runBackendStepOutput.addValue("queryFilterJson", JsonUtils.toJson(queryFilter));
|
||||
}
|
||||
QQueryFilter queryFilter = runBackendStepInput.getCallback().getQueryFilter();
|
||||
runBackendStepOutput.addValue("queryFilterJSON", JsonUtils.toJson(queryFilter));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -30,20 +30,18 @@ import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.AlphaNumericComparator;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Helper class for process steps that want to roll up error summary and/or
|
||||
** warning summary lines. e.g., if the process might have a handful of different
|
||||
** error messages. Will record up to 50 (configurable) unique errors, then throw the rest int
|
||||
** error messages. Will record up to 50 unique errors, then throw the rest int
|
||||
** an "other" errors summary.
|
||||
*******************************************************************************/
|
||||
public class ProcessSummaryWarningsAndErrorsRollup
|
||||
{
|
||||
private Map<String, ProcessSummaryLine> errorSummaries = new HashMap<>();
|
||||
private Map<String, ProcessSummaryLine> errorSummaries = new HashMap<>();
|
||||
private Map<String, ProcessSummaryLine> warningSummaries = new HashMap<>();
|
||||
|
||||
private ProcessSummaryLine otherErrorsSummary;
|
||||
@ -51,8 +49,6 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
private ProcessSummaryLine errorTemplate;
|
||||
private ProcessSummaryLine warningTemplate;
|
||||
|
||||
private int uniqueErrorsToShow = 50;
|
||||
private boolean doReplaceSingletonCountLinesWithSuffixOnly = true;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -171,7 +167,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
ProcessSummaryLine processSummaryLine = summaryLineMap.get(message);
|
||||
if(processSummaryLine == null)
|
||||
{
|
||||
if(summaryLineMap.size() < uniqueErrorsToShow)
|
||||
if(summaryLineMap.size() < 50)
|
||||
{
|
||||
processSummaryLine = new ProcessSummaryLine(status)
|
||||
.withMessageSuffix(message)
|
||||
@ -214,80 +210,17 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
|
||||
** extracts string messages out.
|
||||
**
|
||||
** Makes errors from bulk-insert look better when they report, e.g.
|
||||
** Error parsing line #1: ...
|
||||
** Error parsing line #2: ...
|
||||
** Error parsing line #10: ...
|
||||
*******************************************************************************/
|
||||
private static class PSLAlphaNumericComparator implements Comparator<ProcessSummaryLineInterface>
|
||||
{
|
||||
private static AlphaNumericComparator alphaNumericComparator = new AlphaNumericComparator();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int compare(ProcessSummaryLineInterface psli1, ProcessSummaryLineInterface psli2)
|
||||
{
|
||||
int messageComp = (alphaNumericComparator.compare(Objects.requireNonNullElse(psli1.getMessage(), ""), Objects.requireNonNullElse(psli2.getMessage(), "")));
|
||||
if(messageComp != 0)
|
||||
{
|
||||
return (messageComp);
|
||||
}
|
||||
|
||||
if(psli1 instanceof ProcessSummaryLine psl1 && psli2 instanceof ProcessSummaryLine psl2)
|
||||
{
|
||||
return (alphaNumericComparator.compare(Objects.requireNonNullElse(psl1.getMessageSuffix(), ""), Objects.requireNonNullElse(psl2.getMessageSuffix(), "")));
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** sort the process summary lines by count desc
|
||||
*******************************************************************************/
|
||||
private void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
|
||||
private static void addProcessSummaryLinesFromMap(ArrayList<ProcessSummaryLineInterface> rs, Map<String, ProcessSummaryLine> summaryMap)
|
||||
{
|
||||
summaryMap.values().stream()
|
||||
.sorted(Comparator.comparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getCount(), 0)).reversed()
|
||||
.thenComparing(new PSLAlphaNumericComparator())
|
||||
.thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessage(), ""))
|
||||
.thenComparing((ProcessSummaryLine psl) -> Objects.requireNonNullElse(psl.getMessageSuffix(), ""))
|
||||
)
|
||||
.map(psl ->
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// this is to make lines that are like "1 record had an error: Error parsing line #1: blah" look better, by //
|
||||
// removing the redundant "1 record..." bit. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
if(doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
if(psl.getCount() == 1)
|
||||
{
|
||||
return (new ProcessSummaryRecordLink().withStatus(psl.getStatus()).withLinkPreText(psl.getMessageSuffix()));
|
||||
}
|
||||
}
|
||||
|
||||
return (psl);
|
||||
})
|
||||
.forEach(psli ->
|
||||
{
|
||||
if(psli instanceof ProcessSummaryLine psl)
|
||||
{
|
||||
psl.addSelfToListIfAnyCount(rs);
|
||||
}
|
||||
else
|
||||
{
|
||||
rs.add(psli);
|
||||
}
|
||||
});
|
||||
.forEach(psl -> psl.addSelfToListIfAnyCount(rs));
|
||||
}
|
||||
|
||||
|
||||
@ -414,67 +347,4 @@ public class ProcessSummaryWarningsAndErrorsRollup
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public int getUniqueErrorsToShow()
|
||||
{
|
||||
return (this.uniqueErrorsToShow);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public void setUniqueErrorsToShow(int uniqueErrorsToShow)
|
||||
{
|
||||
this.uniqueErrorsToShow = uniqueErrorsToShow;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for uniqueErrorsToShow
|
||||
*******************************************************************************/
|
||||
public ProcessSummaryWarningsAndErrorsRollup withUniqueErrorsToShow(int uniqueErrorsToShow)
|
||||
{
|
||||
this.uniqueErrorsToShow = uniqueErrorsToShow;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public boolean getDoReplaceSingletonCountLinesWithSuffixOnly()
|
||||
{
|
||||
return (this.doReplaceSingletonCountLinesWithSuffixOnly);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public void setDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for doReplaceSingletonCountLinesWithSuffixOnly
|
||||
*******************************************************************************/
|
||||
public ProcessSummaryWarningsAndErrorsRollup withDoReplaceSingletonCountLinesWithSuffixOnly(boolean doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
{
|
||||
this.doReplaceSingletonCountLinesWithSuffixOnly = doReplaceSingletonCountLinesWithSuffixOnly;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@ -35,10 +35,8 @@ import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.audits.AuditSingleInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
@ -73,23 +71,20 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(AbstractTableSyncTransformStep.class);
|
||||
|
||||
private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
|
||||
private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
|
||||
|
||||
private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
|
||||
private ProcessSummaryLine okToInsert = StandardProcessSummaryLineProducer.getOkToInsertLine();
|
||||
private ProcessSummaryLine okToUpdate = StandardProcessSummaryLineProducer.getOkToUpdateLine();
|
||||
private ProcessSummaryLine willNotInsert = new ProcessSummaryLine(Status.INFO)
|
||||
.withMessageSuffix("because of this process' configuration.")
|
||||
.withSingularFutureMessage("will not be inserted ")
|
||||
.withPluralFutureMessage("will not be inserted ")
|
||||
.withSingularPastMessage("was not inserted ")
|
||||
.withPluralPastMessage("were not inserted ");
|
||||
|
||||
private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
|
||||
private ProcessSummaryLine willNotUpdate = new ProcessSummaryLine(Status.INFO)
|
||||
.withMessageSuffix("because of this process' configuration.")
|
||||
.withSingularFutureMessage("will not be updated ")
|
||||
.withPluralFutureMessage("will not be updated ")
|
||||
.withSingularPastMessage("was not updated ")
|
||||
.withPluralPastMessage("were not updated ");
|
||||
|
||||
private ProcessSummaryLine errorMissingKeyField = new ProcessSummaryLine(Status.ERROR)
|
||||
.withMessageSuffix("missing a value for the key field.")
|
||||
.withSingularFutureMessage("will not be synced, because it is ")
|
||||
@ -97,16 +92,8 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
.withSingularPastMessage("was not synced, because it is ")
|
||||
.withPluralPastMessage("were not synced, because they are ");
|
||||
|
||||
private ProcessSummaryLine unspecifiedError = new ProcessSummaryLine(Status.ERROR)
|
||||
.withMessageSuffix("of an unexpected error: ")
|
||||
.withSingularFutureMessage("will not be synced, ")
|
||||
.withPluralFutureMessage("will not be synced, ")
|
||||
.withSingularPastMessage("was not synced, ")
|
||||
.withPluralPastMessage("were not synced, ");
|
||||
|
||||
protected RunBackendStepInput runBackendStepInput = null;
|
||||
protected RunBackendStepOutput runBackendStepOutput = null;
|
||||
protected RecordLookupHelper recordLookupHelper = null;
|
||||
protected RunBackendStepInput runBackendStepInput = null;
|
||||
protected RecordLookupHelper recordLookupHelper = null;
|
||||
|
||||
private QPossibleValueTranslator possibleValueTranslator;
|
||||
|
||||
@ -118,17 +105,16 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
|
||||
{
|
||||
return StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField, unspecifiedError, willNotInsert, willNotUpdate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected ArrayList<ProcessSummaryLineInterface> getErrorProcessSummaryLines(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
|
||||
{
|
||||
return StandardProcessSummaryLineProducer.toArrayList(errorMissingKeyField, unspecifiedError);
|
||||
ArrayList<ProcessSummaryLineInterface> processSummaryLineList = StandardProcessSummaryLineProducer.toArrayList(okToInsert, okToUpdate, errorMissingKeyField);
|
||||
if(willNotInsert.getCount() > 0)
|
||||
{
|
||||
processSummaryLineList.add(willNotInsert);
|
||||
}
|
||||
if(willNotUpdate.getCount() > 0)
|
||||
{
|
||||
processSummaryLineList.add(willNotUpdate);
|
||||
}
|
||||
return (processSummaryLineList);
|
||||
}
|
||||
|
||||
|
||||
@ -207,7 +193,6 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
}
|
||||
|
||||
this.runBackendStepInput = runBackendStepInput;
|
||||
this.runBackendStepOutput = runBackendStepOutput;
|
||||
|
||||
SyncProcessConfig config = getSyncProcessConfig();
|
||||
|
||||
@ -257,8 +242,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
Set<Serializable> processedSourceKeys = new HashSet<>();
|
||||
for(QRecord sourceRecord : runBackendStepInput.getRecords())
|
||||
{
|
||||
Serializable sourcePrimaryKey = sourceRecord.getValue(QContext.getQInstance().getTable(config.sourceTable).getPrimaryKeyField());
|
||||
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
|
||||
Serializable sourceKeyValue = sourceRecord.getValue(sourceTableKeyField);
|
||||
if(processedSourceKeys.contains(sourceKeyValue))
|
||||
{
|
||||
LOG.info("Skipping duplicated source-key within page", logPair("key", sourceKeyValue));
|
||||
@ -268,7 +252,7 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
|
||||
if(sourceKeyValue == null || "".equals(sourceKeyValue))
|
||||
{
|
||||
errorMissingKeyField.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
errorMissingKeyField.incrementCount();
|
||||
|
||||
try
|
||||
{
|
||||
@ -287,56 +271,41 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
//////////////////////////////////////////////////////////////
|
||||
// look for the existing record, to determine insert/update //
|
||||
//////////////////////////////////////////////////////////////
|
||||
try
|
||||
{
|
||||
QRecord existingRecord = getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue);
|
||||
QRecord existingRecord = getExistingRecord(existingRecordsByForeignKey, destinationForeignKeyField, sourceKeyValue);
|
||||
|
||||
QRecord recordToStore;
|
||||
if(existingRecord != null && config.performUpdates)
|
||||
QRecord recordToStore;
|
||||
if(existingRecord != null && config.performUpdates)
|
||||
{
|
||||
recordToStore = existingRecord;
|
||||
okToUpdate.incrementCount();
|
||||
}
|
||||
else if(existingRecord == null && config.performInserts)
|
||||
{
|
||||
recordToStore = new QRecord();
|
||||
okToInsert.incrementCount();
|
||||
}
|
||||
else
|
||||
{
|
||||
if(existingRecord != null)
|
||||
{
|
||||
recordToStore = existingRecord;
|
||||
}
|
||||
else if(existingRecord == null && config.performInserts)
|
||||
{
|
||||
recordToStore = new QRecord();
|
||||
LOG.info("Skipping storing existing record because this sync process is set to not perform updates");
|
||||
willNotInsert.incrementCount();
|
||||
}
|
||||
else
|
||||
{
|
||||
if(existingRecord != null)
|
||||
{
|
||||
LOG.info("Skipping storing existing record because this sync process is set to not perform updates");
|
||||
willNotInsert.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG.info("Skipping storing new record because this sync process is set to not perform inserts");
|
||||
willNotUpdate.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
// if we received a record to store add to the output records and summary lines //
|
||||
//////////////////////////////////////////////////////////////////////////////////
|
||||
recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
|
||||
if(recordToStore != null)
|
||||
{
|
||||
if(existingRecord != null)
|
||||
{
|
||||
okToUpdate.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
}
|
||||
else
|
||||
{
|
||||
okToInsert.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
}
|
||||
|
||||
runBackendStepOutput.addRecord(recordToStore);
|
||||
LOG.info("Skipping storing new record because this sync process is set to not perform inserts");
|
||||
willNotUpdate.incrementCount();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
catch(Exception e)
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// if we received a record to store add to the output records //
|
||||
////////////////////////////////////////////////////////////////
|
||||
recordToStore = populateRecordToStore(runBackendStepInput, recordToStore, sourceRecord);
|
||||
if(recordToStore != null)
|
||||
{
|
||||
unspecifiedError.incrementCountAndAddPrimaryKey(sourcePrimaryKey);
|
||||
unspecifiedError.setMessageSuffix(unspecifiedError.getMessageSuffix() + e.getMessage());
|
||||
runBackendStepOutput.addRecord(recordToStore);
|
||||
}
|
||||
}
|
||||
|
||||
@ -457,17 +426,4 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Let the subclass "easily" add an audit to be inserted on the Execute step.
|
||||
*******************************************************************************/
|
||||
protected void addAuditForExecuteStep(AuditSingleInput auditSingleInput)
|
||||
{
|
||||
if(StreamedETLWithFrontendProcess.STEP_NAME_EXECUTE.equals(this.runBackendStepInput.getStepName()))
|
||||
{
|
||||
this.runBackendStepOutput.addAuditSingleInput(auditSingleInput);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -90,15 +90,4 @@ public class InMemoryStateProvider implements StateProviderInterface
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove a block of data, under a key, from the state store.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void remove(AbstractStateKey key)
|
||||
{
|
||||
map.remove(key);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -52,10 +52,4 @@ public interface StateProviderInterface
|
||||
** Get a block of data, under a key, from the state store.
|
||||
*******************************************************************************/
|
||||
<T extends Serializable> Optional<T> get(Class<? extends T> type, AbstractStateKey key);
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove a block of data, under a key, from the state store.
|
||||
*******************************************************************************/
|
||||
void remove(AbstractStateKey key);
|
||||
|
||||
}
|
||||
|
||||
@ -30,7 +30,6 @@ import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -111,21 +110,6 @@ public class TempFileStateProvider implements StateProviderInterface
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Remove a block of data, under a key, from the state store.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void remove(AbstractStateKey key)
|
||||
{
|
||||
File file = getFile(key);
|
||||
if(!file.delete())
|
||||
{
|
||||
LOG.warn("Error deleting state-providing tempFile", logPair("file", file.getAbsolutePath()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Get the file referenced by a key
|
||||
*******************************************************************************/
|
||||
|
||||
@ -26,6 +26,7 @@ import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
@ -222,6 +223,67 @@ public class JsonUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sorts all fields recursively in all objects in a json object
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Object sortJSON(Object o, int depth) throws Exception
|
||||
{
|
||||
///////////////////////////////////////////////////////
|
||||
// this should only be done on json objects or array //
|
||||
///////////////////////////////////////////////////////
|
||||
if(depth == 0 && !(o instanceof JSONObject) && !(o instanceof JSONArray))
|
||||
{
|
||||
throw (new Exception("Object must be of type JSONObject or JSONArray in order to sort"));
|
||||
}
|
||||
|
||||
if(o instanceof JSONObject jsonObject)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// if json object, create a new object, sort keys, and add to the //
|
||||
// new object passing values to this sort method recursively //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
JSONObject returnObject = new JSONObject();
|
||||
|
||||
List<String> keys = new ArrayList<>(jsonObject.keySet());
|
||||
Collections.sort(keys);
|
||||
for(String key : keys)
|
||||
{
|
||||
returnObject.put(key, sortJSON(jsonObject.get(key), ++depth));
|
||||
}
|
||||
|
||||
return (returnObject);
|
||||
}
|
||||
else if(o instanceof JSONArray jsonArray)
|
||||
{
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// if this is an array, same as above, but no sorting needed (i think) //
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
JSONArray returnObject = new JSONArray();
|
||||
for(int i = 0; i < jsonArray.length(); i++)
|
||||
{
|
||||
returnObject.put(i, sortJSON(jsonArray.get(i), ++depth));
|
||||
}
|
||||
|
||||
return (returnObject);
|
||||
}
|
||||
|
||||
return (o);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sorts all fields recursively in all objects in a json object
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static Object sortJSON(Object o) throws Exception
|
||||
{
|
||||
return (sortJSON(o, 0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** De-serialize a json string into a JSONArray (string must start with "[")
|
||||
**
|
||||
|
||||
@ -95,7 +95,7 @@ public class ValueUtils
|
||||
}
|
||||
else if(value instanceof String s)
|
||||
{
|
||||
return "true".equalsIgnoreCase(s) || "yes".equalsIgnoreCase(s);
|
||||
return (Boolean.parseBoolean(s));
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -496,9 +496,6 @@ public class ValueUtils
|
||||
*******************************************************************************/
|
||||
private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e)
|
||||
{
|
||||
//////////////////////
|
||||
// 1999-12-31T12:59 //
|
||||
//////////////////////
|
||||
if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$"))
|
||||
{
|
||||
//////////////////////////
|
||||
@ -506,34 +503,11 @@ public class ValueUtils
|
||||
//////////////////////////
|
||||
return Instant.parse(s + ":00Z");
|
||||
}
|
||||
|
||||
///////////////////////////
|
||||
// 1999-12-31 12:59:59.0 //
|
||||
///////////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z");
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
/////////////////////////
|
||||
// 1999-12-31 12:59:59 //
|
||||
/////////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T") + "Z";
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
//////////////////////
|
||||
// 1999-12-31 12:59 //
|
||||
//////////////////////
|
||||
else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$"))
|
||||
{
|
||||
s = s.replaceAll(" ", "T") + ":00Z";
|
||||
return Instant.parse(s);
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
try
|
||||
|
||||
@ -1,199 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.collections;
|
||||
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Comparator for strings that are a mix of alpha + numeric, where we want to
|
||||
** sort the numeric substrings like numbers.
|
||||
**
|
||||
** e.g., A1, A2, A10 won't come out as A1, A10, A2
|
||||
*******************************************************************************/
|
||||
public class AlphaNumericComparator implements Comparator<String>
|
||||
{
|
||||
private static final int A_FIRST = -1;
|
||||
private static final int B_FIRST = 1;
|
||||
private static final int TIE = 0;
|
||||
|
||||
private static final Pattern INT_PATTERN = Pattern.compile("^\\d+$");
|
||||
private static final Pattern LEADING_INT_PATTERN = Pattern.compile("^\\d+");
|
||||
private static final Pattern ALPHA_THEN_INT_PATTERN = Pattern.compile("^(\\D+)\\d+");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** compare 2 Strings
|
||||
**
|
||||
*******************************************************************************/
|
||||
public int compare(String a, String b)
|
||||
{
|
||||
try
|
||||
{
|
||||
//////////////////////////////////////
|
||||
// eliminate degenerate cases first //
|
||||
//////////////////////////////////////
|
||||
if(a == null && b == null)
|
||||
{
|
||||
return (TIE);
|
||||
}
|
||||
else if(a == null)
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b == null)
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
else if(a.equals(b))
|
||||
{
|
||||
return (TIE);
|
||||
} // also covers a == "" and b == ""
|
||||
else if(a.equals(""))
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b.equals(""))
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// if both strings are pure numeric, parse as int and compare //
|
||||
////////////////////////////////////////////////////////////////
|
||||
if(INT_PATTERN.matcher(a).matches() && INT_PATTERN.matcher(b).matches())
|
||||
{
|
||||
int intsCompared = new Integer(a).compareTo(new Integer(b));
|
||||
if(intsCompared == TIE)
|
||||
{
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// in case the integers are the same (ie, "0001" vs "1"), compare as strings //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
else
|
||||
{
|
||||
///////////////////////////////////////////////////////////////
|
||||
// else, if the ints were different, return their comparison //
|
||||
///////////////////////////////////////////////////////////////
|
||||
return (intsCompared);
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////
|
||||
// if both start as numbers, extract those numbers //
|
||||
/////////////////////////////////////////////////////
|
||||
Matcher aLeadingIntMatcher = LEADING_INT_PATTERN.matcher(a);
|
||||
Matcher bLeadingIntMatcher = LEADING_INT_PATTERN.matcher(b);
|
||||
if(aLeadingIntMatcher.lookingAt() && bLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
///////////////////////////
|
||||
// extract the int parts //
|
||||
///////////////////////////
|
||||
String aIntPart = a.substring(0, aLeadingIntMatcher.end());
|
||||
String bIntPart = b.substring(0, bLeadingIntMatcher.end());
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// if the ints compare as non-zero, return that comparison //
|
||||
/////////////////////////////////////////////////////////////
|
||||
int intPartCompared = new Integer(aIntPart).compareTo(new Integer(bIntPart));
|
||||
if(intPartCompared != TIE)
|
||||
{
|
||||
return (intPartCompared);
|
||||
}
|
||||
else
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// otherwise, make recursive call to compare the rest of the string //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
String aRest = a.substring(aLeadingIntMatcher.end());
|
||||
String bRest = b.substring(bLeadingIntMatcher.end());
|
||||
return (compare(aRest, bRest));
|
||||
}
|
||||
}
|
||||
//////////////////////////////////////////////////////
|
||||
// if one starts as numeric, but other doesn't //
|
||||
// return the one that starts with the number first //
|
||||
//////////////////////////////////////////////////////
|
||||
else if(aLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(bLeadingIntMatcher.lookingAt())
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// now, if both parts have an alpha part, followed by digit parts, and //
|
||||
// the alpha parts are the same, then discard the alpha parts and recur //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
Matcher aAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(a);
|
||||
Matcher bAlphaThenIntMatcher = ALPHA_THEN_INT_PATTERN.matcher(b);
|
||||
if(aAlphaThenIntMatcher.lookingAt() && bAlphaThenIntMatcher.lookingAt())
|
||||
{
|
||||
String aAlphaPart = aAlphaThenIntMatcher.group(1);
|
||||
String bAlphaPart = bAlphaThenIntMatcher.group(1);
|
||||
|
||||
if(aAlphaPart.equals(bAlphaPart))
|
||||
{
|
||||
String aRest = a.substring(aAlphaPart.length());
|
||||
String bRest = b.substring(bAlphaPart.length());
|
||||
return (compare(aRest, bRest));
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// as last resort, just do pure string compare //
|
||||
/////////////////////////////////////////////////
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
//////////////////////////////////////////////////////////
|
||||
// on exception, don't allow caller to catch -- rather, //
|
||||
// always return something sensible (and null-safe) //
|
||||
//////////////////////////////////////////////////////////
|
||||
if(a == null && b == null)
|
||||
{
|
||||
return (TIE);
|
||||
}
|
||||
else if(a == null)
|
||||
{
|
||||
return (A_FIRST);
|
||||
}
|
||||
else if(b == null)
|
||||
{
|
||||
return (B_FIRST);
|
||||
}
|
||||
else
|
||||
{
|
||||
return (a.compareTo(b));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,184 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.memoization;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Basic memoization functionality - with result timeouts (only when doing a get -
|
||||
** there's no cleanup thread), and max-size.
|
||||
*******************************************************************************/
|
||||
public class Memoization<K, V>
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(Memoization.class);
|
||||
|
||||
private final Map<K, MemoizedResult<V>> map = Collections.synchronizedMap(new LinkedHashMap<>());
|
||||
|
||||
private Duration timeout = Duration.ofSeconds(600);
|
||||
private Integer maxSize = 1000;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Memoization()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Optional<V> getResult(K key)
|
||||
{
|
||||
MemoizedResult<V> result = map.get(key);
|
||||
if(result != null)
|
||||
{
|
||||
if(result.getTime().isAfter(Instant.now().minus(timeout)))
|
||||
{
|
||||
return (Optional.ofNullable(result.getResult()));
|
||||
}
|
||||
}
|
||||
|
||||
return (Optional.empty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Optional<MemoizedResult<V>> getMemoizedResult(K key)
|
||||
{
|
||||
MemoizedResult<V> result = map.get(key);
|
||||
if(result != null)
|
||||
{
|
||||
if(result.getTime().isAfter(Instant.now().minus(timeout)))
|
||||
{
|
||||
return (Optional.ofNullable(result));
|
||||
}
|
||||
}
|
||||
|
||||
return (Optional.empty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void storeResult(K key, V value)
|
||||
{
|
||||
map.put(key, new MemoizedResult<>(value));
|
||||
|
||||
//////////////////////////////////////
|
||||
// make sure map didn't get too big //
|
||||
// do this thread safely, please //
|
||||
//////////////////////////////////////
|
||||
try
|
||||
{
|
||||
if(map.size() > maxSize)
|
||||
{
|
||||
synchronized(map)
|
||||
{
|
||||
Iterator<Map.Entry<K, MemoizedResult<V>>> iterator = null;
|
||||
while(map.size() > maxSize)
|
||||
{
|
||||
if(iterator == null)
|
||||
{
|
||||
iterator = map.entrySet().iterator();
|
||||
}
|
||||
|
||||
if(iterator.hasNext())
|
||||
{
|
||||
iterator.next();
|
||||
iterator.remove();
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.error("Error managing size of a Memoization", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void clear()
|
||||
{
|
||||
this.map.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for timeoutSeconds
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setTimeout(Duration timeout)
|
||||
{
|
||||
this.timeout = timeout;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter for maxSize
|
||||
**
|
||||
*******************************************************************************/
|
||||
public void setMaxSize(Integer maxSize)
|
||||
{
|
||||
this.maxSize = maxSize;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** package-private - for tests to look at the map.
|
||||
**
|
||||
*******************************************************************************/
|
||||
Map<K, MemoizedResult<V>> getMap()
|
||||
{
|
||||
return map;
|
||||
}
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.memoization;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Object stored in the Memoization class. Shouldn't need to be visible outside
|
||||
** its package.
|
||||
*******************************************************************************/
|
||||
public class MemoizedResult<T>
|
||||
{
|
||||
private T result;
|
||||
private Instant time;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor
|
||||
**
|
||||
*******************************************************************************/
|
||||
public MemoizedResult(T result)
|
||||
{
|
||||
this.result = result;
|
||||
this.time = Instant.now();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for result
|
||||
**
|
||||
*******************************************************************************/
|
||||
public T getResult()
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for time
|
||||
**
|
||||
*******************************************************************************/
|
||||
public Instant getTime()
|
||||
{
|
||||
return time;
|
||||
}
|
||||
}
|
||||
@ -31,8 +31,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
|
||||
@ -753,28 +751,4 @@ class InsertActionTest extends BaseTest
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testDefaultValues() throws QException
|
||||
{
|
||||
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
|
||||
.getField("noOfShoes").withDefaultValue(2);
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
insertInput.setRecords(List.of(
|
||||
new QRecord().withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("noOfShoes", 4),
|
||||
new QRecord().withValue("firstName", "Tim").withValue("lastName", "Chamberlain")
|
||||
));
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
List<QRecord> records = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter())).getRecords();
|
||||
assertEquals(4, records.get(0).getValueInteger("noOfShoes"));
|
||||
assertEquals(2, records.get(1).getValueInteger("noOfShoes"));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -460,72 +460,4 @@ public class QPossibleValueTranslatorTest extends BaseTest
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testClearingInternalCaches() throws QException
|
||||
{
|
||||
QInstance qInstance = QContext.getQInstance();
|
||||
QTableMetaData personTable = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
|
||||
QPossibleValueTranslator possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
|
||||
QFieldMetaData shapeField = qInstance.getTable(TestUtils.TABLE_NAME_PERSON).getField("favoriteShapeId");
|
||||
|
||||
TestUtils.insertDefaultShapes(qInstance);
|
||||
TestUtils.insertExtraShapes(qInstance);
|
||||
|
||||
List<QRecord> personRecords = List.of(
|
||||
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 1),
|
||||
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 2),
|
||||
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 3),
|
||||
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 4),
|
||||
new QRecord().withTableName(TestUtils.TABLE_NAME_PERSON).withValue("favoriteShapeId", 5)
|
||||
);
|
||||
|
||||
MemoryRecordStore.setCollectStatistics(true);
|
||||
MemoryRecordStore.resetStatistics();
|
||||
|
||||
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
|
||||
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
|
||||
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
|
||||
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
|
||||
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
|
||||
|
||||
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
|
||||
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
|
||||
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
|
||||
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
|
||||
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should still just have ran just 1 query");
|
||||
|
||||
possibleValueTranslator.setMaxSizePerPvsCache(2);
|
||||
possibleValueTranslator.primePvsCache(personTable, personRecords, null, null);
|
||||
|
||||
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Now, should have ran another query");
|
||||
|
||||
assertEquals("Triangle", possibleValueTranslator.translatePossibleValue(shapeField, 1));
|
||||
assertEquals("Square", possibleValueTranslator.translatePossibleValue(shapeField, 2));
|
||||
assertEquals("Circle", possibleValueTranslator.translatePossibleValue(shapeField, 3));
|
||||
|
||||
///////////////////////////
|
||||
// reset and start again //
|
||||
///////////////////////////
|
||||
possibleValueTranslator = new QPossibleValueTranslator(qInstance, new QSession());
|
||||
MemoryRecordStore.resetStatistics();
|
||||
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
|
||||
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
|
||||
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
|
||||
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran just 1 query");
|
||||
|
||||
possibleValueTranslator.setMaxSizePerPvsCache(2);
|
||||
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords);
|
||||
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran another query");
|
||||
|
||||
MemoryRecordStore.resetStatistics();
|
||||
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(0, 3));
|
||||
possibleValueTranslator.translatePossibleValuesInRecords(personTable, personRecords.subList(3, 5));
|
||||
assertEquals(2, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN), "Should have ran 2 more queries");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -57,10 +57,6 @@ class QValueFormatterTest extends BaseTest
|
||||
{
|
||||
assertNull(QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), null));
|
||||
|
||||
assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData(), 1));
|
||||
assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat("%s"), 1));
|
||||
assertEquals("Hello", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat("%s"), "Hello"));
|
||||
|
||||
assertEquals("1", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1));
|
||||
assertEquals("1,000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(DisplayFormat.COMMAS), 1000));
|
||||
assertEquals("1000", QValueFormatter.formatValue(new QFieldMetaData().withDisplayFormat(null), 1000));
|
||||
@ -121,12 +117,6 @@ class QValueFormatterTest extends BaseTest
|
||||
table = new QTableMetaData().withPrimaryKeyField("id");
|
||||
assertEquals("42", QValueFormatter.formatRecordLabel(table, new QRecord().withValue("id", 42)));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
// exceptional flow: no recordLabelFormat specified, and record already had a label //
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
table = new QTableMetaData().withPrimaryKeyField("id");
|
||||
assertEquals("my label", QValueFormatter.formatRecordLabel(table, new QRecord().withRecordLabel("my label").withValue("id", 42)));
|
||||
|
||||
/////////////////////////////////////////////////
|
||||
// exceptional flow: no fields for the format //
|
||||
/////////////////////////////////////////////////
|
||||
|
||||
@ -23,18 +23,12 @@ package com.kingsrook.qqq.backend.core.actions.values;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.LocalDate;
|
||||
import java.time.Month;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@ -230,74 +224,6 @@ class SearchPossibleValueSourceActionTest extends BaseTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSearchPvsAction_tableByIdOnlyNonNumeric() throws QException
|
||||
{
|
||||
QContext.getQInstance().getPossibleValueSource(TestUtils.TABLE_NAME_SHAPE)
|
||||
.withSearchFields(List.of("id"));
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// a non-integer input should find nothing //
|
||||
// the catch { (IN, empty) } code makes this happen - without that, all records are found. //
|
||||
// (furthermore, i think that's only exposed if there's only 1 search field, maybe) //
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||
SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutput("A", TestUtils.TABLE_NAME_SHAPE);
|
||||
assertEquals(0, output.getResults().size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSearchPvsAction_tableByLocalDate() throws QException
|
||||
{
|
||||
MemoryRecordStore.getInstance().reset();
|
||||
|
||||
////////////////////////////////////////////
|
||||
// make a PVS for the person-memory table //
|
||||
////////////////////////////////////////////
|
||||
QContext.getQInstance().addPossibleValueSource(QPossibleValueSource.newForTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
|
||||
.withSearchFields(List.of("id", "firstName", "birthDate"))
|
||||
);
|
||||
|
||||
List<QRecord> shapeRecords = List.of(
|
||||
new QRecord().withValue("id", 1).withValue("firstName", "Homer").withValue("birthDate", LocalDate.of(1960, Month.JANUARY, 1)),
|
||||
new QRecord().withValue("id", 2).withValue("firstName", "Marge").withValue("birthDate", LocalDate.of(1961, Month.FEBRUARY, 2)),
|
||||
new QRecord().withValue("id", 3).withValue("firstName", "Bart").withValue("birthDate", LocalDate.of(1980, Month.MARCH, 3)));
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
insertInput.setRecords(shapeRecords);
|
||||
new InsertAction().execute(insertInput);
|
||||
|
||||
/////////////////////////////////////
|
||||
// a parseable date yields a match //
|
||||
/////////////////////////////////////
|
||||
SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutput("1960-01-01", TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
assertEquals(1, output.getResults().size());
|
||||
assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// alternative date format also works (thanks to ValueUtils parsing) //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
output = getSearchPossibleValueSourceOutput("1/1/1960", TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
assertEquals(1, output.getResults().size());
|
||||
assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1));
|
||||
|
||||
///////////////////////////////////
|
||||
// incomplete date finds nothing //
|
||||
///////////////////////////////////
|
||||
output = getSearchPossibleValueSourceOutput("1960-01", TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||
assertEquals(0, output.getResults().size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -22,19 +22,15 @@
|
||||
package com.kingsrook.qqq.backend.core.adapters;
|
||||
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QIndexBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
@ -399,70 +395,4 @@ class CsvToQRecordAdapterTest extends BaseTest
|
||||
assertEquals("john@doe.com", records.get(0).getValueString("email"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_buildRecordsFromCsv_doCorrectValueTypes() throws QException
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(TestUtils.defineTablePerson().withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN)))
|
||||
.withCsv("""
|
||||
firstName,birthDate,isEmployed
|
||||
John,1/1/1980,true
|
||||
Paul,1970-06-15,Yes
|
||||
George,,anything-else
|
||||
"""));
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
assertEquals("John", qRecord.getValue("firstName"));
|
||||
assertEquals(LocalDate.parse("1980-01-01"), qRecord.getValue("birthDate"));
|
||||
assertEquals(true, qRecord.getValue("isEmployed"));
|
||||
|
||||
qRecord = qRecords.get(1);
|
||||
assertEquals("Paul", qRecord.getValue("firstName"));
|
||||
assertEquals(LocalDate.parse("1970-06-15"), qRecord.getValue("birthDate"));
|
||||
assertEquals(true, qRecord.getValue("isEmployed"));
|
||||
|
||||
qRecord = qRecords.get(2);
|
||||
assertEquals("George", qRecord.getValue("firstName"));
|
||||
assertNull(qRecord.getValue("birthDate"));
|
||||
assertEquals(false, qRecord.getValue("isEmployed"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_buildRecordsFromCsv_doCorrectValueTypesErrorsForUnparseable() throws QException
|
||||
{
|
||||
CsvToQRecordAdapter csvToQRecordAdapter = new CsvToQRecordAdapter();
|
||||
csvToQRecordAdapter.buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper()
|
||||
.withDoCorrectValueTypes(true)
|
||||
.withTable(TestUtils.defineTablePerson())
|
||||
.withCsv("""
|
||||
firstName,birthDate,favoriteShapeId
|
||||
John,1980,1
|
||||
Paul,1970-06-15,green
|
||||
"""));
|
||||
List<QRecord> qRecords = csvToQRecordAdapter.getRecordList();
|
||||
|
||||
QRecord qRecord = qRecords.get(0);
|
||||
assertEquals("John", qRecord.getValue("firstName"));
|
||||
assertThat(qRecord.getErrors()).hasSize(1);
|
||||
assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #1: Could not parse value [1980] to a local date");
|
||||
|
||||
qRecord = qRecords.get(1);
|
||||
assertEquals("Paul", qRecord.getValue("firstName"));
|
||||
assertThat(qRecord.getErrors()).hasSize(1);
|
||||
assertThat(qRecord.getErrors().get(0).toString()).isEqualTo("Error parsing line #2: Value [green] could not be converted to an Integer.");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.adapters;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QRecordToCsvAdapter
|
||||
*******************************************************************************/
|
||||
class QRecordToCsvAdapterTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testSanitize()
|
||||
{
|
||||
assertEquals("foo", QRecordToCsvAdapter.sanitize("foo"));
|
||||
|
||||
assertEquals("""
|
||||
Homer ""Jay"" Simpson""", QRecordToCsvAdapter.sanitize("""
|
||||
Homer "Jay" Simpson"""));
|
||||
|
||||
assertEquals("""
|
||||
one ""quote"" two ""quotes"".""", QRecordToCsvAdapter.sanitize("""
|
||||
one "quote" two "quotes"."""));
|
||||
|
||||
assertEquals("""
|
||||
new line""", QRecordToCsvAdapter.sanitize("""
|
||||
new
|
||||
line"""));
|
||||
|
||||
assertEquals("""
|
||||
end ""quote"" new line""", QRecordToCsvAdapter.sanitize("""
|
||||
end "quote" new
|
||||
line"""));
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,155 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.model.data;
|
||||
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS;
|
||||
import static com.kingsrook.qqq.backend.core.model.data.QRecord.BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotSame;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertSame;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for QRecord
|
||||
*******************************************************************************/
|
||||
class QRecordTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCopyConstructor()
|
||||
{
|
||||
String jsonValue = """
|
||||
{"key": [1,2]}
|
||||
""";
|
||||
Map<String, Integer> fieldLengths = MapBuilder.of("a", 1, "b", 2);
|
||||
|
||||
QRecord original = new QRecord()
|
||||
.withTableName("myTable")
|
||||
.withRecordLabel("My Record")
|
||||
.withValue("one", 1)
|
||||
.withValue("two", "two")
|
||||
.withValue("three", new BigDecimal("3"))
|
||||
.withValue("false", false)
|
||||
.withValue("empty", null)
|
||||
.withDisplayValue("three", "3.00")
|
||||
.withBackendDetail(BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT, jsonValue)
|
||||
.withBackendDetail(BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>(fieldLengths))
|
||||
.withError(new BadInputStatusMessage("Bad Input"))
|
||||
.withAssociatedRecord("child", new QRecord().withValue("id", "child1"))
|
||||
.withAssociatedRecord("child", new QRecord().withValue("id", "child2"))
|
||||
.withAssociatedRecord("nephew", new QRecord().withValue("id", "nephew1"));
|
||||
|
||||
QRecord clone = new QRecord(original);
|
||||
|
||||
//////////////////////////////////////////////////////////////
|
||||
// assert equality on all the members values in the records //
|
||||
//////////////////////////////////////////////////////////////
|
||||
assertEquals("myTable", clone.getTableName());
|
||||
assertEquals("My Record", clone.getRecordLabel());
|
||||
assertEquals(1, clone.getValue("one"));
|
||||
assertEquals("two", clone.getValue("two"));
|
||||
assertEquals(new BigDecimal("3"), clone.getValue("three"));
|
||||
assertEquals(false, clone.getValue("false"));
|
||||
assertNull(clone.getValue("empty"));
|
||||
assertEquals("3.00", clone.getDisplayValue("three"));
|
||||
assertEquals(jsonValue, clone.getBackendDetail(BACKEND_DETAILS_TYPE_JSON_SOURCE_OBJECT));
|
||||
assertEquals(fieldLengths, clone.getBackendDetail(BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS));
|
||||
assertEquals(1, clone.getErrors().size());
|
||||
assertEquals(BadInputStatusMessage.class, clone.getErrors().get(0).getClass());
|
||||
assertEquals("Bad Input", clone.getErrors().get(0).getMessage());
|
||||
assertEquals(0, clone.getWarnings().size());
|
||||
assertEquals(2, clone.getAssociatedRecords().size());
|
||||
assertEquals(2, clone.getAssociatedRecords().get("child").size());
|
||||
assertEquals("child1", clone.getAssociatedRecords().get("child").get(0).getValue("id"));
|
||||
assertEquals("child2", clone.getAssociatedRecords().get("child").get(1).getValue("id"));
|
||||
assertEquals(1, clone.getAssociatedRecords().get("nephew").size());
|
||||
assertEquals("nephew1", clone.getAssociatedRecords().get("nephew").get(0).getValue("id"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// make sure the associated record data structures are not the same (e.g., not the same map & lists) //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
assertNotSame(clone.getAssociatedRecords(), original.getAssociatedRecords());
|
||||
assertNotSame(clone.getAssociatedRecords().get("child"), original.getAssociatedRecords().get("child"));
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////
|
||||
// but we'll be okay with the same records inside the associated records structure //
|
||||
/////////////////////////////////////////////////////////////////////////////////////
|
||||
assertSame(clone.getAssociatedRecords().get("child").get(0), original.getAssociatedRecords().get("child").get(0));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCopyConstructorEdgeCases()
|
||||
{
|
||||
QRecord nullValuesRecord = new QRecord();
|
||||
nullValuesRecord.setValues(null);
|
||||
assertNull(new QRecord(nullValuesRecord).getValues());
|
||||
|
||||
QRecord nullDisplayValuesRecord = new QRecord();
|
||||
nullDisplayValuesRecord.setDisplayValues(null);
|
||||
assertNull(new QRecord(nullDisplayValuesRecord).getDisplayValues());
|
||||
|
||||
QRecord nullBackendDetailsRecord = new QRecord();
|
||||
nullBackendDetailsRecord.setBackendDetails(null);
|
||||
assertNull(new QRecord(nullBackendDetailsRecord).getBackendDetails());
|
||||
|
||||
QRecord nullAssociations = new QRecord();
|
||||
nullAssociations.setAssociatedRecords(null);
|
||||
assertNull(new QRecord(nullAssociations).getAssociatedRecords());
|
||||
|
||||
QRecord nullErrors = new QRecord();
|
||||
nullErrors.setErrors(null);
|
||||
assertNull(new QRecord(nullErrors).getErrors());
|
||||
|
||||
QRecord nullWarnings = new QRecord();
|
||||
nullWarnings.setWarnings(null);
|
||||
assertNull(new QRecord(nullWarnings).getWarnings());
|
||||
|
||||
QRecord emptyRecord = new QRecord();
|
||||
QRecord emptyClone = new QRecord(emptyRecord);
|
||||
assertNull(emptyClone.getTableName());
|
||||
assertNull(emptyClone.getRecordLabel());
|
||||
assertEquals(0, emptyClone.getValues().size());
|
||||
assertEquals(0, emptyClone.getDisplayValues().size());
|
||||
assertEquals(0, emptyClone.getBackendDetails().size());
|
||||
assertEquals(0, emptyClone.getErrors().size());
|
||||
assertEquals(0, emptyClone.getWarnings().size());
|
||||
assertEquals(0, emptyClone.getAssociatedRecords().size());
|
||||
}
|
||||
|
||||
}
|
||||
@ -23,17 +23,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.basepull;
|
||||
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
@ -74,27 +71,4 @@ class ExtractViaBasepullQueryStepTest extends BaseTest
|
||||
assertTrue(queryFilter.getOrderBys().get(0).getIsAscending());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testWillTheBasePullQueryBeUsed()
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// only time the base-pull query will be used is if there isn't a filter or records in the process input. //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
assertTrue(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()));
|
||||
|
||||
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
|
||||
.withValues(Map.of("recordIds", "1,2,3", StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, "person"))));
|
||||
|
||||
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
|
||||
.withValues(Map.of(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER, new QQueryFilter()))));
|
||||
|
||||
assertFalse(new ExtractViaBasepullQueryStep().willTheBasePullQueryBeUsed(new RunBackendStepInput()
|
||||
.withValues(Map.of("queryFilterJson", "{}"))));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -32,8 +32,6 @@ import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryRecordLink;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
@ -312,7 +310,7 @@ class BulkEditTest extends BaseTest
|
||||
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
List<ProcessSummaryLineInterface> processSummaryLines = (List<ProcessSummaryLineInterface>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
List<ProcessSummaryLine> processSummaryLines = (List<ProcessSummaryLine>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
assertThat(processSummaryLines).hasSize(1 + 50 + 1 + 30 + 1);
|
||||
|
||||
int index = 0;
|
||||
@ -325,7 +323,7 @@ class BulkEditTest extends BaseTest
|
||||
{
|
||||
assertThat(processSummaryLines.get(index++))
|
||||
.hasFieldOrPropertyWithValue("status", Status.ERROR)
|
||||
.isInstanceOf(ProcessSummaryRecordLink.class) // this is because it's a singleton, so we get rid of the "1 had an error" thing (doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
.hasFieldOrPropertyWithValue("count", 1)
|
||||
.matches(psl -> psl.getMessage().contains("less than 60 is error"), "expected message");
|
||||
}
|
||||
|
||||
@ -338,7 +336,7 @@ class BulkEditTest extends BaseTest
|
||||
{
|
||||
assertThat(processSummaryLines.get(index++))
|
||||
.hasFieldOrPropertyWithValue("status", Status.WARNING)
|
||||
.isInstanceOf(ProcessSummaryRecordLink.class) // this is because it's a singleton, so we get rid of the "1 had an error" thing (doReplaceSingletonCountLinesWithSuffixOnly)
|
||||
.hasFieldOrPropertyWithValue("count", 1)
|
||||
.matches(psl -> psl.getMessage().contains("less than 90 is warning"), "expected message");
|
||||
}
|
||||
|
||||
|
||||
@ -62,42 +62,6 @@ class BulkInsertTest extends BaseTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow1()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow2()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvHeaderUsingLabels()
|
||||
{
|
||||
return ("""
|
||||
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -113,7 +77,7 @@ class BulkInsertTest extends BaseTest
|
||||
// create an uploaded file, similar to how an http server may //
|
||||
////////////////////////////////////////////////////////////////
|
||||
QUploadedFile qUploadedFile = new QUploadedFile();
|
||||
qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
|
||||
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
|
||||
qUploadedFile.setFilename("test.csv");
|
||||
UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
|
||||
TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);
|
||||
|
||||
@ -40,6 +40,7 @@ import org.json.JSONException;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
@ -81,7 +82,7 @@ class JsonUtilsTest extends BaseTest
|
||||
{
|
||||
objectMapper.setSerializationInclusion(JsonInclude.Include.ALWAYS);
|
||||
});
|
||||
|
||||
|
||||
assertThat(json).contains("""
|
||||
"values":{"foo":"Foo","bar":3.14159,"baz":null}""");
|
||||
}
|
||||
@ -138,6 +139,86 @@ class JsonUtilsTest extends BaseTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_sortJSONArray() throws Exception
|
||||
{
|
||||
JSONArray jsonArray1 = new JSONArray("""
|
||||
[
|
||||
{
|
||||
"Foo": "Bar",
|
||||
"Baz": [1, 2, 3]
|
||||
}
|
||||
]
|
||||
""");
|
||||
|
||||
JSONArray jsonArray2 = new JSONArray("""
|
||||
[
|
||||
{
|
||||
"Baz": [1, 2, 3],
|
||||
"Foo": "Bar"
|
||||
}
|
||||
]
|
||||
""");
|
||||
|
||||
JSONArray sortedJSONArray1 = (JSONArray) JsonUtils.sortJSON(jsonArray1);
|
||||
JSONArray sortedJSONArray2 = (JSONArray) JsonUtils.sortJSON(jsonArray2);
|
||||
assertEquals(sortedJSONArray1.toString(), sortedJSONArray2.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_sortJSONObject() throws Exception
|
||||
{
|
||||
JSONObject jsonObject1 = new JSONObject("""
|
||||
{
|
||||
"Foo": "Bar",
|
||||
"Baz": [1, 2, 3]
|
||||
}
|
||||
""");
|
||||
JSONObject jsonObject2 = new JSONObject("""
|
||||
{
|
||||
"Baz": [1, 2, 3],
|
||||
"Foo": "Bar"
|
||||
}
|
||||
""");
|
||||
|
||||
JSONObject sortedJSONObject1 = (JSONObject) JsonUtils.sortJSON(jsonObject1);
|
||||
JSONObject sortedJSONObject2 = (JSONObject) JsonUtils.sortJSON(jsonObject2);
|
||||
assertEquals(sortedJSONObject1.toString(), sortedJSONObject2.toString());
|
||||
|
||||
JSONObject bigObject1 = new JSONObject("""
|
||||
{ "cubeiq": { "settings": { "setting": [ { "maxruntime": 60, "maxnonimproveiters": 750, "settingsid": "Box Default", "sequencemixok": true } ] }, "containerstoload": { "containertoload": [ { "containernum": 99, "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "containerid": "25_4976f3f8-1208-430f-aee4-59fc1d1f4b3f" } ] }, "loads": { "load": [ { "date": "1-1-2015", "notes": "", "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0" } ] }, "stages": { "stage": [ { "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "stage": 1 } ] }, "containers": { "container": [ { "depth": 5, "bottomtotoploading": false, "width": 5, "action": "overwrite", "partialloadonfloor": true, "settingsid": "Box Default", "containerid": "25_4976f3f8-1208-430f-aee4-59fc1d1f4b3f", "type": "Rectangular", "height": 10 } ] }, "productstoload": { "producttoload": [ { "quantity": 1, "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "productid": "DRY-ICE", "batch": 1 } ] }, "products": { "product": [ { "productid": "DRY-ICE", "color": 16760576, "length": 3, "weight": 1, "maxinlayer": 999999, "type": "Box", "sideupok": true, "endupok": true, "turnable": true, "bottomonly": false, "width": 3, "toponly": false, "flatok": true, "height": 1 } ] } } }
|
||||
""");
|
||||
|
||||
JSONObject bigObject2 = new JSONObject("""
|
||||
{ "cubeiq": { "containerstoload": { "containertoload": [ { "containernum": 99, "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "containerid": "25_4976f3f8-1208-430f-aee4-59fc1d1f4b3f" } ] }, "settings": { "setting": [ { "maxruntime": 60, "maxnonimproveiters": 750, "settingsid": "Box Default", "sequencemixok": true } ] }, "loads": { "load": [ { "date": "1-1-2015", "notes": "", "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0" } ] }, "stages": { "stage": [ { "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "stage": 1 } ] }, "containers": { "container": [ { "depth": 5, "bottomtotoploading": false, "action": "overwrite", "width": 5, "partialloadonfloor": true, "settingsid": "Box Default", "containerid": "25_4976f3f8-1208-430f-aee4-59fc1d1f4b3f", "type": "Rectangular", "height": 10 } ] }, "products": { "product": [ { "productid": "DRY-ICE", "color": 16760576, "weight": 1, "length": 3, "sideupok": true, "maxinlayer": 999999, "type": "Box", "endupok": true, "toponly": false, "turnable": true, "bottomonly": false, "width": 3, "flatok": true, "height": 1 } ] }, "productstoload": { "producttoload": [ { "quantity": 1, "loadid": "4814b7a4-dc8a-43b8-8772-af86cb11d9a0", "productid": "DRY-ICE", "batch": 1 } ] } } }
|
||||
""");
|
||||
sortedJSONObject1 = (JSONObject) JsonUtils.sortJSON(bigObject1);
|
||||
sortedJSONObject2 = (JSONObject) JsonUtils.sortJSON(bigObject2);
|
||||
assertEquals(sortedJSONObject1.toString(), sortedJSONObject2.toString());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void test_sortNonJSONThrows() throws Exception
|
||||
{
|
||||
assertThatThrownBy(() -> JsonUtils.sortJSON("derp"))
|
||||
.hasMessageContaining("must be of type");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
|
||||
@ -1222,24 +1222,6 @@ public class TestUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static void insertExtraShapes(QInstance qInstance) throws QException
|
||||
{
|
||||
List<QRecord> shapeRecords = List.of(
|
||||
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 4).withValue("name", "Rectangle"),
|
||||
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 5).withValue("name", "Pentagon"),
|
||||
new QRecord().withTableName(TABLE_NAME_SHAPE).withValue("id", 6).withValue("name", "Hexagon"));
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TABLE_NAME_SHAPE);
|
||||
insertInput.setRecords(shapeRecords);
|
||||
new InsertAction().execute(insertInput);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -1252,6 +1234,41 @@ public class TestUtils
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvHeaderUsingLabels()
|
||||
{
|
||||
return ("""
|
||||
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow1()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
public static String getPersonCsvRow2()
|
||||
{
|
||||
return ("""
|
||||
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com"
|
||||
""");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
|
||||
@ -83,15 +83,7 @@ class ValueUtilsTest extends BaseTest
|
||||
assertTrue(ValueUtils.getValueAsBoolean("True"));
|
||||
assertTrue(ValueUtils.getValueAsBoolean("TRUE"));
|
||||
assertFalse(ValueUtils.getValueAsBoolean("false"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// time used to be, that "yes" was false... changing that 2023-10-20 //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
assertTrue(ValueUtils.getValueAsBoolean("yes"));
|
||||
assertTrue(ValueUtils.getValueAsBoolean("Yes"));
|
||||
assertTrue(ValueUtils.getValueAsBoolean("YES"));
|
||||
assertTrue(ValueUtils.getValueAsBoolean("yES"));
|
||||
|
||||
assertFalse(ValueUtils.getValueAsBoolean("yes"));
|
||||
assertFalse(ValueUtils.getValueAsBoolean("t"));
|
||||
assertFalse(ValueUtils.getValueAsBoolean(new Object()));
|
||||
assertFalse(ValueUtils.getValueAsBoolean(1));
|
||||
|
||||
@ -1,149 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.collections;
|
||||
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for AlphaNumericComparator
|
||||
*******************************************************************************/
|
||||
class AlphaNumericComparatorTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
** test odd-balls
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testFringeCases()
|
||||
{
|
||||
test(sort("", null, "foo", " ", "", "1", null),
|
||||
null, null, "", "", "1", " ", "foo");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test alpha-strings only
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testAlphasOnly()
|
||||
{
|
||||
test(sort("F", "G", "A", "AB", "BB", "BA", "BD"),
|
||||
"A", "AB", "BA", "BB", "BD", "F", "G");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test numbers only
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testNumbersOnly()
|
||||
{
|
||||
test(sort("1", "273", "271", "102", "101", "10", "13", "2", "22", "273"),
|
||||
"1", "2", "10", "13", "22", "101", "102", "271", "273", "273");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test mixed
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMixed1()
|
||||
{
|
||||
test(sort("1", "A", "A1", "1A", "10", "10AA", "11", "A11", "11B", "1B", "A10B2", "A10B10", "D1", "D10", "D2", "F20G11H10", "F3", "F20G11H2", "A1", "A10", "A2", "01", "001"),
|
||||
"001", "01", "1", "1A", "1B", "10", "10AA", "11", "11B", "A", "A1", "A1", "A2", "A10", "A10B2", "A10B10", "A11", "D1", "D2", "D10", "F3", "F20G11H2", "F20G11H10");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** test mixed
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testMixed2()
|
||||
{
|
||||
test(sort("A", "A001", "A1", "A0000", "A00001", "000023", "023", "000023", "023A", "23", "2", "0002", "02"),
|
||||
"0002", "02", "2", "000023", "000023", "023", "23", "023A", "A", "A0000", "A00001", "A001", "A1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
**
|
||||
*******************************************************************************/
|
||||
private void test(List<String> a, String... b)
|
||||
{
|
||||
System.out.println("Expecting: " + Arrays.asList(b));
|
||||
|
||||
assertEquals(a.size(), b.length);
|
||||
|
||||
for(int i = 0; i < a.size(); i++)
|
||||
{
|
||||
String aString = a.get(i);
|
||||
String bString = b[i];
|
||||
|
||||
assertEquals(aString, bString);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> sort(String... input)
|
||||
{
|
||||
List<String> inputList = Arrays.asList(input);
|
||||
System.out.println("Sorting: " + inputList);
|
||||
|
||||
try
|
||||
{
|
||||
List<String> naturalSortList = Arrays.asList(input);
|
||||
Collections.sort(naturalSortList);
|
||||
System.out.println("Natural: " + naturalSortList);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
System.out.println("Natural: FAILED");
|
||||
}
|
||||
|
||||
inputList.sort(new AlphaNumericComparator());
|
||||
System.out.println("Produced: " + inputList);
|
||||
return (inputList);
|
||||
}
|
||||
}
|
||||
@ -1,174 +0,0 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2023. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qqq.backend.core.utils.memoization;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||
import com.kingsrook.qqq.backend.core.utils.SleepUtils;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for Memoization
|
||||
*******************************************************************************/
|
||||
class MemoizationTest extends BaseTest
|
||||
{
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void test()
|
||||
{
|
||||
Memoization<String, Integer> memoization = new Memoization<>();
|
||||
memoization.setMaxSize(3);
|
||||
memoization.setTimeout(Duration.ofMillis(100));
|
||||
|
||||
assertThat(memoization.getResult("one")).isEmpty();
|
||||
memoization.storeResult("one", 1);
|
||||
assertThat(memoization.getResult("one")).isPresent().get().isEqualTo(1);
|
||||
|
||||
////////////////////////////////////////////////////
|
||||
// store 3 more results - this should force 1 out //
|
||||
////////////////////////////////////////////////////
|
||||
memoization.storeResult("two", 2);
|
||||
memoization.storeResult("three", 3);
|
||||
memoization.storeResult("four", 4);
|
||||
assertThat(memoization.getResult("one")).isEmpty();
|
||||
|
||||
//////////////////////////////////
|
||||
// make sure others are present //
|
||||
//////////////////////////////////
|
||||
assertThat(memoization.getResult("two")).isPresent().get().isEqualTo(2);
|
||||
assertThat(memoization.getResult("three")).isPresent().get().isEqualTo(3);
|
||||
assertThat(memoization.getResult("four")).isPresent().get().isEqualTo(4);
|
||||
|
||||
/////////////////////////////////////////////////////////////
|
||||
// wait more than the timeout, then make sure all are gone //
|
||||
/////////////////////////////////////////////////////////////
|
||||
SleepUtils.sleep(150, TimeUnit.MILLISECONDS);
|
||||
assertThat(memoization.getResult("two")).isEmpty();
|
||||
assertThat(memoization.getResult("three")).isEmpty();
|
||||
assertThat(memoization.getResult("four")).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testCanStoreNull()
|
||||
{
|
||||
Memoization<String, Integer> memoization = new Memoization<>();
|
||||
memoization.storeResult("null", null);
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
// note - we can't tell a stored null apart from a non-stored value by calling getResult //
|
||||
///////////////////////////////////////////////////////////////////////////////////////////
|
||||
Optional<Integer> optionalNull = memoization.getResult("null");
|
||||
assertNotNull(optionalNull);
|
||||
assertTrue(optionalNull.isEmpty());
|
||||
|
||||
////////////////////////////////////////////
|
||||
// instead, we must use getMemoizedResult //
|
||||
////////////////////////////////////////////
|
||||
Optional<MemoizedResult<Integer>> optionalMemoizedResult = memoization.getMemoizedResult("null");
|
||||
assertNotNull(optionalMemoizedResult);
|
||||
assertTrue(optionalMemoizedResult.isPresent());
|
||||
assertNull(optionalMemoizedResult.get().getResult());
|
||||
|
||||
/////////////////////////////////////////////////////////////////
|
||||
// make sure getMemoizedResult returns empty for an un-set key //
|
||||
/////////////////////////////////////////////////////////////////
|
||||
optionalMemoizedResult = memoization.getMemoizedResult("never-stored");
|
||||
assertNotNull(optionalMemoizedResult);
|
||||
assertTrue(optionalMemoizedResult.isEmpty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
@Disabled("Slow, so not for CI - but good to demonstrate thread-safety during dev")
|
||||
void testMultiThread() throws InterruptedException, ExecutionException
|
||||
{
|
||||
Memoization<String, Integer> memoization = new Memoization<>();
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(20);
|
||||
|
||||
List<Future<?>> futures = new ArrayList<>();
|
||||
|
||||
for(int i = 0; i < 20; i++)
|
||||
{
|
||||
int finalI = i;
|
||||
futures.add(executorService.submit(() ->
|
||||
{
|
||||
System.out.println("Start " + finalI);
|
||||
for(int n = 0; n < 1_000_000; n++)
|
||||
{
|
||||
memoization.storeResult(String.valueOf(n), n);
|
||||
memoization.getResult(String.valueOf(n));
|
||||
|
||||
if(n % 100_000 == 0)
|
||||
{
|
||||
System.out.format("Thread %d at %,d\n", finalI, +n);
|
||||
}
|
||||
}
|
||||
System.out.println("End " + finalI);
|
||||
}));
|
||||
}
|
||||
|
||||
while(!futures.isEmpty())
|
||||
{
|
||||
Iterator<Future<?>> iterator = futures.iterator();
|
||||
while(iterator.hasNext())
|
||||
{
|
||||
Future<?> next = iterator.next();
|
||||
if(next.isDone())
|
||||
{
|
||||
Object o = next.get();
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
System.out.println("All Done");
|
||||
}
|
||||
|
||||
}
|
||||
@ -27,7 +27,6 @@ import java.io.Serializable;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
@ -787,7 +786,7 @@ public class BaseAPIActionUtil
|
||||
try(CloseableHttpClient client = HttpClients.custom().setConnectionManager(new PoolingHttpClientConnectionManager()).build())
|
||||
{
|
||||
HttpPost request = new HttpPost(fullURL);
|
||||
request.setEntity(new StringEntity(postBody, getCharsetForEntity()));
|
||||
request.setEntity(new StringEntity(postBody));
|
||||
|
||||
if(setCredentialsInHeader)
|
||||
{
|
||||
@ -830,16 +829,6 @@ public class BaseAPIActionUtil
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Let a subclass change what charset to use for entities (bodies) being posted/put/etc.
|
||||
*******************************************************************************/
|
||||
protected static Charset getCharsetForEntity()
|
||||
{
|
||||
return StandardCharsets.UTF_8;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** one-line method, factored out so mock/tests can override
|
||||
*******************************************************************************/
|
||||
@ -925,7 +914,7 @@ public class BaseAPIActionUtil
|
||||
body.put(wrapperObjectName, new JSONObject(json));
|
||||
json = body.toString();
|
||||
}
|
||||
return (new StringEntity(json, getCharsetForEntity()));
|
||||
return (new StringEntity(json));
|
||||
}
|
||||
|
||||
|
||||
@ -954,7 +943,7 @@ public class BaseAPIActionUtil
|
||||
body.put(wrapperObjectName, new JSONArray(json));
|
||||
json = body.toString();
|
||||
}
|
||||
return (new StringEntity(json, getCharsetForEntity()));
|
||||
return (new StringEntity(json));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
|
||||
@ -63,18 +63,14 @@ import com.kingsrook.qqq.backend.module.api.model.OutboundAPILog;
|
||||
import com.kingsrook.qqq.backend.module.api.model.OutboundAPILogMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.junit.jupiter.api.Assertions.fail;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
@ -343,32 +339,10 @@ class BaseAPIActionUtilTest extends BaseTest
|
||||
mockApiUtilsHelper.enqueueMockResponse("""
|
||||
{"id": 6}
|
||||
""");
|
||||
mockApiUtilsHelper.withMockRequestAsserter(httpRequestBase ->
|
||||
{
|
||||
HttpEntity entity = ((HttpEntityEnclosingRequestBase) httpRequestBase).getEntity();
|
||||
byte[] bytes = entity.getContent().readAllBytes();
|
||||
String requestBody = new String(bytes, StandardCharsets.UTF_8);
|
||||
|
||||
///////////////////////////////////////
|
||||
// default ISO-8559-1: ... a0 ... //
|
||||
// updated UTF-8: ... c2 a0 ... //
|
||||
///////////////////////////////////////
|
||||
byte previousByte = 0;
|
||||
for(byte b : bytes)
|
||||
{
|
||||
if(b == (byte) 0xa0 && previousByte != (byte) 0xc2)
|
||||
{
|
||||
fail("Found byte 0xa0 (without being prefixed by 0xc2) - so this is invalid UTF-8!");
|
||||
}
|
||||
previousByte = b;
|
||||
}
|
||||
|
||||
assertThat(requestBody).contains("van Houten");
|
||||
});
|
||||
|
||||
InsertInput insertInput = new InsertInput();
|
||||
insertInput.setTableName(TestUtils.MOCK_TABLE_NAME);
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse van Houten")));
|
||||
insertInput.setRecords(List.of(new QRecord().withValue("name", "Milhouse")));
|
||||
InsertOutput insertOutput = new InsertAction().execute(insertInput);
|
||||
assertEquals(6, insertOutput.getRecords().get(0).getValueInteger("id"));
|
||||
}
|
||||
|
||||
@ -33,9 +33,7 @@
|
||||
|
||||
<properties>
|
||||
<!-- props specifically to this module -->
|
||||
|
||||
<!-- temp - disable this when localstack is fixed -->
|
||||
<coverage.haltOnFailure>false</coverage.haltOnFailure>
|
||||
<!-- none at this time -->
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@ -44,7 +44,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModuleSubclassFor
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
@ -53,7 +52,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
/*******************************************************************************
|
||||
** Unit test for FilesystemSyncProcess using S3 backend
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
class FilesystemSyncProcessS3Test extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -31,14 +31,12 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Unit test for S3BackendModule
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3BackendModuleTest extends BaseS3Test
|
||||
{
|
||||
private final String PATH_THAT_WONT_EXIST = "some/path/that/wont/exist";
|
||||
|
||||
@ -28,14 +28,12 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3CountActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -26,7 +26,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@ -34,7 +33,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3DeleteActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -35,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
@ -45,7 +44,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3InsertActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -29,14 +29,12 @@ import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3QueryActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -26,7 +26,6 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.lang.NotImplementedException;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
@ -34,7 +33,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3UpdateActionTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -28,7 +28,6 @@ import java.util.List;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.jupiter.api.Disabled;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
@ -36,7 +35,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Disabled("Because localstack won't start")
|
||||
public class S3UtilsTest extends BaseS3Test
|
||||
{
|
||||
|
||||
|
||||
@ -149,7 +149,7 @@ public abstract class AbstractRDBMSAction implements QActionInterface
|
||||
** and type conversions that we can do "better" than jdbc...
|
||||
**
|
||||
*******************************************************************************/
|
||||
protected Serializable scrubValue(QFieldMetaData field, Serializable value)
|
||||
protected Serializable scrubValue(QFieldMetaData field, Serializable value, boolean isInsert)
|
||||
{
|
||||
if("".equals(value))
|
||||
{
|
||||
@ -193,7 +193,8 @@ public abstract class AbstractRDBMSAction implements QActionInterface
|
||||
{
|
||||
try
|
||||
{
|
||||
if(table.getFields().containsKey(fieldName))
|
||||
QFieldMetaData field = table.getField(fieldName);
|
||||
if(field != null)
|
||||
{
|
||||
record.setValue(fieldName, value);
|
||||
}
|
||||
@ -724,10 +725,9 @@ public abstract class AbstractRDBMSAction implements QActionInterface
|
||||
throw new IllegalArgumentException("Incorrect number of values given for criteria [" + field.getName() + "]");
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// replace any expression-type values with their evaluation //
|
||||
// also, "scrub" non-expression values, which type-converts them (e.g., strings in various supported date formats become LocalDate) //
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////
|
||||
// replace any expression-type values with their evaluation //
|
||||
//////////////////////////////////////////////////////////////
|
||||
ListIterator<Serializable> valueListIterator = values.listIterator();
|
||||
while(valueListIterator.hasNext())
|
||||
{
|
||||
@ -736,11 +736,6 @@ public abstract class AbstractRDBMSAction implements QActionInterface
|
||||
{
|
||||
valueListIterator.set(expression.evaluate());
|
||||
}
|
||||
else
|
||||
{
|
||||
Serializable scrubbedValue = scrubValue(field, value);
|
||||
valueListIterator.set(scrubbedValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -135,7 +135,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
|
||||
for(QFieldMetaData field : insertableFields)
|
||||
{
|
||||
Serializable value = record.getValue(field.getName());
|
||||
value = scrubValue(field, value);
|
||||
value = scrubValue(field, value, true);
|
||||
params.add(value);
|
||||
}
|
||||
}
|
||||
|
||||
@ -214,7 +214,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
|
||||
for(String fieldName : fieldsBeingUpdated)
|
||||
{
|
||||
Serializable value = record.getValue(fieldName);
|
||||
value = scrubValue(table.getField(fieldName), value);
|
||||
value = scrubValue(table.getField(fieldName), value, false);
|
||||
rowValues.add(value);
|
||||
}
|
||||
rowValues.add(record.getValue(table.getPrimaryKeyField()));
|
||||
@ -286,7 +286,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
|
||||
for(String fieldName : fieldsBeingUpdated)
|
||||
{
|
||||
Serializable value = record0.getValue(fieldName);
|
||||
value = scrubValue(table.getField(fieldName), value);
|
||||
value = scrubValue(table.getField(fieldName), value, false);
|
||||
params.add(value);
|
||||
}
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
0.20.0
|
||||
0.19.0
|
||||
|
||||
@ -859,9 +859,6 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
|
||||
|
||||
apiProcessMetaDataList.add(Pair.of(apiProcessMetaData, processMetaData));
|
||||
}
|
||||
|
||||
apiProcessMetaDataList.sort(Comparator.comparing(apiProcessMetaDataQProcessMetaDataPair -> getProcessSummary(apiProcessMetaDataQProcessMetaDataPair.getA(), apiProcessMetaDataQProcessMetaDataPair.getB())));
|
||||
|
||||
return (apiProcessMetaDataList);
|
||||
}
|
||||
|
||||
@ -888,7 +885,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
|
||||
Method methodForProcess = new Method()
|
||||
.withOperationId(apiProcessMetaData.getApiProcessName())
|
||||
.withTags(tags)
|
||||
.withSummary(getProcessSummary(apiProcessMetaData, processMetaData))
|
||||
.withSummary(ObjectUtils.requireConditionElse(apiProcessMetaData.getSummary(), StringUtils::hasContent, processMetaData.getLabel()))
|
||||
.withDescription(description)
|
||||
.withSecurity(getSecurity(apiInstanceMetaData, processMetaData.getName()));
|
||||
|
||||
@ -1021,16 +1018,6 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String getProcessSummary(ApiProcessMetaData apiProcessMetaData, QProcessMetaData processMetaData)
|
||||
{
|
||||
return ObjectUtils.requireConditionElse(apiProcessMetaData.getSummary(), StringUtils::hasContent, processMetaData.getLabel());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -1042,7 +1029,7 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
|
||||
Method methodForProcess = new Method()
|
||||
.withOperationId("getStatusFor" + StringUtils.ucFirst(apiProcessMetaData.getApiProcessName()))
|
||||
.withTags(tags)
|
||||
.withSummary("Get Status of Job: " + getProcessSummary(apiProcessMetaData, processMetaData))
|
||||
.withSummary("Get Status of Job: " + ObjectUtils.requireConditionElse(apiProcessMetaData.getSummary(), StringUtils::hasContent, processMetaData.getLabel()))
|
||||
.withDescription("Get the status for a previous asynchronous call to the process named " + processMetaData.getLabel())
|
||||
.withSecurity(getSecurity(apiInstanceMetaData, processMetaData.getName()));
|
||||
|
||||
@ -1171,12 +1158,6 @@ public class GenerateOpenApiSpecAction extends AbstractQActionFunction<GenerateO
|
||||
|
||||
apiProcessMetaDataList.add(Pair.of(apiProcessMetaData, processMetaData));
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
// sort by process summary (for stability, and just to be better) //
|
||||
/////////////////////////////////////////////////////////////////////
|
||||
apiProcessMetaDataList.sort(Comparator.comparing(apiProcessMetaDataQProcessMetaDataPair -> getProcessSummary(apiProcessMetaDataQProcessMetaDataPair.getA(), apiProcessMetaDataQProcessMetaDataPair.getB())));
|
||||
|
||||
return (apiProcessMetaDataList);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user