QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-17 20:50:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feature/string-utils-safe-equals-ignore-case

Browse Source
This commit is contained in:
Tim Chamberlain
2025-05-12 15:47:09 -05:00
parent bc76a7f66f ce2ca3f413
commit 6d0f5d4fb3
19 changed files with 572 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java
View File

@ -35,6 +35,7 @@ import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
import org.jsoup.Jsoup;
import org.jsoup.helper.W3CDom;
import org.jsoup.nodes.Document;
@ -67,6 +68,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
//////////////////////////////////////////////////////////////////
Document document = Jsoup.parse(input.getHtml());
document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
org.w3c.dom.Document w3cDoc = new W3CDom().fromJsoup(document);
//////////////////////////////
// convert the XHTML to PDF //
@ -74,7 +76,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
PdfRendererBuilder builder = new PdfRendererBuilder();
builder.toStream(input.getOutputStream());
builder.useFastMode();
builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
builder.withW3cDocument(w3cDoc, input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
{

91
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java Normal file
View File

@ -0,0 +1,91 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.values;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
/*******************************************************************************
** Basic implementation of a possible value provider, for where there's a limited
** set of possible source objects - so you just have to define how to make one
** PV from a source object, how to list all of the source objects, and how to
** look up a PV from an id.
*******************************************************************************/
public abstract class BasicCustomPossibleValueProvider<S, ID extends Serializable> implements QCustomPossibleValueProvider<ID>
{
/***************************************************************************
**
***************************************************************************/
protected abstract QPossibleValue<ID> makePossibleValue(S sourceObject);
/***************************************************************************
**
***************************************************************************/
protected abstract S getSourceObject(Serializable id);
/***************************************************************************
**
***************************************************************************/
protected abstract List<S> getAllSourceObjects();
/***************************************************************************
**
***************************************************************************/
@Override
public QPossibleValue<ID> getPossibleValue(Serializable idValue)
{
S sourceObject = getSourceObject(idValue);
if(sourceObject == null)
{
return (null);
}
return makePossibleValue(sourceObject);
}
/***************************************************************************
**
***************************************************************************/
@Override
public List<QPossibleValue<ID>> search(SearchPossibleValueSourceInput input) throws QException
{
List<QPossibleValue<ID>> allPossibleValues = new ArrayList<>();
List<S> allSourceObjects = getAllSourceObjects();
for(S sourceObject : allSourceObjects)
{
allPossibleValues.add(makePossibleValue(sourceObject));
}
return completeCustomPVSSearch(input, allPossibleValues);
}
}

1
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java
View File

@ -66,6 +66,7 @@ public enum WidgetType
// record view/edit widgets //
//////////////////////////////
CHILD_RECORD_LIST("childRecordList"),
CUSTOM_COMPONENT("customComponent"),
DYNAMIC_FORM("dynamicForm"),
DATA_BAG_VIEWER("dataBagViewer"),
PIVOT_TABLE_SETUP("pivotTableSetup"),

70
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java
View File

@ -27,11 +27,9 @@ import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
** possible-value source provider for the `Tables` PVS - a list of all tables
** in an application/qInstance.
*******************************************************************************/
public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider<QTableMetaData, String>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QPossibleValue<String> getPossibleValue(Serializable idValue)
protected QPossibleValue<String> makePossibleValue(QTableMetaData sourceObject)
{
QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
if(table != null && !table.getIsHidden())
{
PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
{
return (new QPossibleValue<>(table.getName(), table.getLabel()));
}
}
return null;
return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel()));
}
@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr
**
***************************************************************************/
@Override
public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
protected QTableMetaData getSourceObject(Serializable id)
{
/////////////////////////////////////////////////////////////////////////////////////
// build all of the possible values (note, will be filtered by user's permissions) //
/////////////////////////////////////////////////////////////////////////////////////
List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id));
return isTableAllowed(table) ? table : null;
}
/***************************************************************************
**
***************************************************************************/
@Override
protected List<QTableMetaData> getAllSourceObjects()
{
ArrayList<QTableMetaData> rs = new ArrayList<>();
for(QTableMetaData table : QContext.getQInstance().getTables().values())
{
QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
if(possibleValue != null)
if(isTableAllowed(table))
{
allPossibleValues.add(possibleValue);
rs.add(table);
}
}
return rs;
}
return completeCustomPVSSearch(input, allPossibleValues);
/***************************************************************************
**
***************************************************************************/
private boolean isTableAllowed(QTableMetaData table)
{
if(table == null)
{
return (false);
}
if(table.getIsHidden())
{
return (false);
}
PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult))
{
return (false);
}
return (true);
}
}

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider
{
QPossibleValueSource possibleValueSource = new QPossibleValueSource()
.withName(NAME)
.withIdType(QFieldType.STRING)
.withType(QPossibleValueSourceType.CUSTOM)
.withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);

2
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java
View File

@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);

65
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java
View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
/////////////////////////////////////////////////////////////////////////////////////////////
// the transform step builds summary lines that it predicts will insert successfully. //
// but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
// might link to the built record). also, it's possible that there was a fail that only //
// happened in the actual insert, so, basically, re-do the summary here //
/////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
ProcessSummaryLine okSummary = transformStep.okSummary;
okSummary.setCount(0);
okSummary.setPrimaryKeys(new ArrayList<>());
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// but - since errors from the transform step don't even make it through to us here in the load step, //
// do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: //
// clear out its warnings - we'll completely rebuild them here (with primary keys) //
// and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
processSummaryWarningsAndErrorsRollup.resetWarnings();
List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
for(QRecord insertedRecord : insertedRecords)
{
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null)
{
/////////////////////////////////////////////////////////////////////////
// if the record had no errors, and we have a primary key for it, then //
// keep track of the range of primary keys (first and last) //
/////////////////////////////////////////////////////////////////////////
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
firstInsertedPrimaryKey = primaryKey;
}
lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
lastInsertedPrimaryKey = primaryKey;
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings()))
{
/////////////////////////////////////////////////////////////////////////////
// if there were warnings on the inserted record, put it in a warning line //
/////////////////////////////////////////////////////////////////////////////
String message = insertedRecord.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
}
else
{
////////////////////////////////////////////////////////////////////////
// if no warnings for the inserted record, then put it in the OK line //
////////////////////////////////////////////////////////////////////////
okSummary.incrementCountAndAddPrimaryKey(primaryKey);
}
}
else
{
//////////////////////////////////////////////////////////////////////
// else if there were errors or no primary key, build an error line //
//////////////////////////////////////////////////////////////////////
String message = "Failed to insert";
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
{
//////////////////////////////////////////////////////////
// use the error message from the record if we have one //
//////////////////////////////////////////////////////////
message = insertedRecord.getErrors().get(0).getMessage();
}
processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
}
}
okSummary.pickMessage(true);
}

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java
View File

@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////
// for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

14
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java
View File

@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
//////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null)
{
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load"));
}
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
{
throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load"));
}
try
{
@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
}
catch(Exception e)
{
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e));
}
}
else
@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
}
}
}
catch(QUserFacingException ufe)
{
LOG.warn("User-facing error in bulk insert receive mapping", ufe);
throw ufe;
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);

4
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java
View File

@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
.withDoReplaceSingletonCountLinesWithSuffixOnly(false);
private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();

23
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java
View File

@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
{
if(otherWarningsSummary == null)
{
otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
otherWarningsSummary = buildOtherWarningsSummary();
}
processSummaryLine = otherWarningsSummary;
}
@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup
/***************************************************************************
**
***************************************************************************/
private static ProcessSummaryLine buildOtherWarningsSummary()
{
return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
}
/***************************************************************************
**
***************************************************************************/
public void resetWarnings()
{
warningSummaries.clear();
otherWarningsSummary = buildOtherWarningsSummary();
}
/*******************************************************************************
** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
** extracts string messages out.

45
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.utils;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -51,13 +52,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class RecordLookupHelper
{
private Map<String, Map<Serializable, QRecord>> recordMaps = new HashMap<>();
private Map<String, Map<Serializable, QRecord>> recordMaps;
private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps = new HashMap<>();
private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps;
private Set<String> preloadedKeys = new HashSet<>();
private Set<String> preloadedKeys;
private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
private Set<Pair<String, String>> disallowedOneOffLookups;
private boolean useSynchronizedCollections;
@ -67,6 +69,33 @@ public class RecordLookupHelper
*******************************************************************************/
public RecordLookupHelper()
{
this(false);
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public RecordLookupHelper(boolean useSynchronizedCollections)
{
this.useSynchronizedCollections = useSynchronizedCollections;
if(useSynchronizedCollections)
{
recordMaps = Collections.synchronizedMap(new HashMap<>());
uniqueKeyMaps = Collections.synchronizedMap(new HashMap<>());
preloadedKeys = Collections.synchronizedSet(new HashSet<>());
disallowedOneOffLookups = Collections.synchronizedSet(new HashSet<>());
}
else
{
recordMaps = new HashMap<>();
uniqueKeyMaps = new HashMap<>();
preloadedKeys = new HashSet<>();
disallowedOneOffLookups = new HashSet<>();
}
}
@ -77,7 +106,7 @@ public class RecordLookupHelper
public QRecord getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey) throws QException
{
String mapKey = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(","));
Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
if(!recordMap.containsKey(uniqueKey))
{
@ -96,7 +125,7 @@ public class RecordLookupHelper
public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
////////////////////////////////////////////////////////////
// make sure we have they key object in the expected type //
@ -150,7 +179,7 @@ public class RecordLookupHelper
public void preloadRecords(String tableName, String keyFieldName, QQueryFilter filter) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));
}
@ -170,7 +199,7 @@ public class RecordLookupHelper
}
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));

3
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state;
import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@ -58,7 +59,7 @@ public class InMemoryStateProvider implements StateProviderInterface
*******************************************************************************/
private InMemoryStateProvider()
{
this.map = new HashMap<>();
this.map = Collections.synchronizedMap(new HashMap<>());
///////////////////////////////////////////////////////////
// Start a single thread executor to handle the cleaning //

1
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfActionTest.java
View File

@ -79,6 +79,7 @@ class ConvertHtmlToPdfActionTest extends BaseTest
</h1>
<div class="myclass">
<p>This is a test of converting HTML to PDF!!</p>
<p>This is &nbsp; a line with &bull; some entities &lt;</p>
<p style="font-family: SF-Pro; font-size: 24px;">(btw, is this in SF-Pro???)</p>
</div>
</div>

9
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java
View File

@ -185,4 +185,13 @@ public class ProcessSummaryLineInterfaceAssert extends AbstractAssert<ProcessSum
return (this);
}
/***************************************************************************
**
***************************************************************************/
public ProcessSummaryLineInterface getLine()
{
return actual;
}
}

185
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java
View File

@ -29,15 +29,26 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
@ -176,7 +187,15 @@ class BulkInsertFullProcessTest extends BaseTest
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2");
ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
.hasLineWithMessageContaining("Person Memory records were inserted")
.hasStatus(Status.OK)
.hasCount(2)
.getLine();
assertEquals(List.of(1, 2), ((ProcessSummaryLine) okLine).getPrimaryKeys());
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO);
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2").hasStatus(Status.INFO);
////////////////////////////////////
// query for the inserted records //
@ -201,6 +220,86 @@ class BulkInsertFullProcessTest extends BaseTest
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSummaryLinePrimaryKeys() throws Exception
{
assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
/////////////////////////////////////////////////////////
// start the process - expect to go to the upload step //
/////////////////////////////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput();
RunProcessOutput runProcessOutput = startProcess(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForWarningCase());
continueProcessPostFileMapping(runProcessInput);
continueProcessPostValueMapping(runProcessInput);
runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
ProcessSummaryLineInterface okLine = ProcessSummaryAssert.assertThat(runProcessOutput)
.hasLineWithMessageContaining("Person Memory record was inserted")
.hasStatus(Status.OK)
.hasCount(1)
.getLine();
assertEquals(List.of(1), ((ProcessSummaryLine) okLine).getPrimaryKeys());
ProcessSummaryLineInterface warnTornadoLine = ProcessSummaryAssert.assertThat(runProcessOutput)
.hasLineWithMessageContaining("records were inserted, but had a warning: Tornado warning")
.hasStatus(Status.WARNING)
.hasCount(2)
.getLine();
assertEquals(List.of(2, 3), ((ProcessSummaryLine) warnTornadoLine).getPrimaryKeys());
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("record was inserted, but had a warning: Hurricane warning").hasStatus(Status.WARNING).hasCount(1);
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("records were processed from the file").hasStatus(Status.INFO).hasCount(4);
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 4").hasStatus(Status.INFO);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testSummaryLineErrors() throws Exception
{
assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)
.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PersonWarnOrErrorCustomizer.class));
/////////////////////////////////////////////////////////
// start the process - expect to go to the upload step //
/////////////////////////////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput();
RunProcessOutput runProcessOutput = startProcess(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
continueProcessPostUpload(runProcessInput, processUUID, simulateFileUploadForErrorCase());
continueProcessPostFileMapping(runProcessInput);
continueProcessPostValueMapping(runProcessInput);
runProcessOutput = continueProcessPostReviewScreen(runProcessInput);
ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was inserted.").hasStatus(Status.OK).hasCount(1);
ProcessSummaryAssert.assertThat(runProcessOutput)
.hasLineWithMessageContaining("plane")
.hasStatus(Status.ERROR)
.hasCount(1);
ProcessSummaryAssert.assertThat(runProcessOutput)
.hasLineWithMessageContaining("purifier")
.hasStatus(Status.ERROR)
.hasCount(1);
}
/*******************************************************************************
**
*******************************************************************************/
@ -301,6 +400,47 @@ class BulkInsertFullProcessTest extends BaseTest
/***************************************************************************
**
***************************************************************************/
private static StorageInput simulateFileUploadForWarningCase() throws Exception
{
String storageReference = UUID.randomUUID() + ".csv";
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
{
outputStream.write((getPersonCsvHeaderUsingLabels() + """
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doe","1980-01-01","john@doe.com","Missouri",42
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Tornado warning","Doey","1980-01-01","john@doe.com","Missouri",42
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Hurricane warning","Doe","1980-01-01","john@doe.com","Missouri",42
""").getBytes());
}
return storageInput;
}
/***************************************************************************
**
***************************************************************************/
private static StorageInput simulateFileUploadForErrorCase() throws Exception
{
String storageReference = UUID.randomUUID() + ".csv";
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
{
outputStream.write((getPersonCsvHeaderUsingLabels() + """
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","not-pre-Error plane","Doe","1980-01-01","john@doe.com","Missouri",42
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Error purifier","Doe","1980-01-01","john@doe.com","Missouri",42
""").getBytes());
}
return storageInput;
}
/***************************************************************************
**
***************************************************************************/
@ -331,4 +471,47 @@ class BulkInsertFullProcessTest extends BaseTest
)));
}
/***************************************************************************
**
***************************************************************************/
public static class PersonWarnOrErrorCustomizer implements TableCustomizerInterface
{
/***************************************************************************
**
***************************************************************************/
@Override
public AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview)
{
return AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS;
}
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput insertInput, List<QRecord> records, boolean isPreview) throws QException
{
for(QRecord record : records)
{
if(record.getValueString("firstName").toLowerCase().contains("warn"))
{
record.addWarning(new QWarningMessage(record.getValueString("firstName")));
}
else if(record.getValueString("firstName").toLowerCase().contains("error"))
{
if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error"))
{
continue;
}
record.addError(new BadInputStatusMessage(record.getValueString("firstName")));
}
}
return records;
}
}
}

58
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java
View File

@ -22,8 +22,15 @@
package com.kingsrook.qqq.backend.core.processes.utils;
import java.util.ArrayList;
import java.util.ConcurrentModificationException;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -32,6 +39,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Fail.fail;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@ -195,4 +203,54 @@ class RecordLookupHelperTest extends BaseTest
assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN));
}
/*******************************************************************************
** run a lot of threads (eg, 100), each trying to do lots of work in a
** shared recordLookupHelper. w/o the flag to use sync'ed collections, this
** (usually?) fails with a ConcurrentModificationException - but with the sync'ed
** collections, is safe.
*******************************************************************************/
@Test
void testConcurrentModification() throws InterruptedException, ExecutionException
{
ExecutorService executorService = Executors.newFixedThreadPool(100);
RecordLookupHelper recordLookupHelper = new RecordLookupHelper(true);
CapturedContext capture = QContext.capture();
List<Future<?>> futures = new ArrayList<>();
for(int i = 0; i < 100; i++)
{
int finalI = i;
Future<?> future = executorService.submit(() ->
{
QContext.init(capture);
for(int j = 0; j < 25000; j++)
{
try
{
recordLookupHelper.getRecordByKey(String.valueOf(j), "id", j);
}
catch(ConcurrentModificationException cme)
{
fail("CME!", cme);
}
catch(Exception e)
{
//////////////
// expected //
//////////////
}
}
});
futures.add(future);
}
for(Future<?> future : futures)
{
future.get();
}
}
}

40
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java
View File

@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.state;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
@ -103,17 +105,17 @@ public class InMemoryStateProviderTest extends BaseTest
/////////////////////////////////////////////////////////////
// Add an entry that is 3 hours old, should not be cleaned //
/////////////////////////////////////////////////////////////
UUIDAndTypeStateKey newKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
String newUUID = UUID.randomUUID().toString();
QRecord newQRecord = new QRecord().withValue("uuid", newUUID);
UUIDAndTypeStateKey newKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS));
String newUUID = UUID.randomUUID().toString();
QRecord newQRecord = new QRecord().withValue("uuid", newUUID);
stateProvider.put(newKey, newQRecord);
////////////////////////////////////////////////////////////
// Add an entry that is 5 hours old, it should be cleaned //
////////////////////////////////////////////////////////////
UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
String oldUUID = UUID.randomUUID().toString();
QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID);
UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
String oldUUID = UUID.randomUUID().toString();
QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID);
stateProvider.put(oldKey, oldQRecord);
///////////////////
@ -125,7 +127,33 @@ public class InMemoryStateProviderTest extends BaseTest
Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence");
Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty");
}
/*******************************************************************************
** originally written with N=100000, but showed the error as small as 1000.
*******************************************************************************/
@Test
void testDemonstrateConcurrentModificationIfNonSynchronizedMap()
{
int N = 1000;
InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance();
ExecutorService executorService = Executors.newFixedThreadPool(10);
executorService.submit(() ->
{
for(int i = 0; i < N; i++)
{
UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS));
stateProvider.put(oldKey, UUID.randomUUID());
}
});
for(int i = 0; i < N; i++)
{
stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS));
}
}
}

5
qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java
View File

@ -163,7 +163,6 @@ public class ProcessSpecUtilsV1
JSONObject outputJsonObject = convertResponseToJSONObject(response);
String json = outputJsonObject.toString(3);
System.out.println(json);
context.result(json);
}
@ -308,8 +307,8 @@ public class ProcessSpecUtilsV1
private static void archiveUploadedFile(String processName, QUploadedFile qUploadedFile)
{
String fileName = QValueFormatter.formatDate(LocalDate.now())
+ File.separator + processName
+ File.separator + qUploadedFile.getFilename();
+ File.separator + processName
+ File.separator + qUploadedFile.getFilename();
InsertInput insertInput = new InsertInput();
insertInput.setTableName(QJavalinImplementation.getJavalinMetaData().getUploadedFileArchiveTableName());