Merge branch 'dev' into feature/string-utils-safe-equals-ignore-case

This commit is contained in:
Tim Chamberlain
2025-05-12 15:47:09 -05:00
19 changed files with 572 additions and 55 deletions

View File

@ -35,6 +35,7 @@ import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver;
import com.openhtmltopdf.pdfboxout.PdfBoxRenderer;
import com.openhtmltopdf.pdfboxout.PdfRendererBuilder;
import org.jsoup.Jsoup;
import org.jsoup.helper.W3CDom;
import org.jsoup.nodes.Document;
@ -67,6 +68,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
//////////////////////////////////////////////////////////////////
Document document = Jsoup.parse(input.getHtml());
document.outputSettings().syntax(Document.OutputSettings.Syntax.xml);
org.w3c.dom.Document w3cDoc = new W3CDom().fromJsoup(document);
//////////////////////////////
// convert the XHTML to PDF //
@ -74,7 +76,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction<ConvertHtmlT
PdfRendererBuilder builder = new PdfRendererBuilder();
builder.toStream(input.getOutputStream());
builder.useFastMode();
builder.withHtmlContent(document.html(), input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
builder.withW3cDocument(w3cDoc, input.getBasePath() == null ? "./" : input.getBasePath().toUri().toString());
try(PdfBoxRenderer pdfBoxRenderer = builder.buildPdfRenderer())
{

View File

@ -0,0 +1,91 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.values;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
/*******************************************************************************
** Basic implementation of a possible value provider, for where there's a limited
** set of possible source objects - so you just have to define how to make one
** PV from a source object, how to list all of the source objects, and how to
** look up a PV from an id.
*******************************************************************************/
public abstract class BasicCustomPossibleValueProvider<S, ID extends Serializable> implements QCustomPossibleValueProvider<ID>
{
/***************************************************************************
**
***************************************************************************/
protected abstract QPossibleValue<ID> makePossibleValue(S sourceObject);
/***************************************************************************
**
***************************************************************************/
protected abstract S getSourceObject(Serializable id);
/***************************************************************************
**
***************************************************************************/
protected abstract List<S> getAllSourceObjects();
/***************************************************************************
**
***************************************************************************/
@Override
public QPossibleValue<ID> getPossibleValue(Serializable idValue)
{
S sourceObject = getSourceObject(idValue);
if(sourceObject == null)
{
return (null);
}
return makePossibleValue(sourceObject);
}
/***************************************************************************
**
***************************************************************************/
@Override
public List<QPossibleValue<ID>> search(SearchPossibleValueSourceInput input) throws QException
{
List<QPossibleValue<ID>> allPossibleValues = new ArrayList<>();
List<S> allSourceObjects = getAllSourceObjects();
for(S sourceObject : allSourceObjects)
{
allPossibleValues.add(makePossibleValue(sourceObject));
}
return completeCustomPVSSearch(input, allPossibleValues);
}
}

View File

@ -66,6 +66,7 @@ public enum WidgetType
// record view/edit widgets //
//////////////////////////////
CHILD_RECORD_LIST("childRecordList"),
CUSTOM_COMPONENT("customComponent"),
DYNAMIC_FORM("dynamicForm"),
DATA_BAG_VIEWER("dataBagViewer"),
PIVOT_TABLE_SETUP("pivotTableSetup"),

View File

@ -27,11 +27,9 @@ import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
** possible-value source provider for the `Tables` PVS - a list of all tables
** in an application/qInstance.
*******************************************************************************/
public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider<String>
public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider<QTableMetaData, String>
{
/***************************************************************************
**
***************************************************************************/
@Override
public QPossibleValue<String> getPossibleValue(Serializable idValue)
protected QPossibleValue<String> makePossibleValue(QTableMetaData sourceObject)
{
QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue));
if(table != null && !table.getIsHidden())
{
PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
if(PermissionCheckResult.ALLOW.equals(permissionCheckResult))
{
return (new QPossibleValue<>(table.getName(), table.getLabel()));
}
}
return null;
return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel()));
}
@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr
**
***************************************************************************/
@Override
public List<QPossibleValue<String>> search(SearchPossibleValueSourceInput input) throws QException
protected QTableMetaData getSourceObject(Serializable id)
{
/////////////////////////////////////////////////////////////////////////////////////
// build all of the possible values (note, will be filtered by user's permissions) //
/////////////////////////////////////////////////////////////////////////////////////
List<QPossibleValue<String>> allPossibleValues = new ArrayList<>();
QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id));
return isTableAllowed(table) ? table : null;
}
/***************************************************************************
**
***************************************************************************/
@Override
protected List<QTableMetaData> getAllSourceObjects()
{
ArrayList<QTableMetaData> rs = new ArrayList<>();
for(QTableMetaData table : QContext.getQInstance().getTables().values())
{
QPossibleValue<String> possibleValue = getPossibleValue(table.getName());
if(possibleValue != null)
if(isTableAllowed(table))
{
allPossibleValues.add(possibleValue);
rs.add(table);
}
}
return rs;
}
return completeCustomPVSSearch(input, allPossibleValues);
/***************************************************************************
**
***************************************************************************/
private boolean isTableAllowed(QTableMetaData table)
{
if(table == null)
{
return (false);
}
if(table.getIsHidden())
{
return (false);
}
PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table);
if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult))
{
return (false);
}
return (true);
}
}

View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider
{
QPossibleValueSource possibleValueSource = new QPossibleValueSource()
.withName(NAME)
.withIdType(QFieldType.STRING)
.withType(QPossibleValueSourceType.CUSTOM)
.withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class))
.withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);

View File

@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);

View File

@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm
QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName"));
/////////////////////////////////////////////////////////////////////////////////////////////
// the transform step builds summary lines that it predicts will insert successfully. //
// but those lines don't have ids, which we'd like to have (e.g., for a process trace that //
// might link to the built record). also, it's possible that there was a fail that only //
// happened in the actual insert, so, basically, re-do the summary here //
/////////////////////////////////////////////////////////////////////////////////////////////
BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep();
ProcessSummaryLine okSummary = transformStep.okSummary;
okSummary.setCount(0);
okSummary.setPrimaryKeys(new ArrayList<>());
//////////////////////////////////////////////////////////////////////////////////////////////////////////
// but - since errors from the transform step don't even make it through to us here in the load step, //
// do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: //
// clear out its warnings - we'll completely rebuild them here (with primary keys) //
// and add new error lines, e.g., in case of errors that only happened past the validation if possible. //
//////////////////////////////////////////////////////////////////////////////////////////////////////////
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup;
processSummaryWarningsAndErrorsRollup.resetWarnings();
List<QRecord> insertedRecords = runBackendStepOutput.getRecords();
for(QRecord insertedRecord : insertedRecords)
{
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null)
{
/////////////////////////////////////////////////////////////////////////
// if the record had no errors, and we have a primary key for it, then //
// keep track of the range of primary keys (first and last) //
/////////////////////////////////////////////////////////////////////////
if(firstInsertedPrimaryKey == null)
{
firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
firstInsertedPrimaryKey = primaryKey;
}
lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField());
lastInsertedPrimaryKey = primaryKey;
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings()))
{
/////////////////////////////////////////////////////////////////////////////
// if there were warnings on the inserted record, put it in a warning line //
/////////////////////////////////////////////////////////////////////////////
String message = insertedRecord.getWarnings().get(0).getMessage();
processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey);
}
else
{
////////////////////////////////////////////////////////////////////////
// if no warnings for the inserted record, then put it in the OK line //
////////////////////////////////////////////////////////////////////////
okSummary.incrementCountAndAddPrimaryKey(primaryKey);
}
}
else
{
//////////////////////////////////////////////////////////////////////
// else if there were errors or no primary key, build an error line //
//////////////////////////////////////////////////////////////////////
String message = "Failed to insert";
if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()))
{
//////////////////////////////////////////////////////////
// use the error message from the record if we have one //
//////////////////////////////////////////////////////////
message = insertedRecord.getErrors().get(0).getMessage();
}
processSummaryWarningsAndErrorsRollup.addError(message, primaryKey);
}
}
okSummary.pickMessage(true);
}

View File

@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
////////////////////////////////////////////////////////////////////////////////////////
// for headless-bulk load (e.g., sftp import), set up the process tracer's key record //
////////////////////////////////////////////////////////////////////////////////////////
runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput));
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) //
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

View File

@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import org.apache.commons.lang3.BooleanUtils;
@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
//////////////////////////////////////////////////////////////////////////////
if(savedBulkLoadProfileRecord == null)
{
throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load"));
throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load"));
}
SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord);
if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson()))
{
throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load"));
}
try
{
@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
}
catch(Exception e)
{
throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e));
throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e));
}
}
else
@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep
}
}
}
catch(QUserFacingException ufe)
{
LOG.warn("User-facing error in bulk insert receive mapping", ufe);
throw ufe;
}
catch(Exception e)
{
LOG.warn("Error in bulk insert receive mapping", e);

View File

@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class BulkInsertTransformStep extends AbstractTransformStep
{
private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK);
private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted")
.withDoReplaceSingletonCountLinesWithSuffixOnly(false);
private ListingHash<String, RowValue> errorToExampleRowValueMap = new ListingHash<>();

View File

@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup
{
if(otherWarningsSummary == null)
{
otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
otherWarningsSummary = buildOtherWarningsSummary();
}
processSummaryLine = otherWarningsSummary;
}
@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup
/***************************************************************************
**
***************************************************************************/
private static ProcessSummaryLine buildOtherWarningsSummary()
{
return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning.");
}
/***************************************************************************
**
***************************************************************************/
public void resetWarnings()
{
warningSummaries.clear();
otherWarningsSummary = buildOtherWarningsSummary();
}
/*******************************************************************************
** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that
** extracts string messages out.

View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.utils;
import java.io.Serializable;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -51,13 +52,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
*******************************************************************************/
public class RecordLookupHelper
{
private Map<String, Map<Serializable, QRecord>> recordMaps = new HashMap<>();
private Map<String, Map<Serializable, QRecord>> recordMaps;
private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps = new HashMap<>();
private Map<String, Map<Map<String, Serializable>, QRecord>> uniqueKeyMaps;
private Set<String> preloadedKeys = new HashSet<>();
private Set<String> preloadedKeys;
private Set<Pair<String, String>> disallowedOneOffLookups = new HashSet<>();
private Set<Pair<String, String>> disallowedOneOffLookups;
private boolean useSynchronizedCollections;
@ -67,6 +69,33 @@ public class RecordLookupHelper
*******************************************************************************/
public RecordLookupHelper()
{
this(false);
}
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public RecordLookupHelper(boolean useSynchronizedCollections)
{
this.useSynchronizedCollections = useSynchronizedCollections;
if(useSynchronizedCollections)
{
recordMaps = Collections.synchronizedMap(new HashMap<>());
uniqueKeyMaps = Collections.synchronizedMap(new HashMap<>());
preloadedKeys = Collections.synchronizedSet(new HashSet<>());
disallowedOneOffLookups = Collections.synchronizedSet(new HashSet<>());
}
else
{
recordMaps = new HashMap<>();
uniqueKeyMaps = new HashMap<>();
preloadedKeys = new HashSet<>();
disallowedOneOffLookups = new HashSet<>();
}
}
@ -77,7 +106,7 @@ public class RecordLookupHelper
public QRecord getRecordByUniqueKey(String tableName, Map<String, Serializable> uniqueKey) throws QException
{
String mapKey = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(","));
Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
Map<Map<String, Serializable>, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
if(!recordMap.containsKey(uniqueKey))
{
@ -96,7 +125,7 @@ public class RecordLookupHelper
public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>());
Map<Serializable, QRecord> recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
////////////////////////////////////////////////////////////
// make sure we have they key object in the expected type //
@ -150,7 +179,7 @@ public class RecordLookupHelper
public void preloadRecords(String tableName, String keyFieldName, QQueryFilter filter) throws QException
{
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));
}
@ -170,7 +199,7 @@ public class RecordLookupHelper
}
String mapKey = tableName + "." + keyFieldName;
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>());
Map<Serializable, QRecord> tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>());
QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList));
tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter));

View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state;
import java.io.Serializable;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@ -58,7 +59,7 @@ public class InMemoryStateProvider implements StateProviderInterface
*******************************************************************************/
private InMemoryStateProvider()
{
this.map = new HashMap<>();
this.map = Collections.synchronizedMap(new HashMap<>());
///////////////////////////////////////////////////////////
// Start a single thread executor to handle the cleaning //