From 78892b36421be22205e09447f395c9203ca4d164 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 18 Apr 2025 13:56:26 -0500 Subject: [PATCH 1/9] Fix to allow html entities by going through a w3c DOM --- .../core/actions/templates/ConvertHtmlToPdfAction.java | 4 +++- .../core/actions/templates/ConvertHtmlToPdfActionTest.java | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java index d2dbdac1..61d97489 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/templates/ConvertHtmlToPdfAction.java @@ -35,6 +35,7 @@ import com.openhtmltopdf.pdfboxout.PdfBoxFontResolver; import com.openhtmltopdf.pdfboxout.PdfBoxRenderer; import com.openhtmltopdf.pdfboxout.PdfRendererBuilder; import org.jsoup.Jsoup; +import org.jsoup.helper.W3CDom; import org.jsoup.nodes.Document; @@ -67,6 +68,7 @@ public class ConvertHtmlToPdfAction extends AbstractQActionFunction

This is a test of converting HTML to PDF!!

+

This is   a line with • some entities <

(btw, is this in SF-Pro???)

From 1b9d93e92438a10a00965ed2d0576ad4ab990f2b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 18 Apr 2025 13:57:01 -0500 Subject: [PATCH 2/9] Add CUSTOM_COMPONENT widget type --- .../qqq/backend/core/model/dashboard/widgets/WidgetType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java index 9989f670..3a773a84 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/dashboard/widgets/WidgetType.java @@ -66,6 +66,7 @@ public enum WidgetType // record view/edit widgets // ////////////////////////////// CHILD_RECORD_LIST("childRecordList"), + CUSTOM_COMPONENT("customComponent"), DYNAMIC_FORM("dynamicForm"), DATA_BAG_VIEWER("dataBagViewer"), PIVOT_TABLE_SETUP("pivotTableSetup"), From 97434ebb666f563cdfb443ced4dbb8fd50f80f9f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 18 Apr 2025 13:57:43 -0500 Subject: [PATCH 3/9] Initial checkin of BasicCustomPossibleValueProvider, and migrate TablesCustomPossibleValueProvider to use it. --- .../BasicCustomPossibleValueProvider.java | 91 +++++++++++++++++++ .../TablesCustomPossibleValueProvider.java | 70 +++++++++----- ...esPossibleValueSourceMetaDataProvider.java | 2 + 3 files changed, 138 insertions(+), 25 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java new file mode 100644 index 00000000..da1421bc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/BasicCustomPossibleValueProvider.java @@ -0,0 +1,91 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.values; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; + + +/******************************************************************************* + ** Basic implementation of a possible value provider, for where there's a limited + ** set of possible source objects - so you just have to define how to make one + ** PV from a source object, how to list all of the source objects, and how to + ** look up a PV from an id. + *******************************************************************************/ +public abstract class BasicCustomPossibleValueProvider implements QCustomPossibleValueProvider +{ + + /*************************************************************************** + ** + ***************************************************************************/ + protected abstract QPossibleValue makePossibleValue(S sourceObject); + + /*************************************************************************** + ** + ***************************************************************************/ + protected abstract S getSourceObject(Serializable id); + + /*************************************************************************** + ** + ***************************************************************************/ + protected abstract List getAllSourceObjects(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + S sourceObject = getSourceObject(idValue); + if(sourceObject == null) + { + return (null); + } + + return makePossibleValue(sourceObject); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + List> allPossibleValues = new ArrayList<>(); + List allSourceObjects = getAllSourceObjects(); + for(S sourceObject : allSourceObjects) + { + allPossibleValues.add(makePossibleValue(sourceObject)); + } + + return completeCustomPVSSearch(input, allPossibleValues); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java index e72d188f..e2d98b4a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -27,11 +27,9 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; -import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.actions.values.BasicCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; -import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -40,26 +38,16 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; ** possible-value source provider for the `Tables` PVS - a list of all tables ** in an application/qInstance. *******************************************************************************/ -public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider +public class TablesCustomPossibleValueProvider extends BasicCustomPossibleValueProvider { /*************************************************************************** ** ***************************************************************************/ @Override - public QPossibleValue getPossibleValue(Serializable idValue) + protected QPossibleValue makePossibleValue(QTableMetaData sourceObject) { - QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue)); - if(table != null && !table.getIsHidden()) - { - PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table); - if(PermissionCheckResult.ALLOW.equals(permissionCheckResult)) - { - return (new QPossibleValue<>(table.getName(), table.getLabel())); - } - } - - return null; + return (new QPossibleValue<>(sourceObject.getName(), sourceObject.getLabel())); } @@ -68,22 +56,54 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr ** ***************************************************************************/ @Override - public List> search(SearchPossibleValueSourceInput input) throws QException + protected QTableMetaData getSourceObject(Serializable id) { - ///////////////////////////////////////////////////////////////////////////////////// - // build all of the possible values (note, will be filtered by user's permissions) // - ///////////////////////////////////////////////////////////////////////////////////// - List> allPossibleValues = new ArrayList<>(); + QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(id)); + return isTableAllowed(table) ? table : null; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + protected List getAllSourceObjects() + { + ArrayList rs = new ArrayList<>(); for(QTableMetaData table : QContext.getQInstance().getTables().values()) { - QPossibleValue possibleValue = getPossibleValue(table.getName()); - if(possibleValue != null) + if(isTableAllowed(table)) { - allPossibleValues.add(possibleValue); + rs.add(table); } } + return rs; + } - return completeCustomPVSSearch(input, allPossibleValues); + + /*************************************************************************** + ** + ***************************************************************************/ + private boolean isTableAllowed(QTableMetaData table) + { + if(table == null) + { + return (false); + } + + if(table.getIsHidden()) + { + return (false); + } + + PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table); + if(!PermissionCheckResult.ALLOW.equals(permissionCheckResult)) + { + return (false); + } + + return (true); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java index 75087ae7..cdc9b6d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; @@ -45,6 +46,7 @@ public class TablesPossibleValueSourceMetaDataProvider { QPossibleValueSource possibleValueSource = new QPossibleValueSource() .withName(NAME) + .withIdType(QFieldType.STRING) .withType(QPossibleValueSourceType.CUSTOM) .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class)) .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); From f81b257dd44f1366b3565f4e73c62595212ad17b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 21 Apr 2025 10:58:56 -0500 Subject: [PATCH 4/9] Improving process traces built by bulk load --- .../bulk/insert/BulkInsertExtractStep.java | 2 -- .../insert/BulkInsertPrepareFileUploadStep.java | 5 +++++ .../insert/BulkInsertReceiveFileMappingStep.java | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index dabfb3f4..694ae332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,8 +55,6 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); - int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index ea4810a9..4b15b720 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -52,6 +52,11 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + //////////////////////////////////////////////////////////////////////////////////////// + // for headless-bulk load (e.g., sftp import), set up the process tracer's key record // + //////////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 94032fe5..aa460540 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -46,6 +46,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -77,10 +78,14 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ////////////////////////////////////////////////////////////////////////////// if(savedBulkLoadProfileRecord == null) { - throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + throw (new QUserFacingException("Did not receive a Bulk Load Profile record as input. Unable to perform headless bulk load")); } SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + if(!StringUtils.hasContent(savedBulkLoadProfile.getMappingJson())) + { + throw (new QUserFacingException("Bulk Load Profile record's Mapping is empty. Unable to perform headless bulk load")); + } try { @@ -88,7 +93,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } catch(Exception e) { - throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + throw (new QUserFacingException("Error processing Bulk Load Profile record. Unable to perform headless bulk load", e)); } } else @@ -240,6 +245,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep } } } + catch(QUserFacingException ufe) + { + LOG.warn("User-facing error in bulk insert receive mapping", ufe); + throw ufe; + } catch(Exception e) { LOG.warn("Error in bulk insert receive mapping", e); From e5987238e6c804f61f84a15538a48695d4cc1160 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 25 Apr 2025 16:05:54 -0500 Subject: [PATCH 5/9] Add primary keys to process summary lines (and thus traces) for bulk load; better handling of errors and warnings also from bulk insert result step --- .../bulk/insert/BulkInsertLoadStep.java | 65 +++++- .../bulk/insert/BulkInsertTransformStep.java | 4 +- ...ProcessSummaryWarningsAndErrorsRollup.java | 23 ++- .../ProcessSummaryLineInterfaceAssert.java | 9 + .../insert/BulkInsertFullProcessTest.java | 185 +++++++++++++++++- 5 files changed, 279 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index 30f98c91..c60503fe 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -77,19 +78,77 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + ///////////////////////////////////////////////////////////////////////////////////////////// + // the transform step builds summary lines that it predicts will insert successfully. // + // but those lines don't have ids, which we'd like to have (e.g., for a process trace that // + // might link to the built record). also, it's possible that there was a fail that only // + // happened in the actual insert, so, basically, re-do the summary here // + ///////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertTransformStep transformStep = (BulkInsertTransformStep) getTransformStep(); + ProcessSummaryLine okSummary = transformStep.okSummary; + okSummary.setCount(0); + okSummary.setPrimaryKeys(new ArrayList<>()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // but - since errors from the transform step don't even make it through to us here in the load step, // + // do re-use the ProcessSummaryWarningsAndErrorsRollup from transform step as follows: // + // clear out its warnings - we'll completely rebuild them here (with primary keys) // + // and add new error lines, e.g., in case of errors that only happened past the validation if possible. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = transformStep.processSummaryWarningsAndErrorsRollup; + processSummaryWarningsAndErrorsRollup.resetWarnings(); + List insertedRecords = runBackendStepOutput.getRecords(); for(QRecord insertedRecord : insertedRecords) { - if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + Serializable primaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors()) && primaryKey != null) { + ///////////////////////////////////////////////////////////////////////// + // if the record had no errors, and we have a primary key for it, then // + // keep track of the range of primary keys (first and last) // + ///////////////////////////////////////////////////////////////////////// if(firstInsertedPrimaryKey == null) { - firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + firstInsertedPrimaryKey = primaryKey; } - lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + lastInsertedPrimaryKey = primaryKey; + + if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getWarnings())) + { + ///////////////////////////////////////////////////////////////////////////// + // if there were warnings on the inserted record, put it in a warning line // + ///////////////////////////////////////////////////////////////////////////// + String message = insertedRecord.getWarnings().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addWarning(message, primaryKey); + } + else + { + //////////////////////////////////////////////////////////////////////// + // if no warnings for the inserted record, then put it in the OK line // + //////////////////////////////////////////////////////////////////////// + okSummary.incrementCountAndAddPrimaryKey(primaryKey); + } + } + else + { + ////////////////////////////////////////////////////////////////////// + // else if there were errors or no primary key, build an error line // + ////////////////////////////////////////////////////////////////////// + String message = "Failed to insert"; + if(!CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + ////////////////////////////////////////////////////////// + // use the error message from the record if we have one // + ////////////////////////////////////////////////////////// + message = insertedRecord.getErrors().get(0).getMessage(); + } + processSummaryWarningsAndErrorsRollup.addError(message, primaryKey); } } + + okSummary.pickMessage(true); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 410fccf2..e3979a07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -75,9 +75,9 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class BulkInsertTransformStep extends AbstractTransformStep { - private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); + ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") .withDoReplaceSingletonCountLinesWithSuffixOnly(false); private ListingHash errorToExampleRowValueMap = new ListingHash<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 6296f31c..79dbe9ca 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -195,7 +195,7 @@ public class ProcessSummaryWarningsAndErrorsRollup { if(otherWarningsSummary == null) { - otherWarningsSummary = new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning."); + otherWarningsSummary = buildOtherWarningsSummary(); } processSummaryLine = otherWarningsSummary; } @@ -214,6 +214,27 @@ public class ProcessSummaryWarningsAndErrorsRollup + /*************************************************************************** + ** + ***************************************************************************/ + private static ProcessSummaryLine buildOtherWarningsSummary() + { + return new ProcessSummaryLine(Status.WARNING).withMessageSuffix("records had an other warning."); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void resetWarnings() + { + warningSummaries.clear(); + otherWarningsSummary = buildOtherWarningsSummary(); + } + + + /******************************************************************************* ** Wrapper around AlphaNumericComparator for ProcessSummaryLineInterface that ** extracts string messages out. diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java index 60d4561c..59b79181 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -185,4 +185,13 @@ public class ProcessSummaryLineInterfaceAssert extends AbstractAssert preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + for(QRecord record : records) + { + if(record.getValueString("firstName").toLowerCase().contains("warn")) + { + record.addWarning(new QWarningMessage(record.getValueString("firstName"))); + } + else if(record.getValueString("firstName").toLowerCase().contains("error")) + { + if(isPreview && record.getValueString("firstName").toLowerCase().contains("not-pre-error")) + { + continue; + } + + record.addError(new BadInputStatusMessage(record.getValueString("firstName"))); + } + } + return records; + } + } } \ No newline at end of file From be16d5f0cf69545ee7bf2e9671a672163aa02612 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 25 Apr 2025 16:13:17 -0500 Subject: [PATCH 6/9] Checkstyle! --- .../implementations/bulk/insert/BulkInsertFullProcessTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index edcc41e8..cb11631b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -285,7 +285,7 @@ class BulkInsertFullProcessTest extends BaseTest continueProcessPostValueMapping(runProcessInput); runProcessOutput = continueProcessPostReviewScreen(runProcessInput); - ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was inserted.").hasStatus(Status.OK).hasCount(1); + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Person Memory record was inserted.").hasStatus(Status.OK).hasCount(1); ProcessSummaryAssert.assertThat(runProcessOutput) .hasLineWithMessageContaining("plane") From e58190f15dbeccdf6758bce6198c64b5763b445d Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Tue, 29 Apr 2025 15:42:24 -0500 Subject: [PATCH 7/9] removed unnecessary sop --- .../javalin/specs/v1/utils/ProcessSpecUtilsV1.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java index e339d957..7387e0ed 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/utils/ProcessSpecUtilsV1.java @@ -163,7 +163,6 @@ public class ProcessSpecUtilsV1 JSONObject outputJsonObject = convertResponseToJSONObject(response); String json = outputJsonObject.toString(3); - System.out.println(json); context.result(json); } @@ -308,8 +307,8 @@ public class ProcessSpecUtilsV1 private static void archiveUploadedFile(String processName, QUploadedFile qUploadedFile) { String fileName = QValueFormatter.formatDate(LocalDate.now()) - + File.separator + processName - + File.separator + qUploadedFile.getFilename(); + + File.separator + processName + + File.separator + qUploadedFile.getFilename(); InsertInput insertInput = new InsertInput(); insertInput.setTableName(QJavalinImplementation.getJavalinMetaData().getUploadedFileArchiveTableName()); From 625ed5209cb3e7633e56327a1b0a618b03e6d0e0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 5 May 2025 10:59:12 -0500 Subject: [PATCH 8/9] switch InMemoryStateProvider to use synchronizedMap, to avoid ConcurrentModificationException in clean method --- .../core/state/InMemoryStateProvider.java | 3 +- .../core/state/InMemoryStateProviderTest.java | 40 ++++++++++++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java index 84ddb346..91d615eb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProvider.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.state; import java.io.Serializable; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -58,7 +59,7 @@ public class InMemoryStateProvider implements StateProviderInterface *******************************************************************************/ private InMemoryStateProvider() { - this.map = new HashMap<>(); + this.map = Collections.synchronizedMap(new HashMap<>()); /////////////////////////////////////////////////////////// // Start a single thread executor to handle the cleaning // diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java index b8492fe8..fb87febf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/state/InMemoryStateProviderTest.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.core.state; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -103,17 +105,17 @@ public class InMemoryStateProviderTest extends BaseTest ///////////////////////////////////////////////////////////// // Add an entry that is 3 hours old, should not be cleaned // ///////////////////////////////////////////////////////////// - UUIDAndTypeStateKey newKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS)); - String newUUID = UUID.randomUUID().toString(); - QRecord newQRecord = new QRecord().withValue("uuid", newUUID); + UUIDAndTypeStateKey newKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(3, ChronoUnit.HOURS)); + String newUUID = UUID.randomUUID().toString(); + QRecord newQRecord = new QRecord().withValue("uuid", newUUID); stateProvider.put(newKey, newQRecord); //////////////////////////////////////////////////////////// // Add an entry that is 5 hours old, it should be cleaned // //////////////////////////////////////////////////////////// - UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS)); - String oldUUID = UUID.randomUUID().toString(); - QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID); + UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS)); + String oldUUID = UUID.randomUUID().toString(); + QRecord oldQRecord = new QRecord().withValue("uuid", oldUUID); stateProvider.put(oldKey, oldQRecord); /////////////////// @@ -125,7 +127,33 @@ public class InMemoryStateProviderTest extends BaseTest Assertions.assertEquals(newUUID, qRecordFromState.getValueString("uuid"), "Should read value from state persistence"); Assertions.assertTrue(stateProvider.get(QRecord.class, oldKey).isEmpty(), "Key not found in state should return empty"); + } + + + /******************************************************************************* + ** originally written with N=100000, but showed the error as small as 1000. + *******************************************************************************/ + @Test + void testDemonstrateConcurrentModificationIfNonSynchronizedMap() + { + int N = 1000; + InMemoryStateProvider stateProvider = InMemoryStateProvider.getInstance(); + + ExecutorService executorService = Executors.newFixedThreadPool(10); + executorService.submit(() -> + { + for(int i = 0; i < N; i++) + { + UUIDAndTypeStateKey oldKey = new UUIDAndTypeStateKey(UUID.randomUUID(), StateType.PROCESS_STATUS, Instant.now().minus(5, ChronoUnit.HOURS)); + stateProvider.put(oldKey, UUID.randomUUID()); + } + }); + + for(int i = 0; i < N; i++) + { + stateProvider.clean(Instant.now().minus(4, ChronoUnit.HOURS)); + } } } \ No newline at end of file From ce2ca3f413b0422c102305e8268a4435644e1e65 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 5 May 2025 14:11:04 -0500 Subject: [PATCH 9/9] Option to useSynchronizedCollections in RecordLookupHelper --- .../processes/utils/RecordLookupHelper.java | 45 +++++++++++--- .../utils/RecordLookupHelperTest.java | 58 +++++++++++++++++++ 2 files changed, 95 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java index 94ca1986..899fd468 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelper.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.utils; import java.io.Serializable; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -51,13 +52,14 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public class RecordLookupHelper { - private Map> recordMaps = new HashMap<>(); + private Map> recordMaps; - private Map, QRecord>> uniqueKeyMaps = new HashMap<>(); + private Map, QRecord>> uniqueKeyMaps; - private Set preloadedKeys = new HashSet<>(); + private Set preloadedKeys; - private Set> disallowedOneOffLookups = new HashSet<>(); + private Set> disallowedOneOffLookups; + private boolean useSynchronizedCollections; @@ -67,6 +69,33 @@ public class RecordLookupHelper *******************************************************************************/ public RecordLookupHelper() { + this(false); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RecordLookupHelper(boolean useSynchronizedCollections) + { + this.useSynchronizedCollections = useSynchronizedCollections; + + if(useSynchronizedCollections) + { + recordMaps = Collections.synchronizedMap(new HashMap<>()); + uniqueKeyMaps = Collections.synchronizedMap(new HashMap<>()); + preloadedKeys = Collections.synchronizedSet(new HashSet<>()); + disallowedOneOffLookups = Collections.synchronizedSet(new HashSet<>()); + } + else + { + recordMaps = new HashMap<>(); + uniqueKeyMaps = new HashMap<>(); + preloadedKeys = new HashSet<>(); + disallowedOneOffLookups = new HashSet<>(); + } } @@ -77,7 +106,7 @@ public class RecordLookupHelper public QRecord getRecordByUniqueKey(String tableName, Map uniqueKey) throws QException { String mapKey = tableName + "." + uniqueKey.keySet().stream().sorted().collect(Collectors.joining(",")); - Map, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>()); + Map, QRecord> recordMap = uniqueKeyMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>()); if(!recordMap.containsKey(uniqueKey)) { @@ -96,7 +125,7 @@ public class RecordLookupHelper public QRecord getRecordByKey(String tableName, String keyFieldName, Serializable key) throws QException { String mapKey = tableName + "." + keyFieldName; - Map recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> new HashMap<>()); + Map recordMap = recordMaps.computeIfAbsent(mapKey, (k) -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>()); //////////////////////////////////////////////////////////// // make sure we have they key object in the expected type // @@ -150,7 +179,7 @@ public class RecordLookupHelper public void preloadRecords(String tableName, String keyFieldName, QQueryFilter filter) throws QException { String mapKey = tableName + "." + keyFieldName; - Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>()); + Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>()); tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter)); } @@ -170,7 +199,7 @@ public class RecordLookupHelper } String mapKey = tableName + "." + keyFieldName; - Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> new HashMap<>()); + Map tableMap = recordMaps.computeIfAbsent(mapKey, s -> useSynchronizedCollections ? Collections.synchronizedMap(new HashMap<>()) : new HashMap<>()); QQueryFilter filter = new QQueryFilter(new QFilterCriteria(keyFieldName, QCriteriaOperator.IN, inList)); tableMap.putAll(GeneralProcessUtils.loadTableToMap(tableName, keyFieldName, filter)); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java index 4e2f3b30..35e8cc21 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/utils/RecordLookupHelperTest.java @@ -22,8 +22,15 @@ package com.kingsrook.qqq.backend.core.processes.utils; +import java.util.ArrayList; +import java.util.ConcurrentModificationException; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.CapturedContext; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -32,6 +39,7 @@ import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Fail.fail; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -195,4 +203,54 @@ class RecordLookupHelperTest extends BaseTest assertEquals(1, MemoryRecordStore.getStatistics().get(MemoryRecordStore.STAT_QUERIES_RAN)); } + + + /******************************************************************************* + ** run a lot of threads (eg, 100), each trying to do lots of work in a + ** shared recordLookupHelper. w/o the flag to use sync'ed collections, this + ** (usually?) fails with a ConcurrentModificationException - but with the sync'ed + ** collections, is safe. + *******************************************************************************/ + @Test + void testConcurrentModification() throws InterruptedException, ExecutionException + { + ExecutorService executorService = Executors.newFixedThreadPool(100); + RecordLookupHelper recordLookupHelper = new RecordLookupHelper(true); + + CapturedContext capture = QContext.capture(); + + List> futures = new ArrayList<>(); + for(int i = 0; i < 100; i++) + { + int finalI = i; + Future future = executorService.submit(() -> + { + QContext.init(capture); + for(int j = 0; j < 25000; j++) + { + try + { + recordLookupHelper.getRecordByKey(String.valueOf(j), "id", j); + } + catch(ConcurrentModificationException cme) + { + fail("CME!", cme); + } + catch(Exception e) + { + ////////////// + // expected // + ////////////// + } + } + }); + futures.add(future); + } + + for(Future future : futures) + { + future.get(); + } + } + } \ No newline at end of file