From 73200b2fd2c5e27fe7e209a16368c40e784c1a23 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:13:12 -0600 Subject: [PATCH 001/209] CE-1955 Mark as Serializable --- .../core/model/actions/tables/storage/StorageInput.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java index 5407faeb..19d84ce3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java @@ -22,13 +22,14 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.storage; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; /******************************************************************************* ** Input for Storage actions. *******************************************************************************/ -public class StorageInput extends AbstractTableActionInput +public class StorageInput extends AbstractTableActionInput implements Serializable { private String reference; From 7d058530d53fdb382565e409fde7e5f208712d90 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:13:34 -0600 Subject: [PATCH 002/209] CE-1955 Initial checkin --- .../SavedBulkLoadProfile.java | 285 ++++++++++++++++++ .../SavedBulkLoadProfileMetaDataProvider.java | 159 ++++++++++ .../SharedSavedBulkLoadProfile.java | 268 ++++++++++++++++ 3 files changed, 712 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java new file mode 100644 index 00000000..65f07c77 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; + + +/******************************************************************************* + ** Entity bean for the savedBulkLoadProfile table + *******************************************************************************/ +public class SavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "savedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Profile Name") + private String label; + + @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true) + private String tableName; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner") + private String userId; + + @QField(label = "Mapping JSON") + private String mappingJson; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public SavedBulkLoadProfile withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public SavedBulkLoadProfile withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + ** + *******************************************************************************/ + public String getUserId() + { + return userId; + } + + + + /******************************************************************************* + ** Setter for userId + ** + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + ** + *******************************************************************************/ + public SavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + + /******************************************************************************* + ** Getter for mappingJson + *******************************************************************************/ + public String getMappingJson() + { + return (this.mappingJson); + } + + + + /******************************************************************************* + ** Setter for mappingJson + *******************************************************************************/ + public void setMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + } + + + + /******************************************************************************* + ** Fluent setter for mappingJson + *******************************************************************************/ + public SavedBulkLoadProfile withMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java new file mode 100644 index 00000000..d8d376e1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -0,0 +1,159 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileMetaDataProvider +{ + public static final String SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE = "sharedSavedBulkLoadProfileJoinSavedBulkLoadProfile"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String recordTablesBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(SavedBulkLoadProfile.TABLE_NAME)); + + ///////////////////////////////////// + // todo - param to enable sharing? // + ///////////////////////////////////// + instance.addTable(defineSharedSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addJoin(defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile()); + if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null) + { + instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile() + { + return (new QJoinMetaData() + .withName(SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE) + .withLeftTable(SharedSavedBulkLoadProfile.TABLE_NAME) + .withRightTable(SavedBulkLoadProfile.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("savedBulkLoadProfileId", "id"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SavedBulkLoadProfile.TABLE_NAME) + .withLabel("Bulk Load Profile") + .withIcon(new QIcon().withName("drive_folder_upload")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SavedBulkLoadProfile.class) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("mappingJson")).withIsHidden(true)) + .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId")).withIsHidden(true)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + // todo - want one of these? + // table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + + table.withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withAssetIdFieldName("savedBulkLoadProfileId") + .withScopeFieldName("scope") + .withThisTableOwnerIdFieldName("userId") + .withAudienceType(new ShareableAudienceType().withName("user").withFieldName("userId"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSharedSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withLabel("Shared Bulk Load Profile") + .withIcon(new QIcon().withName("share")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("savedBulkLoadProfileId") + .withBackendName(backendName) + .withUniqueKey(new UniqueKey("savedBulkLoadProfileId", "userId")) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SharedSavedBulkLoadProfile.class) + // todo - security key + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedBulkLoadProfileId", "userId"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java new file mode 100644 index 00000000..a02e5c3b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java @@ -0,0 +1,268 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; + + +/******************************************************************************* + ** Entity bean for the shared saved bulk load profile table + *******************************************************************************/ +public class SharedSavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "sharedSavedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = SavedBulkLoadProfile.TABLE_NAME, label = "Bulk Load Profile") + private Integer savedBulkLoadProfileId; + + @QField(label = "User") + private String userId; + + @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME) + private String scope; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public SharedSavedBulkLoadProfile withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public String getScope() + { + return (this.scope); + } + + + + /******************************************************************************* + ** Setter for scope + *******************************************************************************/ + public void setScope(String scope) + { + this.scope = scope; + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public SharedSavedBulkLoadProfile withScope(String scope) + { + this.scope = scope; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedBulkLoadProfileId + *******************************************************************************/ + public Integer getSavedBulkLoadProfileId() + { + return (this.savedBulkLoadProfileId); + } + + + + /******************************************************************************* + ** Setter for savedBulkLoadProfileId + *******************************************************************************/ + public void setSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + } + + + + /******************************************************************************* + ** Fluent setter for savedBulkLoadProfileId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + return (this); + } + + +} From 7ba205a26267a81e6e48d9604b7e6c54ddf8e53b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:16:59 -0600 Subject: [PATCH 003/209] CE-1955 Initial checkin --- .../insert/BulkInsertReceiveFileStep.java | 81 +++ .../insert/BulkInsertReceiveMappingStep.java | 59 ++ .../bulk/insert/BulkInsertStepUtils.java | 58 ++ .../bulk/insert/BulkInsertV2ExtractStep.java | 126 +++++ .../bulk/insert/BulkLoadFileRow.java | 167 ++++++ .../AbstractIteratorBasedFileToRows.java | 125 +++++ .../insert/filehandling/CsvFileToRows.java | 112 ++++ .../filehandling/FileToRowsInterface.java | 74 +++ .../insert/filehandling/XlsxFileToRows.java | 102 ++++ .../insert/mapping/BulkInsertMapping.java | 503 ++++++++++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 75 +++ .../insert/mapping/RowsToRecordInterface.java | 69 +++ .../bulk/insert/mapping/TallRowsToRecord.java | 316 +++++++++++ .../bulk/insert/mapping/ValueMapper.java | 79 +++ .../bulk/insert/mapping/WideRowsToRecord.java | 333 ++++++++++++ .../filehandling/CsvFileToRowsTest.java | 60 +++ .../insert/filehandling/TestFileToRows.java | 86 +++ .../filehandling/XlsxFileToRowsTest.java | 106 ++++ .../insert/mapping/FlatRowsToRecordTest.java | 150 ++++++ .../insert/mapping/TallRowsToRecordTest.java | 284 ++++++++++ .../bulk/insert/mapping/ValueMapperTest.java | 134 +++++ .../insert/mapping/WideRowsToRecordTest.java | 305 +++++++++++ 22 files changed, 3404 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java new file mode 100644 index 00000000..6ebcf3a8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + + List bodyRows = new ArrayList<>(); + while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) + { + bodyRows.add(fileToRowsInterface.next().toString()); + } + + runBackendStepOutput.addValue("header", headerRow.toString()); + runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); + System.out.println("Done receiving file"); + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java new file mode 100644 index 00000000..0071aa49 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(true); + bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name" + )); + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + // probably need to what, receive the mapping object, store it into state + // what, do we maybe return to a different sub-mapping screen (e.g., values) + // then at some point - cool - proceed to ETL's steps + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java new file mode 100644 index 00000000..e3dac892 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertStepUtils +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException + { + @SuppressWarnings("unchecked") + ArrayList storageInputs = (ArrayList) input.getValue("theFile"); + if(storageInputs == null) + { + throw (new QException("StorageInputs for theFile were not found in process state")); + } + + if(storageInputs.isEmpty()) + { + throw (new QException("StorageInputs for theFile was an empty list")); + } + + StorageInput storageInput = storageInputs.get(0); + return (storageInput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java new file mode 100644 index 00000000..2e49e48a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; + + +/******************************************************************************* + ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. + *******************************************************************************/ +public class BulkInsertV2ExtractStep extends AbstractExtractStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); + + if(page.size() > remainingLimit) + { + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java new file mode 100644 index 00000000..487f606a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + + +/******************************************************************************* + ** A row of values, e.g., from a file, for bulk-load + *******************************************************************************/ +public class BulkLoadFileRow implements Serializable +{ + private Serializable[] values; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadFileRow(Serializable[] values) + { + this.values = values; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public int size() + { + if(values == null) + { + return (0); + } + + return (values.length); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public boolean hasIndex(int i) + { + if(values == null) + { + return (false); + } + + if(i >= values.length || i < 0) + { + return (false); + } + + return (true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValue(int i) + { + if(values == null) + { + throw new IllegalStateException("Row has no values"); + } + + if(i >= values.length || i < 0) + { + throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValueElseNull(int i) + { + if(!hasIndex(i)) + { + return (null); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + if(values == null) + { + return ("null"); + } + + return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(",")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BulkLoadFileRow row = (BulkLoadFileRow) o; + return Objects.deepEquals(values, row.values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int hashCode() + { + return Arrays.hashCode(values); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java new file mode 100644 index 00000000..383dfa21 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -0,0 +1,125 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.util.Iterator; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private Iterator iterator; + + private boolean useLast = false; + private BulkLoadFileRow last; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean hasNext() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + return true; + } + + return iterator.hasNext(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow next() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + useLast = false; + return (this.last); + } + + E e = iterator.next(); + + BulkLoadFileRow row = makeRow(e); + + this.last = row; + return (this.last); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract BulkLoadFileRow makeRow(E e); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void unNext() + { + useLast = true; + } + + + + /******************************************************************************* + ** Getter for iterator + *******************************************************************************/ + public Iterator getIterator() + { + return (this.iterator); + } + + + + /******************************************************************************* + ** Setter for iterator + *******************************************************************************/ + public void setIterator(Iterator iterator) + { + this.iterator = iterator; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java new file mode 100644 index 00000000..f39f4d45 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class CsvFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private CSVParser csvParser; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static CsvFileToRows forString(String csv) throws QException + { + CsvFileToRows csvFileToRows = new CsvFileToRows(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes()); + csvFileToRows.init(byteArrayInputStream); + + return (csvFileToRows); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT + .withIgnoreSurroundingSpaces() + ); + setIterator(csvParser.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening CSV Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(CSVRecord csvRecord) + { + Serializable[] values = new Serializable[csvRecord.size()]; + int i = 0; + for(String s : csvRecord) + { + values[i++] = s; + } + + return (new BulkLoadFileRow(values)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(csvParser != null) + { + csvParser.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java new file mode 100644 index 00000000..d2d6c78a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.util.Iterator; +import java.util.Locale; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface FileToRowsInterface extends AutoCloseable, Iterator +{ + + /*************************************************************************** + ** + ***************************************************************************/ + static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException + { + FileToRowsInterface rs; + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) + { + rs = new CsvFileToRows(); + } + else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx")) + { + rs = new XlsxFileToRows(); + } + else + { + throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx")); + } + + rs.init(inputStream); + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + void init(InputStream inputStream) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + void unNext(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java new file mode 100644 index 00000000..21c9d928 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -0,0 +1,102 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.stream.Stream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.Sheet; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private ReadableWorkbook workbook; + private Stream rows; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + workbook = new ReadableWorkbook(inputStream); + Sheet sheet = workbook.getFirstSheet(); + + rows = sheet.openStream(); + setIterator(rows.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening XLSX Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow) + { + Serializable[] values = new Serializable[readerRow.getCellCount()]; + + for(int i = 0; i < readerRow.getCellCount(); i++) + { + values[i] = readerRow.getCell(i).getText(); + } + + return new BulkLoadFileRow(values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(workbook != null) + { + workbook.close(); + } + + if(rows != null) + { + rows.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java new file mode 100644 index 00000000..d95d48e4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -0,0 +1,503 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertMapping implements Serializable +{ + private String tableName; + private Boolean hasHeaderRow; + + private Layout layout; + + ///////////////////////////////////////////////////////////////////// + // keys in here are: // + // fieldName (for the main table) // + // association.fieldName (for an associated child table) // + // association.association.fieldName (for grandchild associations) // + ///////////////////////////////////////////////////////////////////// + private Map fieldNameToHeaderNameMap = new HashMap<>(); + private Map fieldNameToIndexMap = new HashMap<>(); + private Map fieldNameToDefaultValueMap = new HashMap<>(); + private Map> fieldNameToValueMapping = new HashMap<>(); + + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private List mappedAssociations = new ArrayList<>(); + + private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum Layout + { + FLAT(FlatRowsToRecord::new), + TALL(TallRowsToRecord::new), + WIDE(WideRowsToRecord::new); + + + /*************************************************************************** + ** + ***************************************************************************/ + private final Supplier supplier; + + + + /*************************************************************************** + ** + ***************************************************************************/ + Layout(Supplier supplier) + { + this.supplier = supplier; + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public RowsToRecordInterface newRowsToRecordInterface() + { + return (supplier.get()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + if(hasHeaderRow && fieldNameToHeaderNameMap != null) + { + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + } + else if(fieldNameToIndexMap != null) + { + return (fieldNameToIndexMap); + } + + throw (new QException("Mapping was not properly configured.")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map> getFieldNameToValueMappingForTable(String associatedTableName) + { + Map> rs = new HashMap<>(); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet()) + { + if(shouldProcessFieldForTable(entry.getKey(), associatedTableName)) + { + String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey(); + rs.put(key, entry.getValue()); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain) + { + return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p -> + { + List fieldNameParts = new ArrayList<>(); + List associationParts = new ArrayList<>(); + + if(StringUtils.hasContent(fieldNameWithAssociationPrefix)) + { + fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\."))); + } + + if(StringUtils.hasContent(associationChain)) + { + associationParts.addAll(Arrays.asList(associationChain.split("\\."))); + } + + if(!fieldNameParts.isEmpty()) + { + fieldNameParts.remove(fieldNameParts.size() - 1); + } + + return (fieldNameParts.equals(associationParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + { + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(field.getName(), headerIndex); + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkInsertMapping withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMap + *******************************************************************************/ + public Map getFieldNameToHeaderNameMap() + { + return (this.fieldNameToHeaderNameMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public void setFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToIndexMap + *******************************************************************************/ + public Map getFieldNameToIndexMap() + { + return (this.fieldNameToIndexMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToIndexMap + *******************************************************************************/ + public void setFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToIndexMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for mappedAssociations + *******************************************************************************/ + public List getMappedAssociations() + { + return (this.mappedAssociations); + } + + + + /******************************************************************************* + ** Setter for mappedAssociations + *******************************************************************************/ + public void setMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + } + + + + /******************************************************************************* + ** Fluent setter for mappedAssociations + *******************************************************************************/ + public BulkInsertMapping withMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToDefaultValueMap + *******************************************************************************/ + public Map getFieldNameToDefaultValueMap() + { + if(this.fieldNameToDefaultValueMap == null) + { + this.fieldNameToDefaultValueMap = new HashMap<>(); + } + + return (this.fieldNameToDefaultValueMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public void setFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToValueMapping + *******************************************************************************/ + public Map> getFieldNameToValueMapping() + { + return (this.fieldNameToValueMapping); + } + + + + /******************************************************************************* + ** Setter for fieldNameToValueMapping + *******************************************************************************/ + public void setFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToValueMapping + *******************************************************************************/ + public BulkInsertMapping withFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public Layout getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(Layout layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkInsertMapping withLayout(Layout layout) + { + this.layout = layout; + return (this); + } + + + + /******************************************************************************* + ** Getter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public Map> getTallLayoutGroupByIndexMap() + { + return (this.tallLayoutGroupByIndexMap); + } + + + + /******************************************************************************* + ** Setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public void setTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public BulkInsertMapping withTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java new file mode 100644 index 00000000..ac98adf4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FlatRowsToRecord implements RowsToRecordInterface +{ + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java new file mode 100644 index 00000000..91481980 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RowsToRecordInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + { + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + Serializable value = null; + if(index != null && row != null) + { + value = row.getValueElseNull(index); + } + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + { + value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + } + + if(value != null) + { + record.setValue(fieldName, value); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java new file mode 100644 index 00000000..1dbccd92 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -0,0 +1,316 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TallRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + + String associationNameChain = ""; + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + recordGroupByValues = null; + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // i wrote this condition in here: && rs.size() < limit // + // but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... // + ///////////////////////////////////////////////////////////////////////////////////////////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + QRecord record = new QRecord(); + + Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); + + ////////////////////////////////////////////////////// + // get all rows for the main table from the 0th row // + ////////////////////////////////////////////////////// + BulkLoadFileRow row = rows.get(0); + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + List rs = new ArrayList<>(); + + QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName()); + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + for(BulkLoadFileRow row : rows) + { + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + } + + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + + ////////////////////////////////////////////////// + // use the current row to start the next record // + ////////////////////////////////////////////////// + rowsForCurrentRecord.add(row); + recordGroupByValues = rowGroupByValues; + } + } + + /////////// + // final // + /////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getGroupByValues(BulkLoadFileRow row, List indexes) + { + List rowGroupByValues = new ArrayList<>(); + boolean haveAnyGroupByValues = false; + for(Integer index : indexes) + { + Serializable value = row.getValueElseNull(index); + rowGroupByValues.add(value); + + if(value != null && !"".equals(value)) + { + haveAnyGroupByValues = true; + } + } + + if(!haveAnyGroupByValues) + { + return (null); + } + + return (rowGroupByValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java new file mode 100644 index 00000000..09e074d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ValueMapper +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping) + { + valueMapping(records, mapping, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry> entry : mappingForTable.entrySet()) + { + String fieldName = entry.getKey(); + Map map = entry.getValue(); + String value = record.getValueString(fieldName); + if(value != null && map.containsKey(value)) + { + record.setValue(fieldName, map.get(value)); + } + } + + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java new file mode 100644 index 00000000..40e37071 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java @@ -0,0 +1,333 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException + { + for(String associationName : mapping.getMappedAssociations()) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + // List associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record); + List associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + { + List rs = new ArrayList<>(); + + Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + { + if(entry.getKey().startsWith(associationName + ".")) + { + String fieldName = entry.getKey().substring(associationName.length() + 1); + + ////////////////////////////////////////////////////////////////////////// + // make sure the name here is for this table - not a sub-table under it // + ////////////////////////////////////////////////////////////////////////// + if(!fieldName.contains(".")) + { + fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + } + } + } + + ///////////////////////////////////////////////////////////////////// + // loop over the length of the record, building associated records // + ///////////////////////////////////////////////////////////////////// + QRecord associatedRecord = new QRecord(); + Set processedFieldNames = new HashSet<>(); + boolean gotAnyValues = false; + int subStartIndex = -1; + + for(int i = startIndex; i < endIndex; i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + { + if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + { + /////////////////////////////////////////////// + // ok - this is a value for this association // + /////////////////////////////////////////////// + if(subStartIndex == -1) + { + subStartIndex = i; + } + + String fieldName = entry.getKey(); + if(processedFieldNames.contains(fieldName)) + { + ///////////////////////////////////////////////// + // this means we're starting a new sub-record! // + ///////////////////////////////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + rs.add(associatedRecord); + } + + associatedRecord = new QRecord(); + processedFieldNames = new HashSet<>(); + gotAnyValues = false; + subStartIndex = i + 1; + } + + processedFieldNames.add(fieldName); + + Serializable value = row.getValueElseNull(i); + if(value != null && !"".equals(value)) + { + gotAnyValues = true; + } + + setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + } + } + } + + //////////////////////// + // handle final value // + //////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + rs.add(associatedRecord); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + } + } + } + + /*************************************************************************** + ** + ***************************************************************************/ + // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException + // { + // List rs = new ArrayList<>(); + // String associationNameChainForRecursiveCalls = associationName; + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); + // } + // } + + // Map> indexes = new HashMap<>(); + // for(int i = 0; i < headerRow.size(); i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); + // } + // } + // } + + // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); + + // ////////////////////////////////////////////////////// + // // figure out how many sub-rows we'll be processing // + // ////////////////////////////////////////////////////// + // for(int i = 0; i < maxIndex; i++) + // { + // QRecord associatedRecord = new QRecord(); + // boolean gotAnyValues = false; + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // String fieldName = entry.getKey(); + // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) + // { + // Integer index = indexes.get(fieldName).get(i); + // Serializable value = row.getValueElseNull(index); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); + // } + // } + + // if(gotAnyValues) + // { + // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); + // rs.add(associatedRecord); + // } + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java new file mode 100644 index 00000000..84bdce2f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for CsvFileToRows + *******************************************************************************/ +class CsvFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] csvBytes = """ + one,two,three + 1,2,3,4 + """.getBytes(); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }), headerRow); + assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }), bodyRow); + assertFalse(fileToRowsInterface.hasNext()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java new file mode 100644 index 00000000..e1142e89 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class TestFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private final List rows; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TestFileToRows(List rows) + { + this.rows = rows; + setIterator(this.rows.iterator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + /////////// + // noop! // + /////////// + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + /////////// + // noop! // + /////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(Serializable[] values) + { + return (new BulkLoadFileRow(values)); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java new file mode 100644 index 00000000..8a381ad1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.time.LocalDate; +import java.time.Month; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for XlsxFileToRows + *******************************************************************************/ +class XlsxFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] byteArray = writeExcelBytes(); + + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow); + assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow); + + /////////////////////////////////////////////////////////////////////////////////////// + // make sure there's at least a limit (less than 20) to how many more rows there are // + /////////////////////////////////////////////////////////////////////////////////////// + int otherRowCount = 0; + while(fileToRowsInterface.hasNext() && otherRowCount < 20) + { + fileToRowsInterface.next(); + otherRowCount++; + } + assertFalse(fileToRowsInterface.hasNext()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static byte[] writeExcelBytes() throws QException + { + ReportFormat format = ReportFormat.XLSX; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport()); + GenerateReportActionTest.insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + + byte[] byteArray = baos.toByteArray(); + return byteArray; + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java new file mode 100644 index 00000000..e5474032 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FlatRowsToRecord + *******************************************************************************/ +class FlatRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToHeaderNameMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" }, + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "cost", "cost" + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { 1, "Homer", "Simpson", true }, + new Serializable[] { 2, "Marge", "Simpson", false }, + new Serializable[] { 3, "Bart", "Simpson", "A" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1 } + )); + + BulkLoadFileRow header = null; + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java new file mode 100644 index 00000000..dfd872f5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -0,0 +1,284 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for TallRowsToRecord + *******************************************************************************/ +class TallRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLines() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsic() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart + 1, , , BEER, 500, Coupon Code, 10QOff + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException + { + Integer DEFAULT_STORE_ID = 101; + Integer DEFAULT_LINE_NO = 102; + String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, , , DONUT, , Coupon Code, 10QOff, Size, Large + 1, , , BEER, 500, , , Flavor, Hops + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5), + "orderLine.extrinsics", List.of(7) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testShouldProcessAssociation() + { + TallRowsToRecord tallRowsToRecord = new TallRowsToRecord(); + assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz")); + + assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java new file mode 100644 index 00000000..9e4108e9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for ValueMapper + *******************************************************************************/ +class ValueMapperTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( + "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), + "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), + "lineItem.sku", Map.of("ABC", "Alphabet"), + "lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "extrinsics.key", Map.of("1", "one", "2", "two") + )); + + QRecord inputRecord = new QRecord() + .withValue("storeId", "QQQMart") + .withValue("shipToName", "HoJu") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "ABC") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "foo") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "bar") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", 1) + .withValue("value", "foo") + ); + JSONObject beforeJson = recordToJson(inputRecord); + + QRecord expectedRecord = new QRecord() + .withValue("storeId", 1) + .withValue("shipToName", "Homer") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "Alphabet") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "bar") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "baz") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "one") + .withValue("value", "foo") + ); + JSONObject expectedJson = recordToJson(expectedRecord); + + ValueMapper.valueMapping(List.of(inputRecord), mapping); + JSONObject actualJson = recordToJson(inputRecord); + + System.out.println("Before"); + System.out.println(beforeJson.toString(3)); + System.out.println("Actual"); + System.out.println(actualJson.toString(3)); + System.out.println("Expected"); + System.out.println(expectedJson.toString(3)); + + assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static JSONObject recordToJson(QRecord record) + { + JSONObject jsonObject = new JSONObject(); + for(Map.Entry valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet()) + { + jsonObject.put(valueEntry.getKey(), valueEntry.getValue()); + } + for(Map.Entry> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + JSONArray jsonArray = new JSONArray(); + for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue())) + { + jsonArray.put(recordToJson(associationRecord)); + } + jsonObject.put(associationEntry.getKey(), jsonArray); + } + return (jsonObject); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java new file mode 100644 index 00000000..990ad1ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java @@ -0,0 +1,305 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderAndLines(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testOrderLinesAndOrderExtrinsic(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2, Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException + { + Integer DEFAULT_STORE_ID = 42; + Integer DEFAULT_LINE_NO = 47; + String DEFAULT_LINE_EXTRA_VALUE = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file From d8a0a6c68d00ee27308db29e5a1b661d64fdf915 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:43:17 -0600 Subject: [PATCH 004/209] CE-1955 Move prime-test-database into mainline, to be loaded when javalin starts --- qqq-sample-project/pom.xml | 1 - .../sampleapp/SampleJavalinServer.java | 2 + .../metadata/SampleMetaDataProvider.java | 184 +++++++++++++++++- .../resources/prime-test-database.sql | 21 ++ 4 files changed, 206 insertions(+), 2 deletions(-) rename qqq-sample-project/src/{test => main}/resources/prime-test-database.sql (82%) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index d7e8e2bf..2ff7aee7 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -74,7 +74,6 @@ com.h2database h2 2.2.220 - test diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index 7ec4a911..37f3e3d1 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -64,6 +64,8 @@ public class SampleJavalinServer { qInstance = SampleMetaDataProvider.defineInstance(); + SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); + QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); javalinService = Javalin.create(config -> { diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index ec2aa618..37526837 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -22,7 +22,10 @@ package com.kingsrook.sampleapp.metadata; +import java.io.InputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,7 +40,10 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -48,8 +54,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMe import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -58,6 +69,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -70,9 +82,13 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; +import org.apache.commons.io.IOUtils; /******************************************************************************* @@ -80,7 +96,7 @@ import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; *******************************************************************************/ public class SampleMetaDataProvider { - public static boolean USE_MYSQL = true; + public static boolean USE_MYSQL = false; public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; @@ -97,6 +113,7 @@ public class SampleMetaDataProvider public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PET = "pet"; public static final String TABLE_NAME_CARRIER = "carrier"; public static final String TABLE_NAME_CITY = "city"; @@ -106,6 +123,9 @@ public class SampleMetaDataProvider public static final String SCREEN_0 = "screen0"; public static final String SCREEN_1 = "screen1"; + public static final String BACKEND_NAME_UPLOAD_ARCHIVE = "uploadArchive"; + public static final String UPLOAD_FILE_ARCHIVE_TABLE_NAME = "uploadFileArchive"; + /******************************************************************************* @@ -120,6 +140,10 @@ public class SampleMetaDataProvider qInstance.addBackend(defineFilesystemBackend()); qInstance.addTable(defineTableCarrier()); qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(TABLE_NAME_PERSON)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum(PetSpecies.NAME, PetSpecies.values())); + qInstance.addTable(defineTablePet()); + qInstance.addJoin(defineTablePersonJoinPet()); qInstance.addTable(defineTableCityFile()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); @@ -128,6 +152,9 @@ public class SampleMetaDataProvider qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + qInstance.add(defineUploadArchiveBackend()); + qInstance.add(defineTableUploadFileArchive()); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, SampleMetaDataProvider.class.getPackageName()); defineWidgets(qInstance); @@ -139,6 +166,26 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + public static void primeTestDatabase(String sqlFileName) throws Exception + { + try(Connection connection = ConnectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend())) + { + InputStream primeTestDatabaseSqlStream = SampleMetaDataProvider.class.getResourceAsStream("/" + sqlFileName); + List lines = IOUtils.readLines(primeTestDatabaseSqlStream, StandardCharsets.UTF_8); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -192,6 +239,7 @@ public class SampleMetaDataProvider .withIcon(new QIcon().withName("emoji_people")) .withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people"))) .withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_PET).withIcon(new QIcon().withName("pets"))) .withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand"))) .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName(), QuickSightChartRenderer.class.getSimpleName())) @@ -340,11 +388,62 @@ public class SampleMetaDataProvider QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + qTableMetaData.withAssociation(new Association() + .withAssociatedTableName(TABLE_NAME_PET) + .withName("pets") + .withJoinName(QJoinMetaData.makeInferredJoinName(TABLE_NAME_PERSON, TABLE_NAME_PET))); + return (qTableMetaData); } + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePet() + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(TABLE_NAME_PET) + .withLabel("Pet") + .withBackendName(RDBMS_BACKEND_NAME) + .withPrimaryKeyField("id") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name").withIsRequired(true)) + .withField(new QFieldMetaData("personId", QFieldType.INTEGER).withBackendName("person_id").withIsRequired(true).withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("speciesId", QFieldType.INTEGER).withBackendName("species_id").withIsRequired(true).withPossibleValueSourceName(PetSpecies.NAME)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + + .withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))) + .withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("personId", "speciesId", "birthDate"))) + .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + + return (qTableMetaData); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QJoinMetaData defineTablePersonJoinPet() + { + return new QJoinMetaData() + .withLeftTable(TABLE_NAME_PERSON) + .withRightTable(TABLE_NAME_PET) + .withInferredName() + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "personId")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -524,6 +623,40 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + public static QBackendMetaData defineUploadArchiveBackend() + { + return new FilesystemBackendMetaData() + .withName(BACKEND_NAME_UPLOAD_ARCHIVE) + .withBasePath("/tmp/" + BACKEND_NAME_UPLOAD_ARCHIVE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableMetaData defineTableUploadFileArchive() + { + return (new QTableMetaData() + .withName(UPLOAD_FILE_ARCHIVE_TABLE_NAME) + .withBackendName(BACKEND_NAME_UPLOAD_ARCHIVE) + .withPrimaryKeyField("fileName") + // .withSupplementalMetaData(new ApiTableMetaDataContainer()) // empty container means no apis for this table + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) + .withBackendDetails(new S3TableBackendDetails() + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") + .withBasePath("upload-file-archive"))); + } + + + /******************************************************************************* ** Testing backend step - just sleeps however long you ask it to (or, throws if ** you don't provide a number of seconds to sleep). @@ -625,4 +758,53 @@ public class SampleMetaDataProvider } } + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum PetSpecies implements PossibleValueEnum + { + DOG(1, "Dog"), + CAT(1, "Cat"); + + private final Integer id; + private final String label; + + public static final String NAME = "petSpecies"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + PetSpecies(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return (id); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (label); + } + } + } diff --git a/qqq-sample-project/src/test/resources/prime-test-database.sql b/qqq-sample-project/src/main/resources/prime-test-database.sql similarity index 82% rename from qqq-sample-project/src/test/resources/prime-test-database.sql rename to qqq-sample-project/src/main/resources/prime-test-database.sql index 10185156..1f5e6bc9 100644 --- a/qqq-sample-project/src/test/resources/prime-test-database.sql +++ b/qqq-sample-project/src/main/resources/prime-test-database.sql @@ -42,6 +42,27 @@ INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, a INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 950000, 75); INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 0, 1500000, 1); +DROP TABLE IF EXISTS pet; +CREATE TABLE pet +( + id INT AUTO_INCREMENT primary key , + create_date TIMESTAMP DEFAULT now(), + modify_date TIMESTAMP DEFAULT now(), + + name VARCHAR(80) NOT NULL, + species_id INTEGER NOT NULL, + person_id INTEGER NOT NULL, + birth_date DATE +); + +INSERT INTO pet (id, name, species_id, person_id) VALUES (1, 'Charlie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (2, 'Coco', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (3, 'Louie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (4, 'Barkley', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (5, 'Toby', 1, 2); +INSERT INTO pet (id, name, species_id, person_id) VALUES (6, 'Mae', 2, 3); + + DROP TABLE IF EXISTS carrier; CREATE TABLE carrier ( From e809c773f9ae0c7759a5473afa101c11997d29d5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:44:00 -0600 Subject: [PATCH 005/209] CE-1955 Switch to handle uploaded files via StorageAction into the uploadFileArchive table --- .../javalin/QJavalinProcessHandler.java | 72 +++++++++---------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 48dfef38..7576c64f 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PipedOutputStream; import java.io.Serializable; import java.time.LocalDate; @@ -49,24 +50,22 @@ import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessAction; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -78,9 +77,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMeta import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -490,7 +486,7 @@ public class QJavalinProcessHandler ** todo - make query params have a "field-" type of prefix?? ** *******************************************************************************/ - private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException + private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException, QException { ////////////////////////// // process query string // @@ -521,20 +517,41 @@ public class QJavalinProcessHandler //////////////////////////// // process uploaded files // //////////////////////////// - for(UploadedFile uploadedFile : context.uploadedFiles()) + for(Map.Entry> entry : context.uploadedFileMap().entrySet()) { - try(InputStream content = uploadedFile.content()) + String name = entry.getKey(); + List uploadedFiles = entry.getValue(); + ArrayList storageInputs = new ArrayList<>(); + runProcessInput.addValue(name, storageInputs); + + String storageTableName = QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName(); + if(!StringUtils.hasContent(storageTableName)) { - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes(content.readAllBytes()); - qUploadedFile.setFilename(uploadedFile.filename()); + throw (new QException("UploadFileArchiveTableName was not specified in javalinMetaData. Cannot accept file uploads.")); + } - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - LOG.info("Stored uploaded file in TempFileStateProvider under key: " + key); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); + for(UploadedFile uploadedFile : uploadedFiles) + { + String reference = QValueFormatter.formatDate(LocalDate.now()) + + File.separator + runProcessInput.getProcessName() + + File.separator + UUID.randomUUID() + "-" + uploadedFile.filename(); - archiveUploadedFile(runProcessInput, qUploadedFile); + StorageInput storageInput = new StorageInput(storageTableName).withReference(reference); + storageInputs.add(storageInput); + + try + ( + InputStream content = uploadedFile.content(); + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + ) + { + content.transferTo(outputStream); + LOG.info("Streamed uploaded file", logPair("storageTable", storageTableName), logPair("reference", reference), logPair("processName", runProcessInput.getProcessName()), logPair("uploadFileName", uploadedFile.filename())); + } + catch(QException e) + { + throw (new QException("Error creating output stream in table [" + storageTableName + "] for storage action", e)); + } } } @@ -565,27 +582,6 @@ public class QJavalinProcessHandler - /******************************************************************************* - ** - *******************************************************************************/ - private static void archiveUploadedFile(RunProcessInput runProcessInput, QUploadedFile qUploadedFile) - { - String fileName = QValueFormatter.formatDate(LocalDate.now()) - + File.separator + runProcessInput.getProcessName() - + File.separator + qUploadedFile.getFilename(); - - InsertInput insertInput = new InsertInput(); - insertInput.setTableName(QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName()); - insertInput.setRecords(List.of(new QRecord() - .withValue("fileName", fileName) - .withValue("contents", qUploadedFile.getBytes()) - )); - - new InsertAction().executeAsync(insertInput); - } - - - /******************************************************************************* ** *******************************************************************************/ From 5f081fce44098e6c1a407facb7b446af4ffca1c8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:48:02 -0600 Subject: [PATCH 006/209] CE-1955 Checkstyle! --- .../insert/mapping/TallRowsToRecordTest.java | 24 +++++++++---------- .../insert/mapping/WideRowsToRecordTest.java | 22 ++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index dfd872f5..c623ddda 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -167,9 +167,9 @@ class TallRowsToRecordTest extends BaseTest @Test void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException { - Integer DEFAULT_STORE_ID = 101; - Integer DEFAULT_LINE_NO = 102; - String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file"; + Integer defaultStoreId = 101; + Integer defaultLineNo = 102; + String defaultOrderLineExtraSource = "file"; CsvFileToRows fileToRows = CsvFileToRows.forString(""" orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value @@ -197,9 +197,9 @@ class TallRowsToRecordTest extends BaseTest "orderLine.extrinsics.value", "Line Extrinsic Value" )) .withFieldNameToDefaultValueMap(Map.of( - "storeId", DEFAULT_STORE_ID, - "orderLine.lineNumber", DEFAULT_LINE_NO, - "orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource )) .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) .withTallLayoutGroupByIndexMap(Map.of( @@ -219,17 +219,17 @@ class TallRowsToRecordTest extends BaseTest QRecord order = records.get(0); assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); lineItem = order.getAssociatedRecords().get("orderLine").get(1); assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); @@ -238,16 +238,16 @@ class TallRowsToRecordTest extends BaseTest order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); lineItem = order.getAssociatedRecords().get("orderLine").get(0); assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java index 990ad1ef..51a6c5f7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java @@ -222,9 +222,9 @@ class WideRowsToRecordTest extends BaseTest ***************************************************************************/ private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException { - Integer DEFAULT_STORE_ID = 42; - Integer DEFAULT_LINE_NO = 47; - String DEFAULT_LINE_EXTRA_VALUE = "bar"; + Integer defaultStoreId = 42; + Integer defaultLineNo = 47; + String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); @@ -245,9 +245,9 @@ class WideRowsToRecordTest extends BaseTest .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) .withFieldNameToDefaultValueMap(Map.of( - "storeId", DEFAULT_STORE_ID, - "orderLine.lineNumber", DEFAULT_LINE_NO, - "orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue )) .withTableName(TestUtils.TABLE_NAME_ORDER) .withLayout(BulkInsertMapping.Layout.WIDE) @@ -259,10 +259,10 @@ class WideRowsToRecordTest extends BaseTest QRecord order = records.get(0); assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -276,15 +276,15 @@ class WideRowsToRecordTest extends BaseTest lineItem = order.getAssociatedRecords().get("orderLine").get(2); assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); lineItem = order.getAssociatedRecords().get("orderLine").get(0); From da2be57a17eabf5e4e66066eb88b006e090e1521 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 10:00:19 -0600 Subject: [PATCH 007/209] CE-1955 Add fastexcel-reader (and a pinned version of commons-io, for compatibility) --- qqq-backend-core/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index c9a3c8f5..0944e1e9 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -102,6 +102,11 @@ fastexcel 0.12.15 + + org.dhatim + fastexcel-reader + 0.18.4 + org.apache.poi poi @@ -112,6 +117,14 @@ poi-ooxml 5.2.5 + + + + commons-io + commons-io + 2.16.0 + + com.auth0 auth0 From 4b590b5653a97e908aea9c77bec2a8096fdb765e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 10:02:44 -0600 Subject: [PATCH 008/209] CE-1955 make public stuff used by another test now --- .../core/actions/reporting/GenerateReportActionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 1cf8404a..984a9aa2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -88,7 +88,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; *******************************************************************************/ public class GenerateReportActionTest extends BaseTest { - private static final String REPORT_NAME = "personReport1"; + public static final String REPORT_NAME = "personReport1"; @@ -651,7 +651,7 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private static QReportMetaData defineTableOnlyReport() + public static QReportMetaData defineTableOnlyReport() { QReportMetaData report = new QReportMetaData() .withName(REPORT_NAME) From 39b322336f55a23d585af59ecc42929509c562af Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 16:06:38 -0600 Subject: [PATCH 009/209] CE-1955 Add transaction to validateSecurityFields --- .../qqq/backend/core/actions/tables/DeleteAction.java | 2 +- .../qqq/backend/core/actions/tables/InsertAction.java | 2 +- .../qqq/backend/core/actions/tables/UpdateAction.java | 4 ++-- .../helpers/ValidateRecordSecurityLockHelper.java | 10 ++++++---- .../implementations/memory/MemoryRecordStore.java | 2 +- .../scripts/StoreScriptRevisionProcessStep.java | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index f964b62e..0ded3c93 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -320,7 +320,7 @@ public class DeleteAction QTableMetaData table = deleteInput.getTable(); List primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get()); - ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE); + ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE, deleteInput.getTransaction()); /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 75b17a22..1c799b34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -258,7 +258,7 @@ public class InsertAction extends AbstractQActionFunction records, Action action) throws QException + public static void validateSecurityFields(QTableMetaData table, List records, Action action, QBackendTransaction transaction) throws QException { MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action); if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks())) @@ -101,7 +102,7 @@ public class ValidateRecordSecurityLockHelper // actually check lock values // //////////////////////////////// Map errorRecords = new HashMap<>(); - evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys, transaction); ///////////////////////////////// // propagate errors to records // @@ -141,7 +142,7 @@ public class ValidateRecordSecurityLockHelper ** BUT - WRITE locks - in their case, we read the record no matter what, and in ** here we need to verify we have a key that allows us to WRITE the record. *******************************************************************************/ - private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys) throws QException + private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys, QBackendTransaction transaction) throws QException { if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) { @@ -152,7 +153,7 @@ public class ValidateRecordSecurityLockHelper for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())) { treePosition.add(i); - evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys, transaction); treePosition.remove(treePosition.size() - 1); i++; } @@ -225,6 +226,7 @@ public class ValidateRecordSecurityLockHelper // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) // //////////////////////////////////////////////////////////////////////////////////////////////// QueryInput queryInput = new QueryInput(); + queryInput.setTransaction(transaction); queryInput.setTableName(leftMostJoin.getLeftTable()); QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); queryInput.setFilter(filter); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 7591becb..b53d69a2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -193,7 +193,7 @@ public class MemoryRecordStore if(recordMatches) { qRecord.setErrors(new ArrayList<>()); - ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT); + ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT, null); if(CollectionUtils.nullSafeHasContents(qRecord.getErrors())) { ////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index 96a2e43f..b042360c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -101,7 +101,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // in case the app added a security field to the scripts table, make sure the user is allowed to edit the script // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE); + ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE, transaction); if(CollectionUtils.nullSafeHasContents(script.getErrors())) { throw (new QPermissionDeniedException(script.getErrors().get(0).getMessage())); From 6aafc3d55392f019d66188407c249efbdde96322 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 20:15:47 -0600 Subject: [PATCH 010/209] CE-1955 Mark Serializable --- .../core/model/metadata/frontend/QFrontendFieldMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java index 54b8aba6..6f424b31 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java @@ -42,7 +42,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; * *******************************************************************************/ @JsonInclude(Include.NON_NULL) -public class QFrontendFieldMetaData +public class QFrontendFieldMetaData implements Serializable { private String name; private String label; From 062240a0a5ce3f5f9451aed94a63b2ea9bf49e76 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 20:16:09 -0600 Subject: [PATCH 011/209] CE-1955 Add BULK_LOAD_* values --- .../backend/core/model/metadata/processes/QComponentType.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index d4bd7ff7..c15eba2a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -29,6 +29,9 @@ public enum QComponentType { HELP_TEXT, BULK_EDIT_FORM, + BULK_LOAD_FILE_MAPPING_FORM, + BULK_LOAD_VALUE_MAPPING_FORM, + BULK_LOAD_PROFILE_FORM, VALIDATION_REVIEW_SCREEN, EDIT_FORM, VIEW_FORM, From c09198eed599dc5b05b5f976fbad628e40d397d7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:37:05 -0600 Subject: [PATCH 012/209] CE-1955 Initial checkin --- .../SavedBulkLoadProfileMetaDataProvider.java | 10 + .../DeleteSavedBulkLoadProfileProcess.java | 88 +++++++++ .../QuerySavedBulkLoadProfileProcess.java | 129 ++++++++++++ .../StoreSavedBulkLoadProfileProcess.java | 171 ++++++++++++++++ .../SavedBulkLoadProfileProcessTests.java | 186 ++++++++++++++++++ 5 files changed, 584 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index d8d376e1..23826835 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -40,6 +40,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.DeleteSavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.QuerySavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.StoreSavedBulkLoadProfileProcess; /******************************************************************************* @@ -68,6 +71,13 @@ public class SavedBulkLoadProfileMetaDataProvider { instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); } + + //////////////////////////////////// + // processes for working with 'em // + //////////////////////////////////// + instance.add(StoreSavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(QuerySavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(DeleteSavedBulkLoadProfileProcess.getProcessMetaData()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..e909751e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; + + +/******************************************************************************* + ** Process used by the delete bulkLoadProfile dialog + *******************************************************************************/ +public class DeleteSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(DeleteSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("deleteSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(DeleteSavedBulkLoadProfileProcess.class)) + .withName("delete") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + DeleteInput input = new DeleteInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKeys(List.of(savedBulkLoadProfileId)); + new DeleteAction().execute(input); + } + catch(Exception e) + { + LOG.warn("Error deleting saved bulkLoadProfile", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..c1bdaa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java @@ -0,0 +1,129 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialogs + *******************************************************************************/ +public class QuerySavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(QuerySavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("querySavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(QuerySavedBulkLoadProfileProcess.class)) + .withName("query") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + try + { + if(savedBulkLoadProfileId != null) + { + GetInput input = new GetInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKey(savedBulkLoadProfileId); + + GetOutput output = new GetAction().execute(input); + if(output.getRecord() == null) + { + throw (new QNotFoundException("The requested bulkLoadProfile was not found.")); + } + + runBackendStepOutput.addRecord(output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfile", output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) List.of(output.getRecord())); + } + else + { + String tableName = runBackendStepInput.getValueString("tableName"); + + QueryInput input = new QueryInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) + .withOrderBy(new QFilterOrderBy("label"))); + + QueryOutput output = new QueryAction().execute(input); + runBackendStepOutput.setRecords(output.getRecords()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) output.getRecords()); + } + } + catch(QNotFoundException qnfe) + { + LOG.info("BulkLoadProfile not found", logPair("savedBulkLoadProfileId", savedBulkLoadProfileId)); + throw (qnfe); + } + catch(Exception e) + { + LOG.warn("Error querying for saved bulkLoadProfiles", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..0d8f33d4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialog + *******************************************************************************/ +public class StoreSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(StoreSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("storeSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(StoreSavedBulkLoadProfileProcess.class)) + .withName("store") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + String userId = QContext.getQSession().getUser().getIdReference(); + String tableName = runBackendStepInput.getValueString("tableName"); + String label = runBackendStepInput.getValueString("label"); + + String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); + + QRecord qRecord = new QRecord() + .withValue("id", runBackendStepInput.getValueInteger("id")) + .withValue("mappingJson", mappingJson) + .withValue("label", label) + .withValue("tableName", tableName) + .withValue("userId", userId); + + List savedBulkLoadProfileList; + if(qRecord.getValueInteger("id") == null) + { + checkForDuplicates(userId, tableName, label, null); + + InsertInput input = new InsertInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + InsertOutput output = new InsertAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + else + { + checkForDuplicates(userId, tableName, label, qRecord.getValueInteger("id")); + + UpdateInput input = new UpdateInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + UpdateOutput output = new UpdateAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) savedBulkLoadProfileList); + } + catch(Exception e) + { + LOG.warn("Error storing saved bulkLoadProfile", e); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String processMappingJson(String mappingJson) + { + return mappingJson; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void checkForDuplicates(String userId, String tableName, String label, Integer id) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(SavedBulkLoadProfile.TABLE_NAME); + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("userId", QCriteriaOperator.EQUALS, userId), + new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName), + new QFilterCriteria("label", QCriteriaOperator.EQUALS, label))); + + if(id != null) + { + queryInput.getFilter().addCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, id)); + } + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + { + throw (new QUserFacingException("You already have a saved Bulk Load Profile on this table with this name.")); + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java new file mode 100644 index 00000000..a1692327 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit tests for all saved-bulk-load-profile processes + *******************************************************************************/ +class SavedBulkLoadProfileProcessTests extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QInstance qInstance = QContext.getQInstance(); + new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + //////////////////////////////////////////// + // query - should be no profiles to start // + //////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + + Integer savedBulkLoadProfileId; + { + ///////////////////////// + // store a new profile // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + savedBulkLoadProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + assertNotNull(savedBulkLoadProfileId); + + ////////////////////////////////////////////////////////////////// + // try to store it again - should throw a "duplicate" exception // + ////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ////////////////////////////////////// + // query - should find our profiles // + ////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + { + //////////////////////// + // update our Profile // + //////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Updated Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + Integer anotherSavedProfileId; + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // store a second one w/ different name (will be used below in update-dupe-check use-case) // + ///////////////////////////////////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Second Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + anotherSavedProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + } + + { + ///////////////////////////////////////////////// + // try to rename the second to match the first // + ///////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + + ////////////////////////////////////////// + // should throw a "duplicate" exception // + ////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ///////////////////////// + // delete our profiles // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(DeleteSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + } + + { + ///////////////////////////////////////// + // query - should be no profiles again // + ///////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + } + +} \ No newline at end of file From b684f2409b27e383627f432ac48fff714980134e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:37:36 -0600 Subject: [PATCH 013/209] CE-1955 Avoid type-based exceptions checking security key values --- .../backend/core/model/session/QSession.java | 39 ++++++++++++++++--- .../core/model/session/QSessionTest.java | 14 ++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java index cf050a18..284942ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java @@ -48,7 +48,7 @@ public class QSession implements Serializable, Cloneable private QUser user; private String uuid; - private Set permissions; + private Set permissions; private Map> securityKeyValues; private Map backendVariants; @@ -360,12 +360,38 @@ public class QSession implements Serializable, Cloneable return (false); } - List values = securityKeyValues.get(keyName); - Serializable valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + List values = securityKeyValues.get(keyName); + + Serializable valueAsType; + try + { + valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means the value isn't in the session. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } + for(Serializable keyValue : values) { - Serializable keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); - if(keyValueAsType.equals(valueAsType)) + Serializable keyValueAsType = null; + try + { + keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means this key isn't a match. + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + } + + if(valueAsType.equals(keyValueAsType)) { return (true); } @@ -561,6 +587,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Getter for valuesForFrontend *******************************************************************************/ @@ -591,6 +618,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Fluent setter for a single valuesForFrontend *******************************************************************************/ @@ -604,5 +632,4 @@ public class QSession implements Serializable, Cloneable return (this); } - } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java index cd676342..dd085268 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java @@ -76,7 +76,7 @@ class QSessionTest extends BaseTest void testMixedValueTypes() { QSession session = new QSession().withSecurityKeyValues(Map.of( - "storeId", List.of("100", "200", 300) + "storeId", List.of("100", "200", 300, "four-hundred") )); for(int i : List.of(100, 200, 300)) @@ -86,6 +86,18 @@ class QSessionTest extends BaseTest assertTrue(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should contain: " + i); assertTrue(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should contain: " + i); } + + //////////////////////////////////////////////////////////////////////////// + // next two blocks - used to throw exceptions - now, gracefully be false. // + //////////////////////////////////////////////////////////////////////////// + int i = 400; + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should not contain: " + i); + + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.STRING), "Should not contain: " + i); } From d8ac14a756ed8c0c5b93a994243efbd2c807faf1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:44:43 -0600 Subject: [PATCH 014/209] CE-1955 Checkpoint on bulk-load backend --- .../BulkInsertPrepareFileMappingStep.java | 240 +++++++++++++++ .../BulkInsertPrepareValueMappingStep.java | 250 ++++++++++++++++ .../BulkInsertReceiveFileMappingStep.java | 200 +++++++++++++ .../insert/BulkInsertReceiveFileStep.java | 81 ------ .../BulkInsertReceiveValueMappingStep.java | 105 +++++++ .../bulk/insert/BulkInsertStepUtils.java | 96 ++++++ .../bulk/insert/BulkInsertTransformStep.java | 68 ++++- .../bulk/insert/BulkInsertV2ExtractStep.java | 1 + .../AbstractIteratorBasedFileToRows.java | 2 +- .../insert/filehandling/CsvFileToRows.java | 2 +- .../filehandling/FileToRowsInterface.java | 2 +- .../insert/filehandling/XlsxFileToRows.java | 2 +- .../insert/mapping/BulkInsertMapping.java | 67 ++++- .../mapping/BulkInsertWideLayoutMapping.java | 216 ++++++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 6 +- .../insert/mapping/RowsToRecordInterface.java | 34 ++- .../bulk/insert/mapping/TallRowsToRecord.java | 27 +- .../bulk/insert/mapping/ValueMapper.java | 72 ++++- .../WideRowsToRecordWithExplicitMapping.java | 260 +++++++++++++++++ ...=> WideRowsToRecordWithSpreadMapping.java} | 78 +---- .../insert/{ => model}/BulkLoadFileRow.java | 2 +- .../bulk/insert/model/BulkLoadProfile.java | 133 +++++++++ .../insert/model/BulkLoadProfileField.java | 195 +++++++++++++ .../insert/model/BulkLoadTableStructure.java | 275 ++++++++++++++++++ .../BulkInsertPrepareFileMappingStepTest.java | 69 +++++ ...ulkInsertPrepareValueMappingStepTest.java} | 42 ++- .../filehandling/CsvFileToRowsTest.java | 2 +- .../insert/filehandling/TestFileToRows.java | 2 +- .../filehandling/XlsxFileToRowsTest.java | 2 +- .../insert/mapping/FlatRowsToRecordTest.java | 2 +- .../insert/mapping/TallRowsToRecordTest.java | 101 ++++++- .../bulk/insert/mapping/ValueMapperTest.java | 7 +- ...deRowsToRecordWithExplicitMappingTest.java | 269 +++++++++++++++++ ...ideRowsToRecordWithSpreadMappingTest.java} | 10 +- .../qqq/backend/core/utils/TestUtils.java | 4 +- 35 files changed, 2682 insertions(+), 242 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{WideRowsToRecord.java => WideRowsToRecordWithSpreadMapping.java} (75%) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{ => model}/BulkLoadFileRow.java (99%) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java rename qqq-backend-core/src/{main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java => test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java} (50%) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{WideRowsToRecordTest.java => WideRowsToRecordWithSpreadMappingTest.java} (97%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java new file mode 100644 index 00000000..f7ee32ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.File; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareFileMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + File file = new File(storageInput.getReference()); + runBackendStepOutput.addValue("fileBaseName", file.getName()); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + ///////////////////////////////////////////////// + // read the 1st row, and assume it is a header // + ///////////////////////////////////////////////// + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + ArrayList headerValues = new ArrayList<>(); + ArrayList headerLetters = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i))); + headerLetters.add(toHeaderLetter(i)); + } + runBackendStepOutput.addValue("headerValues", headerValues); + runBackendStepOutput.addValue("headerLetters", headerLetters); + + /////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under preview-rows limit, read rows // + /////////////////////////////////////////////////////////////////////////////////////////// + int previewRows = 0; + int previewRowsLimit = 5; + ArrayList> bodyValues = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.add(new ArrayList<>()); + } + + while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit) + { + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + previewRows++; + + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i))); + } + } + runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } + catch(Exception e) + { + throw (new QException("Error reading bulk load file", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static String toHeaderLetter(int i) + { + StringBuilder rs = new StringBuilder(); + + do + { + rs.insert(0, (char) ('A' + (i % 26))); + i = (i / 26) - 1; + } + while(i >= 0); + + return (rs.toString()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); + runBackendStepOutput.addValue("tableStructure", tableStructure); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> f.getLabel())); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java new file mode 100644 index 00000000..85397442 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -0,0 +1,250 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + ///////////////////////////////////////////////////////////// + // prep the frontend for what field we're going to map now // + ///////////////////////////////////////////////////////////// + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + if(valueMappingFieldIndex == null) + { + valueMappingFieldIndex = 0; + } + else + { + valueMappingFieldIndex++; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview // + //////////////////////////////////////////////////////////////////////////////////////////////////// + if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size()) + { + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + return; + } + + runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex); + + String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName); + + runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field())); + runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName); + runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName()); + + //////////////////////////////////////////////////// + // get all the values from the file in this field // + // todo - should do all mapping fields at once? // + //////////////////////////////////////////////////// + ArrayList fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput); + runBackendStepOutput.addValue("fileValues", fileValues); + + /////////////////////////////////////////////// + // clear these in case not getting set below // + /////////////////////////////////////////////// + runBackendStepOutput.addValue("valueMapping", new HashMap<>()); + runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + HashMap valueMapping = null; + if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName)) + { + valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {}); + runBackendStepOutput.addValue("valueMapping", valueMapping); + + if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName())) + { + HashMap possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping); + runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels); + } + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert prepare value mapping", e); + throw new QException("Unhandled error in bulk insert prepare value mapping step", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException + { + List parts = new ArrayList<>(List.of(fullFieldName.split("\\."))); + String fieldBaseName = parts.remove(parts.size() - 1); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(String associationName : parts) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + table = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + + TableAndField result = new TableAndField(table, table.getField(fieldBaseName)); + return result; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public record TableAndField(QTableMetaData table, QFieldMetaData field) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + private HashMap loadPossibleValues(QFieldMetaData field, Map valueMapping) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setPossibleValueSourceName(field.getPossibleValueSourceName()); + input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + + HashMap rs = new HashMap<>(); + for(QPossibleValue result : output.getResults()) + { + Serializable id = (Serializable) result.getId(); + rs.put(id, result.getLabel()); + } + return rs; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private ArrayList getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + + String associationNameChain = null; + if(fullFieldName.contains(".")) + { + associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.')); + } + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Set values = new LinkedHashSet<>(); + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow); + int index = fieldIndexes.get(field.getName()); + + while(fileToRowsInterface.hasNext()) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + Serializable value = row.getValueElseNull(index); + if(value != null) + { + values.add(ValueUtils.getValueAsString(value)); + } + + if(values.size() > 100) + { + throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName())); + } + } + + return (new ArrayList<>(values)); + } + catch(Exception e) + { + throw (new QException("Error getting values from file", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java new file mode 100644 index 00000000..8f72bb82 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -0,0 +1,200 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow()); + bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout())); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // handle field to name or index mappings (depending on if there's a header row being used) // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow())) + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Map fieldNameToHeaderNameMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + } + } + } + else + { + Map fieldNameToIndexMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex()); + } + } + } + + ///////////////////////////////////// + // do fields w/ default values now // + ///////////////////////////////////// + HashMap fieldNameToDefaultValueMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getDefaultValue() != null) + { + fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue()); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // frontend at this point will have sent just told us which field names need value mapping // + // store those - and let them drive the value-mapping screens that we'll go through next // + // todo - uh, what if those come from profile, dummy!? + ///////////////////////////////////////////////////////////////////////////////////////////// + ArrayList fieldNamesToDoValueMapping = new ArrayList<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping())) + { + fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName()); + + if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings())) + { + bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings()); + } + } + } + runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping)); + + /////////////////////////////////////////////////////////////////////////////////////// + // figure out what associations are being mapped, by looking at the full field names // + /////////////////////////////////////////////////////////////////////////////////////// + Set associationNameSet = new HashSet<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getFieldName().contains(".")) + { + associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + } + } + bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point we're done populating the bulkInsertMapping object. put it in the process state. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java deleted file mode 100644 index 6ebcf3a8..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertReceiveFileStep implements BackendStep -{ - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); - - try - ( - InputStream inputStream = new StorageAction().getInputStream(storageInput); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); - ) - { - BulkLoadFileRow headerRow = fileToRowsInterface.next(); - - List bodyRows = new ArrayList<>(); - while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) - { - bodyRows.add(fileToRowsInterface.next().toString()); - } - - runBackendStepOutput.addValue("header", headerRow.toString()); - runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); - System.out.println("Done receiving file"); - } - catch(QException qe) - { - throw qe; - } - catch(Exception e) - { - throw new QException("Unhandled error in bulk insert extract step", e); - } - - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java new file mode 100644 index 00000000..f9c17667 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + + String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + Map> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping(); + if(fieldNameToValueMapping == null) + { + fieldNameToValueMapping = new HashMap<>(); + bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping); + } + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + //////////////////////////////////////////////// + // put the mapped values into the mapping map // + //////////////////////////////////////////////// + Map mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {}); + fieldNameToValueMapping.put(fieldName, mappedValues); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput); + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index e3dac892..3e4ffa40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -22,10 +22,22 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; /******************************************************************************* @@ -55,4 +67,88 @@ public class BulkInsertStepUtils return (storageInput); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveValueMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveFileMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput) + { + String version = runBackendStepInput.getValueString("version"); + if("v1".equals(version)) + { + String layout = runBackendStepInput.getValueString("layout"); + Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + + ArrayList fieldList = new ArrayList<>(); + + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); + fieldList.add(bulkLoadProfileField); + bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); + bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); + bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) + { + bulkLoadProfileField.setValueMappings(new HashMap<>()); + JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings"); + for(String fileValue : valueMappingsJsonObject.keySet()) + { + bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue))); + } + } + } + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withFieldList(fieldList) + .withHasHeaderRow(hasHeaderRow) + .withLayout(layout); + + return (bulkLoadProfile); + } + else + { + throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); + if(savedBulkLoadProfileId != null) + { + QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); + runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f4eff9a3..18c03671 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -111,6 +111,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); } @@ -121,8 +126,43 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int rowsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + int recordsInThisPage = runBackendStepInput.getRecords().size(); + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + + // split the records w/o UK errors into those w/ e + List recordsWithoutAnyErrors = new ArrayList<>(); + List recordsWithSomeErrors = new ArrayList<>(); + for(QRecord record : runBackendStepInput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + recordsWithSomeErrors.add(record); + } + else + { + recordsWithoutAnyErrors.add(record); + } + } + + ////////////////////////////////////////////////////////////////// + // propagate errors that came into this step out to the summary // + ////////////////////////////////////////////////////////////////// + if(!recordsWithSomeErrors.isEmpty()) + { + for(QRecord record : recordsWithSomeErrors) + { + String message = record.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, null); + } + } + + if(recordsWithoutAnyErrors.isEmpty()) + { + //////////////////////////////////////////////////////////////////////////////// + // skip th rest of this method if there aren't any records w/o errors in them // + //////////////////////////////////////////////////////////////////////////////// + this.rowsProcessed += recordsInThisPage; + } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -130,7 +170,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep InsertInput insertInput = new InsertInput(); insertInput.setInputSource(QInputSource.USER); insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setRecords(recordsWithoutAnyErrors); insertInput.setSkipUniqueKeyCheck(true); ////////////////////////////////////////////////////////////////////// @@ -145,7 +185,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// @@ -159,13 +199,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet()); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + // todo - move this up (before the early return?) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { @@ -187,12 +228,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - insertInput.setRecords(recordsWithoutUkErrors); + insertInput.setRecords(recordsWithoutAnyErrors); InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); @@ -222,8 +263,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); - - this.rowsProcessed += rowsInThisPage; + this.rowsProcessed += recordsInThisPage; } @@ -231,7 +271,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep /******************************************************************************* ** *******************************************************************************/ - private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + private List getRecordsWithoutUniqueKeyErrors(List records, Map>> existingKeys, List uniqueKeys, QTableMetaData table) { //////////////////////////////////////////////////// // if there are no UK's, proceed with all records // @@ -239,7 +279,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithoutUkErrors = new ArrayList<>(); if(existingKeys.isEmpty()) { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + recordsWithoutUkErrors.addAll(records); } else { @@ -255,7 +295,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep // else, get each records keys and see if it already exists or not // // also, build a set of keys we've seen (within this page (or overall?)) // /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) + for(QRecord record : records) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { @@ -333,8 +373,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index 2e49e48a..efacca0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java index 383dfa21..be80c82a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.util.Iterator; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java index f39f4d45..7a58ddef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -28,7 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java index d2d6c78a..9d02ee0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -27,7 +27,7 @@ import java.util.Iterator; import java.util.Locale; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index 21c9d928..a6e336de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -27,7 +27,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java index d95d48e4..fef870b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -32,8 +32,9 @@ import java.util.function.Supplier; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -62,8 +63,10 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private List mappedAssociations = new ArrayList<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private Map wideLayoutMapping = new HashMap<>(); + + private List mappedAssociations = new ArrayList<>(); private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); @@ -72,11 +75,11 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - public enum Layout + public enum Layout implements PossibleValueEnum { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecord::new); + WIDE(WideRowsToRecordWithExplicitMapping::new); /*************************************************************************** @@ -95,6 +98,7 @@ public class BulkInsertMapping implements Serializable } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +106,28 @@ public class BulkInsertMapping implements Serializable { return (supplier.get()); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return StringUtils.ucFirst(name().toLowerCase()); + } } @@ -500,4 +526,35 @@ public class BulkInsertMapping implements Serializable return (this); } + + + /******************************************************************************* + ** Getter for wideLayoutMapping + *******************************************************************************/ + public Map getWideLayoutMapping() + { + return (this.wideLayoutMapping); + } + + + + /******************************************************************************* + ** Setter for wideLayoutMapping + *******************************************************************************/ + public void setWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + } + + + + /******************************************************************************* + ** Fluent setter for wideLayoutMapping + *******************************************************************************/ + public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java new file mode 100644 index 00000000..ad0cce0f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java @@ -0,0 +1,216 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertWideLayoutMapping +{ + private List childRecordMappings; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkInsertWideLayoutMapping(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class ChildRecordMapping + { + Map fieldNameToHeaderNameMaps; + Map associationNameToChildRecordMappingMap; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public Map getFieldNameToHeaderNameMaps() + { + return (this.fieldNameToHeaderNameMaps); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public Map getAssociationNameToChildRecordMappingMap() + { + return (this.associationNameToChildRecordMappingMap); + } + + + + /******************************************************************************* + ** Setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Fluent setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Map getFieldIndexes(BulkLoadFileRow headerRow) + { + // todo memoize or otherwise don't recompute + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) + { + String headerName = entry.getValue(); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(entry.getKey(), headerIndex); + } + } + } + + return (rs); + } + } + + + + /******************************************************************************* + ** Getter for childRecordMappings + *******************************************************************************/ + public List getChildRecordMappings() + { + return (this.childRecordMappings); + } + + + + /******************************************************************************* + ** Setter for childRecordMappings + *******************************************************************************/ + public void setChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /******************************************************************************* + ** Fluent setter for childRecordMappings + *******************************************************************************/ + public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index ac98adf4..44702312 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -30,8 +30,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* @@ -62,13 +62,13 @@ public class FlatRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 91481980..25aa6543 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -44,26 +46,50 @@ public interface RowsToRecordInterface /*************************************************************************** - ** + ** returns true if value from row was used, else false. ***************************************************************************/ - default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) { - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + String fieldName = field.getName(); + QFieldType type = field.getType(); + + boolean valueFromRowWasUsed = false; + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; Serializable value = null; if(index != null && row != null) { value = row.getValueElseNull(index); + if(value != null && !"".equals(value)) + { + valueFromRowWasUsed = true; + } } else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) { value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); } + /* note - moving this to ValueMapper... + if(value != null) + { + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + */ + if(value != null) { record.setValue(fieldName, value); } + + return (valueFromRowWasUsed); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 1dbccd92..690f705d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -35,8 +35,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -75,7 +75,12 @@ public class TallRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); - List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -126,13 +131,24 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } + /*************************************************************************** + ** + ***************************************************************************/ + private List groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException + { + Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); + return new ArrayList<>(fieldIndexes.values()); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -148,7 +164,7 @@ public class TallRowsToRecord implements RowsToRecordInterface BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); } ///////////////////////////// @@ -230,7 +246,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) { - throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls); + // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } List rowGroupByValues = getGroupByValues(row, groupByIndexes); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 09e074d3..0f1e8cfc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -25,9 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -35,12 +45,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ValueMapper { + private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); + + + /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping) + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException { - valueMapping(records, mapping, null); + valueMapping(records, mapping, table, null); } @@ -48,7 +62,7 @@ public class ValueMapper /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException { if(CollectionUtils.nullSafeIsEmpty(records)) { @@ -58,20 +72,58 @@ public class ValueMapper Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { - for(Map.Entry> entry : mappingForTable.entrySet()) + for(Map.Entry valueEntry : record.getValues().entrySet()) { - String fieldName = entry.getKey(); - Map map = entry.getValue(); - String value = record.getValueString(fieldName); - if(value != null && map.containsKey(value)) + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) { - record.setValue(fieldName, map.get(value)); + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null) + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + + record.setValue(field.getName(), value); } + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) { - valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java new file mode 100644 index 00000000..adc67ec7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java @@ -0,0 +1,260 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException + { + for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) + { + String associationName = entry.getKey(); + BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; + + for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) + { + QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); + if(associatedRecord != null) + { + record.withAssociatedRecord(associationName, associatedRecord); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); + + QRecord associatedRecord = new QRecord(); + boolean usedAnyValuesFromRow = false; + + for(QFieldMetaData field : table.getFields().values()) + { + boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + usedAnyValuesFromRow |= valueFromRowWasUsed; + } + + if(usedAnyValuesFromRow) + { + processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); + return (associatedRecord); + } + else + { + return (null); + } + } + + // /*************************************************************************** + // ** + // ***************************************************************************/ + // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + // { + // List rs = new ArrayList<>(); + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationName + ".")) + // { + // String fieldName = entry.getKey().substring(associationName.length() + 1); + + // ////////////////////////////////////////////////////////////////////////// + // // make sure the name here is for this table - not a sub-table under it // + // ////////////////////////////////////////////////////////////////////////// + // if(!fieldName.contains(".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + // } + // } + // } + + // ///////////////////////////////////////////////////////////////////// + // // loop over the length of the record, building associated records // + // ///////////////////////////////////////////////////////////////////// + // QRecord associatedRecord = new QRecord(); + // Set processedFieldNames = new HashSet<>(); + // boolean gotAnyValues = false; + // int subStartIndex = -1; + + // for(int i = startIndex; i < endIndex; i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // /////////////////////////////////////////////// + // // ok - this is a value for this association // + // /////////////////////////////////////////////// + // if(subStartIndex == -1) + // { + // subStartIndex = i; + // } + + // String fieldName = entry.getKey(); + // if(processedFieldNames.contains(fieldName)) + // { + // ///////////////////////////////////////////////// + // // this means we're starting a new sub-record! // + // ///////////////////////////////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + // rs.add(associatedRecord); + // } + + // associatedRecord = new QRecord(); + // processedFieldNames = new HashSet<>(); + // gotAnyValues = false; + // subStartIndex = i + 1; + // } + + // processedFieldNames.add(fieldName); + + // Serializable value = row.getValueElseNull(i); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + // } + // } + // } + + // //////////////////////// + // // handle final value // + // //////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + // rs.add(associatedRecord); + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java similarity index 75% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 40e37071..f87fa17b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -37,8 +37,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -48,7 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* ** *******************************************************************************/ -public class WideRowsToRecord implements RowsToRecordInterface +public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface { private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -77,7 +77,7 @@ public class WideRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); @@ -85,7 +85,7 @@ public class WideRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } @@ -199,7 +199,7 @@ public class WideRowsToRecord implements RowsToRecordInterface gotAnyValues = true; } - setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i); } } } @@ -228,77 +228,11 @@ public class WideRowsToRecord implements RowsToRecordInterface { if(!processedFieldNames.contains(field.getName())) { - setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); } } } - /*************************************************************************** - ** - ***************************************************************************/ - // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException - // { - // List rs = new ArrayList<>(); - // String associationNameChainForRecursiveCalls = associationName; - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); - // } - // } - - // Map> indexes = new HashMap<>(); - // for(int i = 0; i < headerRow.size(); i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); - // } - // } - // } - - // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); - - // ////////////////////////////////////////////////////// - // // figure out how many sub-rows we'll be processing // - // ////////////////////////////////////////////////////// - // for(int i = 0; i < maxIndex; i++) - // { - // QRecord associatedRecord = new QRecord(); - // boolean gotAnyValues = false; - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // String fieldName = entry.getKey(); - // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) - // { - // Integer index = indexes.get(fieldName).get(i); - // Serializable value = row.getValueElseNull(index); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); - // } - // } - - // if(gotAnyValues) - // { - // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); - // rs.add(associatedRecord); - // } - // } - - // return (rs); - // } - /*************************************************************************** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java similarity index 99% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java index 487f606a..23b80a56 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java new file mode 100644 index 00000000..8bcfb97c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -0,0 +1,133 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; + + +/*************************************************************************** + * this is the model of a saved bulk load profile - which is what passes back + * and forth with the frontend. + ****************************************************************************/ +public class BulkLoadProfile implements Serializable +{ + private ArrayList fieldList; + private Boolean hasHeaderRow; + private String layout; + + + + /******************************************************************************* + ** Getter for fieldList + *******************************************************************************/ + public ArrayList getFieldList() + { + return (this.fieldList); + } + + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public String getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(String layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkLoadProfile withLayout(String layout) + { + this.layout = layout; + return (this); + } + + + /******************************************************************************* + ** Setter for fieldList + *******************************************************************************/ + public void setFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + } + + + + /******************************************************************************* + ** Fluent setter for fieldList + *******************************************************************************/ + public BulkLoadProfile withFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java new file mode 100644 index 00000000..6bedb18b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.Map; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class BulkLoadProfileField +{ + private String fieldName; + private Integer columnIndex; + private Serializable defaultValue; + private Boolean doValueMapping; + private Map valueMappings; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public BulkLoadProfileField withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnIndex + *******************************************************************************/ + public Integer getColumnIndex() + { + return (this.columnIndex); + } + + + + /******************************************************************************* + ** Setter for columnIndex + *******************************************************************************/ + public void setColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + } + + + + /******************************************************************************* + ** Fluent setter for columnIndex + *******************************************************************************/ + public BulkLoadProfileField withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultValue + *******************************************************************************/ + public Serializable getDefaultValue() + { + return (this.defaultValue); + } + + + + /******************************************************************************* + ** Setter for defaultValue + *******************************************************************************/ + public void setDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + } + + + + /******************************************************************************* + ** Fluent setter for defaultValue + *******************************************************************************/ + public BulkLoadProfileField withDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for doValueMapping + *******************************************************************************/ + public Boolean getDoValueMapping() + { + return (this.doValueMapping); + } + + + + /******************************************************************************* + ** Setter for doValueMapping + *******************************************************************************/ + public void setDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for doValueMapping + *******************************************************************************/ + public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueMappings + *******************************************************************************/ + public Map getValueMappings() + { + return (this.valueMappings); + } + + + + /******************************************************************************* + ** Setter for valueMappings + *******************************************************************************/ + public void setValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + } + + + + /******************************************************************************* + ** Fluent setter for valueMappings + *******************************************************************************/ + public BulkLoadProfileField withValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java new file mode 100644 index 00000000..db55198f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -0,0 +1,275 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadTableStructure implements Serializable +{ + private boolean isMain; + private boolean isMany; + + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + + private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? + private ArrayList associations; + + + + /******************************************************************************* + ** Getter for isMain + *******************************************************************************/ + public boolean getIsMain() + { + return (this.isMain); + } + + + + /******************************************************************************* + ** Setter for isMain + *******************************************************************************/ + public void setIsMain(boolean isMain) + { + this.isMain = isMain; + } + + + + /******************************************************************************* + ** Fluent setter for isMain + *******************************************************************************/ + public BulkLoadTableStructure withIsMain(boolean isMain) + { + this.isMain = isMain; + return (this); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public BulkLoadTableStructure withIsMany(boolean isMany) + { + this.isMany = isMany; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkLoadTableStructure withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public BulkLoadTableStructure withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields + *******************************************************************************/ + public ArrayList getFields() + { + return (this.fields); + } + + + + /******************************************************************************* + ** Setter for fields + *******************************************************************************/ + public void setFields(ArrayList fields) + { + this.fields = fields; + } + + + + /******************************************************************************* + ** Fluent setter for fields + *******************************************************************************/ + public BulkLoadTableStructure withFields(ArrayList fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationPath + *******************************************************************************/ + public String getAssociationPath() + { + return (this.associationPath); + } + + + + /******************************************************************************* + ** Setter for associationPath + *******************************************************************************/ + public void setAssociationPath(String associationPath) + { + this.associationPath = associationPath; + } + + + + /******************************************************************************* + ** Fluent setter for associationPath + *******************************************************************************/ + public BulkLoadTableStructure withAssociationPath(String associationPath) + { + this.associationPath = associationPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for associations + *******************************************************************************/ + public ArrayList getAssociations() + { + return (this.associations); + } + + + + /******************************************************************************* + ** Setter for associations + *******************************************************************************/ + public void setAssociations(ArrayList associations) + { + this.associations = associations; + } + + + + /******************************************************************************* + ** Fluent setter for associations + *******************************************************************************/ + public BulkLoadTableStructure withAssociations(ArrayList associations) + { + this.associations = associations; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addAssociation(BulkLoadTableStructure association) + { + if(this.associations == null) + { + this.associations = new ArrayList<>(); + } + this.associations.add(association); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java new file mode 100644 index 00000000..ea6e0e8f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertPrepareMappingStep + *******************************************************************************/ +class BulkInsertPrepareFileMappingStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("PointlessArithmeticExpression") + @Test + void testToHeaderLetter() + { + assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0)); + assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1)); + assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25)); + + assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0)); + assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1)); + assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25)); + + assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0)); + assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1)); + assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25)); + + assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0)); + assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1)); + assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25)); + + assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0)); + assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1)); + assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2)); + + assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0)); + assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1)); + + assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java similarity index 50% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java index 0071aa49..6953c5bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java @@ -22,38 +22,34 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.util.Map; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* - ** + ** Unit test for BulkInsertPrepareValueMappingStep *******************************************************************************/ -public class BulkInsertReceiveMappingStep implements BackendStep +class BulkInsertPrepareValueMappingStepTest extends BaseTest { - /*************************************************************************** + /******************************************************************************* ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + *******************************************************************************/ + @Test + void test() throws QException { - BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); - bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); - bulkInsertMapping.setHasHeaderRow(true); - bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( - "firstName", "firstName", - "lastName", "Last Name" - )); - runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName()); + assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName()); + assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName()); + assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName()); - // probably need to what, receive the mapping object, store it into state - // what, do we maybe return to a different sub-mapping screen (e.g., values) - // then at some point - cool - proceed to ETL's steps } -} +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java index 84bdce2f..77b60bd8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -25,7 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.ByteArrayInputStream; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java index e1142e89..a9d07b63 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -26,7 +26,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /*************************************************************************** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index 8a381ad1..c681f685 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index e5474032..e8c561e5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -29,8 +29,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index c623ddda..5c942fa2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -87,14 +87,14 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -145,7 +145,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -155,7 +155,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -168,7 +168,7 @@ class TallRowsToRecordTest extends BaseTest void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException { Integer defaultStoreId = 101; - Integer defaultLineNo = 102; + String defaultLineNo = "102"; String defaultOrderLineExtraSource = "file"; CsvFileToRows fileToRows = CsvFileToRows.forString(""" @@ -221,7 +221,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -240,7 +240,92 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAutomaticGroupByAllIndexes() throws QException + { + Integer defaultStoreId = 101; + String defaultLineNo = "102"; + String defaultOrderLineExtraSource = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large + 1, Homer, Simpson, BEER, 500, , , Flavor, Hops + 1, Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 9e4108e9..7b868766 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -26,8 +26,11 @@ import java.io.Serializable; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -44,7 +47,7 @@ class ValueMapperTest extends BaseTest ** *******************************************************************************/ @Test - void test() + void test() throws QException { BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), @@ -94,7 +97,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping); + ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java new file mode 100644 index 00000000..39434f78 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java @@ -0,0 +1,269 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordWithExplicitMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """; + + Integer defaultStoreId = 42; + Integer defaultLineNo = 47; + String defaultLineExtraValue = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 1", "quantity", "Quantity 1"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 2", "quantity", "Quantity 2"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 3", "quantity", "Quantity 3"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) + )))) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java similarity index 97% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 51a6c5f7..45bcc751 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* ** Unit test for WideRowsToRecord *******************************************************************************/ -class WideRowsToRecordTest extends BaseTest +class WideRowsToRecordWithSpreadMappingTest extends BaseTest { /******************************************************************************* @@ -80,7 +80,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -150,7 +150,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -229,7 +229,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index ff274dc9..8ca6da8f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -644,6 +644,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() @@ -700,7 +701,8 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER)) .withField(new QFieldMetaData("key", QFieldType.STRING)) - .withField(new QFieldMetaData("value", QFieldType.STRING)); + .withField(new QFieldMetaData("value", QFieldType.STRING)) + .withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests } From 22ce5acf4620ef9a905ae9e9596e5e47caa30537 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:45:24 -0600 Subject: [PATCH 015/209] CE-1955 Make filename its own path element in uploadedFile processing --- .../kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 7576c64f..cb9ee53e 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -534,7 +534,8 @@ public class QJavalinProcessHandler { String reference = QValueFormatter.formatDate(LocalDate.now()) + File.separator + runProcessInput.getProcessName() - + File.separator + UUID.randomUUID() + "-" + uploadedFile.filename(); + + File.separator + UUID.randomUUID() + + File.separator + uploadedFile.filename(); StorageInput storageInput = new StorageInput(storageTableName).withReference(reference); storageInputs.add(storageInput); From 07886214f5f427170015cb0f2e787466db821a84 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:53:49 -0600 Subject: [PATCH 016/209] CE-1955 Test fixes --- .../bulk/insert/BulkInsertTransformStep.java | 2 +- .../insert/mapping/FlatRowsToRecordTest.java | 2 +- .../bulk/insert/mapping/ValueMapperTest.java | 8 ++++---- ...ideRowsToRecordWithExplicitMappingTest.java | 18 +++++++++--------- .../WideRowsToRecordWithSpreadMappingTest.java | 14 +++++++------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 18c03671..6e28c10a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -233,7 +233,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - insertInput.setRecords(recordsWithoutAnyErrors); + insertInput.setRecords(recordsWithoutUkErrors); InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index e8c561e5..435dcf92 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -84,7 +84,7 @@ class FlatRowsToRecordTest extends BaseTest records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); - assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); assertEquals(List.of("Ned"), getValues(records, "firstName")); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 7b868766..7317d963 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -52,15 +52,15 @@ class ValueMapperTest extends BaseTest BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), - "lineItem.sku", Map.of("ABC", "Alphabet"), - "lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "orderLine.sku", Map.of("ABC", "Alphabet"), + "orderLine.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), "extrinsics.key", Map.of("1", "one", "2", "two") )); QRecord inputRecord = new QRecord() .withValue("storeId", "QQQMart") .withValue("shipToName", "HoJu") - .withAssociatedRecord("lineItem", new QRecord() + .withAssociatedRecord("orderLine", new QRecord() .withValue("sku", "ABC") .withAssociatedRecord("extrinsics", new QRecord() .withValue("key", "myKey") @@ -80,7 +80,7 @@ class ValueMapperTest extends BaseTest QRecord expectedRecord = new QRecord() .withValue("storeId", 1) .withValue("shipToName", "Homer") - .withAssociatedRecord("lineItem", new QRecord() + .withAssociatedRecord("orderLine", new QRecord() .withValue("sku", "Alphabet") .withAssociatedRecord("extrinsics", new QRecord() .withValue("key", "myKey") diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java index 39434f78..4f8bd7a1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java @@ -83,13 +83,13 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -139,7 +139,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -147,7 +147,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -166,7 +166,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest """; Integer defaultStoreId = 42; - Integer defaultLineNo = 47; + String defaultLineNo = "47"; String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); @@ -197,8 +197,8 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest new BulkInsertWideLayoutMapping.ChildRecordMapping( Map.of("sku", "SKU 3", "quantity", "Quantity 3"), Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) )))) )), "extrinsics", new BulkInsertWideLayoutMapping(List.of( @@ -225,7 +225,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -247,7 +247,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 45bcc751..61fc6d35 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -101,13 +101,13 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -173,7 +173,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -181,7 +181,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -223,7 +223,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException { Integer defaultStoreId = 42; - Integer defaultLineNo = 47; + String defaultLineNo = "47"; String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); @@ -261,7 +261,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -283,7 +283,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); From 2918235f46fedebbc7af5f66c26a3511245bb848 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:25:17 -0600 Subject: [PATCH 017/209] CE-1955 Add version field to the built BulkLoadProfile --- .../bulk/insert/BulkInsertStepUtils.java | 1 + .../bulk/insert/model/BulkLoadProfile.java | 38 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index 3e4ffa40..7c7f008b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -125,6 +125,7 @@ public class BulkInsertStepUtils } BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion(version) .withFieldList(fieldList) .withHasHeaderRow(hasHeaderRow) .withLayout(layout); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java index 8bcfb97c..2fe07b3c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -33,8 +33,10 @@ import java.util.ArrayList; public class BulkLoadProfile implements Serializable { private ArrayList fieldList; - private Boolean hasHeaderRow; - private String layout; + + private Boolean hasHeaderRow; + private String layout; + private String version; @@ -48,7 +50,6 @@ public class BulkLoadProfile implements Serializable - /******************************************************************************* ** Getter for hasHeaderRow *******************************************************************************/ @@ -110,6 +111,7 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* ** Setter for fieldList *******************************************************************************/ @@ -130,4 +132,34 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public BulkLoadProfile withVersion(String version) + { + this.version = version; + return (this); + } + + } From 07c04132773e583939a6de369680124276740475 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:25:40 -0600 Subject: [PATCH 018/209] CE-1955 Initial checkin (plus add a memory-storage table to testutils) --- .../bulk/insert/BulkInsertV2Test.java | 236 ++++++++++++++++++ .../qqq/backend/core/utils/TestUtils.java | 18 ++ 2 files changed, 254 insertions(+) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java new file mode 100644 index 00000000..8beaa891 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java @@ -0,0 +1,236 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkInsertV2Test extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois", + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + String defaultEmail = "noone@kingsrook.com"; + + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + QInstance qInstance = QContext.getQInstance(); + String processName = "PersonBulkInsertV2"; + new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(processName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); + + ////////////////////////////// + // simulate the file upload // + ////////////////////////////// + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); + } + catch(IOException e) + { + throw (e); + } + + ////////////////////////// + // continue post-upload // + ////////////////////////// + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); + assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); + + //////////////////////////////////////////////////////////////////////////////// + // all subsequent steps will want these data - so set up a lambda to set them // + //////////////////////////////////////////////////////////////////////////////// + Consumer addProfileToRunProcessInput = (RunProcessInput input) -> + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + }; + + //////////////////////////////// + // continue post file-mapping // + //////////////////////////////// + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); + assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); + assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); + assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); + assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); + + ///////////////////////////////// + // continue post value-mapping // + ///////////////////////////////// + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + ///////////////////////////////// + // continue post review screen // + ///////////////////////////////// + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Jane", records.get(1).getValueString("firstName")); + + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + + assertEquals(2, records.get(0).getValueInteger("homeStateId")); + assertEquals(1, records.get(1).getValueInteger("homeStateId")); + + assertEquals(defaultEmail, records.get(0).getValueString("email")); + assertEquals(defaultEmail, records.get(1).getValueString("email")); + + assertEquals(42, records.get(0).getValueInteger("noOfShoes")); + assertNull(records.get(1).getValue("noOfShoes")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 8ca6da8f..89adac13 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -142,6 +142,7 @@ public class TestUtils public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; public static final String TABLE_NAME_TWO_KEYS = "twoKeys"; + public static final String TABLE_NAME_MEMORY_STORAGE = "memoryStorage"; public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache"; @@ -204,6 +205,7 @@ public class TestUtils qInstance.addTable(defineTablePerson()); qInstance.addTable(defineTableTwoKeys()); + qInstance.addTable(defineTableMemoryStorage()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(definePersonMemoryCacheTable()); @@ -594,6 +596,22 @@ public class TestUtils + /******************************************************************************* + ** Define a table in the memory store that can be used for the StorageAction + *******************************************************************************/ + public static QTableMetaData defineTableMemoryStorage() + { + return new QTableMetaData() + .withName(TABLE_NAME_MEMORY_STORAGE) + .withLabel("Memory Storage") + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("reference") + .withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)); + } + + + /******************************************************************************* ** Define the 'person' table used in standard tests. *******************************************************************************/ From 9ad9d526346f3d596fb775c6dcf8e72006254fc3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:29:13 -0600 Subject: [PATCH 019/209] CE-1955 Add method defineTableBulkInsertV2 (needs to not be v2 i guess) --- .../core/instances/QInstanceEnricher.java | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 0e101379..ee307632 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; @@ -74,7 +75,13 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -867,6 +874,116 @@ public class QInstanceEnricher + /******************************************************************************* + ** + *******************************************************************************/ + public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName) + { + qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values())); + + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertV2ExtractStep.class, + BulkInsertTransformStep.class, + BulkInsertLoadStep.class, + values + ) + .withName(processName) + .withIcon(new QIcon().withName("library_add")) + .withLabel(table.getLabel() + " Bulk Insert") + .withTableName(table.getName()) + .withIsHidden(true) + .withPermissionRules(qInstance.getDefaultPermissionRules().clone() + .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); + + List editableFields = new ArrayList<>(); + for(QFieldSection section : CollectionUtils.nonNullList(table.getSections())) + { + for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames())) + { + try + { + QFieldMetaData field = table.getField(fieldName); + if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB)) + { + editableFields.add(field); + } + } + catch(Exception e) + { + // shrug? + } + } + } + + String fieldsForHelpText = editableFields.stream() + .map(QFieldMetaData::getLabel) + .collect(Collectors.joining(", ")); + + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() + .withName("upload") + .withLabel("Upload File") + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData() + .withType(QComponentType.HELP_TEXT) + .withValue("previewText", "file upload instructions") + .withValue("text", "Upload a CSV or Excel (.xlsx) file with the following columns:\n" + fieldsForHelpText)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); + + QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() + .withName("prepareFileMapping") + .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class)); + + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withLabel("File Mapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); + + QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() + .withName("receiveFileMapping") + .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class)); + + QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData() + .withName("prepareValueMapping") + .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class)); + + QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() + .withName("valueMapping") + .withLabel("Value Mapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); + + QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() + .withName("receiveValueMapping") + .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); + + int i = 0; + process.addStep(i++, uploadScreen); + + process.addStep(i++, prepareFileMappingStep); + process.addStep(i++, fileMappingScreen); + process.addStep(i++, receiveFileMappingStep); + + process.addStep(i++, prepareValueMappingStep); + process.addStep(i++, valueMappingScreen); + process.addStep(i++, receiveValueMappingStep); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); + + ////////////////////////////////////////////////////////////////////////////////////////// + // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // + ////////////////////////////////////////////////////////////////////////////////////////// + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + qInstance.addProcess(process); + } + + + /******************************************************************************* ** *******************************************************************************/ From 58ae17bbac3b2d58085066f0c31bc0b080462e0c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 10:07:26 -0600 Subject: [PATCH 020/209] CE-1955 - Bulk load checkpoint: - Switch wide format to identify associations via comma-number-indexes... - Add suggested mappings - use header name instead of column index for mappings - add counts of children process summary lines - excel value/type handling --- .../core/instances/QInstanceEnricher.java | 68 +---- .../BulkInsertPrepareFileMappingStep.java | 128 ++------- .../BulkInsertPrepareValueMappingStep.java | 2 +- .../BulkInsertReceiveFileMappingStep.java | 15 +- .../BulkInsertReceiveValueMappingStep.java | 2 +- .../bulk/insert/BulkInsertStepUtils.java | 1 + .../bulk/insert/BulkInsertTransformStep.java | 29 +- .../bulk/insert/BulkInsertV2ExtractStep.java | 2 +- .../insert/filehandling/XlsxFileToRows.java | 41 ++- .../mapping/BulkInsertWideLayoutMapping.java | 216 -------------- .../mapping/BulkLoadMappingSuggester.java | 232 +++++++++++++++ .../BulkLoadTableStructureBuilder.java | 132 +++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 1 + .../insert/mapping/RowsToRecordInterface.java | 58 ++-- .../bulk/insert/mapping/TallRowsToRecord.java | 1 + .../bulk/insert/mapping/ValueMapper.java | 1 + ...licitFieldNameSuffixIndexBasedMapping.java | 196 +++++++++++++ .../WideRowsToRecordWithExplicitMapping.java | 260 ----------------- .../WideRowsToRecordWithSpreadMapping.java | 1 + .../{mapping => model}/BulkInsertMapping.java | 65 ++--- .../insert/model/BulkLoadProfileField.java | 32 +++ ....java => BulkInsertV2FullProcessTest.java} | 15 +- .../mapping/BulkLoadMappingSuggesterTest.java | 209 ++++++++++++++ .../insert/mapping/FlatRowsToRecordTest.java | 1 + .../insert/mapping/TallRowsToRecordTest.java | 1 + .../bulk/insert/mapping/ValueMapperTest.java | 1 + ...tFieldNameSuffixIndexBasedMappingTest.java | 175 ++++++++++++ ...deRowsToRecordWithExplicitMappingTest.java | 269 ------------------ ...WideRowsToRecordWithSpreadMappingTest.java | 1 + 29 files changed, 1167 insertions(+), 988 deletions(-) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{mapping => model}/BulkInsertMapping.java (93%) rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{BulkInsertV2Test.java => BulkInsertV2FullProcessTest.java} (92%) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index ee307632..9a48204a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -73,7 +73,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; @@ -81,7 +80,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -815,75 +813,11 @@ public class QInstanceEnricher /******************************************************************************* ** *******************************************************************************/ - private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) + public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { Map values = new HashMap<>(); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - BulkInsertExtractStep.class, - BulkInsertTransformStep.class, - BulkInsertLoadStep.class, - values - ) - .withName(processName) - .withLabel(table.getLabel() + " Bulk Insert") - .withTableName(table.getName()) - .withIsHidden(true) - .withPermissionRules(qInstance.getDefaultPermissionRules().clone() - .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); - - List editableFields = new ArrayList<>(); - for(QFieldSection section : CollectionUtils.nonNullList(table.getSections())) - { - for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames())) - { - try - { - QFieldMetaData field = table.getField(fieldName); - if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB)) - { - editableFields.add(field); - } - } - catch(Exception e) - { - // shrug? - } - } - } - - String fieldsForHelpText = editableFields.stream() - .map(QFieldMetaData::getLabel) - .collect(Collectors.joining(", ")); - - QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() - .withName("upload") - .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - - process.addStep(0, uploadScreen); - process.getFrontendStep("review").setRecordListFields(editableFields); - qInstance.addProcess(process); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName) - { - qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values())); - - Map values = new HashMap<>(); - values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( BulkInsertV2ExtractStep.class, BulkInsertTransformStep.class, diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index f7ee32ce..f939df3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -25,27 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.File; import java.io.InputStream; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; -import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -62,7 +54,26 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); - buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildSuggestedMapping(List headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + { + BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); } @@ -121,6 +132,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep } } runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } catch(Exception e) { @@ -147,94 +159,4 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep return (rs.toString()); } - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) - { - String tableName = runBackendStepInput.getValueString("tableName"); - BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); - runBackendStepOutput.addValue("tableStructure", tableStructure); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) - { - QTableMetaData table = QContext.getQInstance().getTable(tableName); - - BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); - tableStructure.setTableName(tableName); - tableStructure.setLabel(table.getLabel()); - - Set associationJoinFieldNamesToExclude = new HashSet<>(); - - if(association == null) - { - tableStructure.setIsMain(true); - tableStructure.setIsMany(false); - tableStructure.setAssociationPath(null); - } - else - { - tableStructure.setIsMain(false); - - QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); - if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) - { - tableStructure.setIsMany(true); - } - - for(JoinOn joinOn : join.getJoinOns()) - { - //////////////////////////////////////////////////////////////////////////////////////////////// - // don't allow the user to map the "join field" from a child up to its parent // - // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // - //////////////////////////////////////////////////////////////////////////////////////////////// - if(join.getLeftTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); - } - else if(join.getRightTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getRightField()); - } - } - - if(!StringUtils.hasContent(parentAssociationPath)) - { - tableStructure.setAssociationPath(association.getName()); - } - else - { - tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); - } - } - - ArrayList fields = new ArrayList<>(); - tableStructure.setFields(fields); - for(QFieldMetaData field : table.getFields().values()) - { - if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) - { - fields.add(field); - } - } - - fields.sort(Comparator.comparing(f -> f.getLabel())); - - for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) - { - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); - tableStructure.addAssociation(associatedStructure); - } - - return (tableStructure); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 85397442..19e696e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -51,7 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 8f72bb82..bc6ae239 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; @@ -102,7 +102,12 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep BulkLoadFileRow headerRow = fileToRowsInterface.next(); for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) { - if(bulkLoadProfileField.getColumnIndex() != null) + if(bulkLoadProfileField.getHeaderName() != null) + { + String headerName = bulkLoadProfileField.getHeaderName(); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + else if(bulkLoadProfileField.getColumnIndex() != null) { String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); @@ -164,7 +169,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { if(bulkLoadProfileField.getFieldName().contains(".")) { - associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0]; + associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.'))); } } bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java index f9c17667..e1b5bc9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.utils.JsonUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index 7c7f008b..b0db3ad0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -109,6 +109,7 @@ public class BulkInsertStepUtils BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); fieldList.add(bulkLoadProfileField); bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null); bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 6e28c10a..09d9ca8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; @@ -67,7 +68,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); - private Map ukErrorSummaries = new HashMap<>(); + private Map ukErrorSummaries = new HashMap<>(); + private Map associationsToInsertSummaries = new HashMap<>(); private QTableMetaData table; @@ -259,6 +261,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep { okSummary.incrementCountAndAddPrimaryKey(null); outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } } } @@ -366,6 +375,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); + for(Map.Entry entry : associationsToInsertSummaries.entrySet()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst(); + if(association.isPresent()) + { + QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + String associationLabel = associationTable.getLabel(); + + ProcessSummaryLine line = entry.getValue(); + line.setSingularFutureMessage(associationLabel + " record will be inserted."); + line.setPluralFutureMessage(associationLabel + " records will be inserted."); + line.setSingularPastMessage(associationLabel + " record was inserted."); + line.setPluralPastMessage(associationLabel + " records were inserted."); + line.pickMessage(isForResultScreen); + line.addSelfToListIfAnyCount(rs); + } + } + for(Map.Entry entry : ukErrorSummaries.entrySet()) { UniqueKey uniqueKey = entry.getKey(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index efacca0f..2da27e7d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -32,8 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index a6e336de..f5cecd9c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -25,9 +25,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.time.LocalDateTime; +import java.util.Optional; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; @@ -74,7 +79,41 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional dateTime = readerRow.getCellAsDate(i); + if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100) + { + yield dateTime.get(); + } + + Optional optionalBigDecimal = readerRow.getCellAsNumber(i); + if(optionalBigDecimal.isPresent()) + { + BigDecimal bigDecimal = optionalBigDecimal.get(); + if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0) + { + yield bigDecimal.intValue(); + } + + yield bigDecimal; + } + + yield (null); + } + case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null); + case STRING, FORMULA -> cell.getText(); + case EMPTY, ERROR -> null; + }; + } } return new BulkLoadFileRow(values); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java deleted file mode 100644 index ad0cce0f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertWideLayoutMapping -{ - private List childRecordMappings; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public BulkInsertWideLayoutMapping(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static class ChildRecordMapping - { - Map fieldNameToHeaderNameMaps; - Map associationNameToChildRecordMappingMap; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Getter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public Map getFieldNameToHeaderNameMaps() - { - return (this.fieldNameToHeaderNameMaps); - } - - - - /******************************************************************************* - ** Setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Fluent setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - return (this); - } - - - - /******************************************************************************* - ** Getter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public Map getAssociationNameToChildRecordMappingMap() - { - return (this.associationNameToChildRecordMappingMap); - } - - - - /******************************************************************************* - ** Setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Fluent setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - return (this); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public Map getFieldIndexes(BulkLoadFileRow headerRow) - { - // todo memoize or otherwise don't recompute - Map rs = new HashMap<>(); - - //////////////////////////////////////////////////////// - // for the current file, map header values to indexes // - //////////////////////////////////////////////////////// - Map headerToIndexMap = new HashMap<>(); - for(int i = 0; i < headerRow.size(); i++) - { - String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - headerToIndexMap.put(headerValue, i); - } - - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - // loop over fields - finding what header name they are mapped to - then what index that header is at. // - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) - { - String headerName = entry.getValue(); - if(headerName != null) - { - Integer headerIndex = headerToIndexMap.get(headerName); - if(headerIndex != null) - { - rs.put(entry.getKey(), headerIndex); - } - } - } - - return (rs); - } - } - - - - /******************************************************************************* - ** Getter for childRecordMappings - *******************************************************************************/ - public List getChildRecordMappings() - { - return (this.childRecordMappings); - } - - - - /******************************************************************************* - ** Setter for childRecordMappings - *******************************************************************************/ - public void setChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /******************************************************************************* - ** Fluent setter for childRecordMappings - *******************************************************************************/ - public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - return (this); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java new file mode 100644 index 00000000..71f0edcc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -0,0 +1,232 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; + + +/******************************************************************************* + ** Given a bulk-upload, create a suggested mapping + *******************************************************************************/ +public class BulkLoadMappingSuggester +{ + private Map massagedHeadersWithoutNumbersToIndexMap; + private Map massagedHeadersWithNumbersToIndexMap; + + private String layout = "FLAT"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + { + massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), true); + + if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i); + } + } + + massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), false); + + if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithNumbersToIndexMap.put(headerValue, i); + } + } + + ArrayList fieldList = new ArrayList<>(); + processTable(tableStructure, fieldList, headerRow); + + ///////////////////////////////////////////////// + // sort the fields to match the column indexes // + ///////////////////////////////////////////////// + fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex())); + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion("v1") + .withLayout(layout) + .withHasHeaderRow(true) + .withFieldList(fieldList); + + return (bulkLoadProfile); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processTable(BulkLoadTableStructure tableStructure, ArrayList fieldList, List headerRow) + { + Map rs = new HashMap<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + String fieldName = massageHeader(field.getName(), false); + String fieldLabel = massageHeader(field.getLabel(), false); + String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false); + String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName(); + + //////////////////////////////////////////////////////////////////////////////////// + // consider, if this is a many-table, if there are many matches, for wide mode... // + //////////////////////////////////////////////////////////////////////////////////// + if(tableStructure.getIsMany()) + { + List matchingIndexes = new ArrayList<>(); + + for(Map.Entry entry : massagedHeadersWithNumbersToIndexMap.entrySet()) + { + String header = entry.getKey(); + if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$")) + { + matchingIndexes.add(entry.getValue()); + } + } + + if(CollectionUtils.nullSafeHasContents(matchingIndexes)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields // + // else, if only 1, allow us to go down into the TALL block below // + // note - should we do a merger at the end, in case we found some wide, some tall? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + if(matchingIndexes.size() > 1) + { + layout = "WIDE"; + + int i = 0; + for(Integer index : matchingIndexes) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName + "," + i) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + i++; + } + + continue; + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Integer index = null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of these potential identities of the field: // + // 1) its label, massaged // + // 2) its name, massaged // + // 3) its label, massaged, with numbers stripped away // + // 4) its name, massaged, with numbers stripped away // + // check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. // + // this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case // + // that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true))) + { + if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity); + } + else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity); + } + + if(index != null) + { + break; + } + } + + if(index != null) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + if(tableStructure.getIsMany() && layout.equals("FLAT")) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // the first time we find an is-many child, if we were still marked as flat, go to tall // + ////////////////////////////////////////////////////////////////////////////////////////// + layout = "TALL"; + } + } + } + + //////////////////////////////////////////// + // recursively process child associations // + //////////////////////////////////////////// + for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations())) + { + processTable(associationTableStructure, fieldList, headerRow); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String massageHeader(String header, boolean stripNumbers) + { + if(header == null) + { + return (null); + } + + String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", ""); + return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java new file mode 100644 index 00000000..0ac81f00 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** utility to build BulkLoadTableStructure objects for a QQQ Table. + *******************************************************************************/ +public class BulkLoadTableStructureBuilder +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadTableStructure buildTableStructure(String tableName) + { + return (buildTableStructure(tableName, null, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 44702312..d1a5d4c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 25aa6543..740781c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -27,9 +27,10 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -48,42 +49,49 @@ public interface RowsToRecordInterface /*************************************************************************** ** returns true if value from row was used, else false. ***************************************************************************/ - default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex) { - String fieldName = field.getName(); - QFieldType type = field.getType(); + return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null); + } - boolean valueFromRowWasUsed = false; - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + /*************************************************************************** + ** returns true if value from row was used, else false. + ***************************************************************************/ + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List wideAssociationIndexes) + { + boolean valueFromRowWasUsed = false; - Serializable value = null; - if(index != null && row != null) + ///////////////////////////////////////////////////////////////////////////////////////////////// + // build full field-name -- possibly associations, then field name, then possibly index-suffix // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String fieldName = field.getName(); + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) { - value = row.getValueElseNull(index); + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix; + + ////////////////////////////////////////////// + // ok - look in the row - then the defaults // + ////////////////////////////////////////////// + Serializable value = null; + if(columnIndex != null && row != null) + { + value = row.getValueElseNull(columnIndex); if(value != null && !"".equals(value)) { valueFromRowWasUsed = true; } } - else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName)) { - value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); } - /* note - moving this to ValueMapper... - if(value != null) - { - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); - } - } - */ - if(value != null) { record.setValue(fieldName, value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 690f705d..4587336c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 0f1e8cfc..fdf1b6bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java new file mode 100644 index 00000000..9c85647e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -0,0 +1,196 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** use a flatter mapping object, where field names look like: + ** associationChain.fieldName,index.subIndex + *******************************************************************************/ +public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** may return null, if there were no values in the row for this (sub-wide) record. + ***************************************************************************/ + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + { + ////////////////////////////////////////////////////// + // start by building the record with its own fields // + ////////////////////////////////////////////////////// + QRecord record = new QRecord(); + boolean hadAnyValuesInRow = false; + for(QFieldMetaData field : table.getFields().values()) + { + hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + } + + if(!hadAnyValuesInRow) + { + return (null); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + List rs = new ArrayList<>(); + + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + for(int i = 0; true; i++) + { + // todo - doesn't support grand-children + List wideAssociationIndexes = List.of(i); + Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + if(fieldIndexes.isEmpty()) + { + break; + } + + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + if(record != null) + { + rs.add(record); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java deleted file mode 100644 index adc67ec7..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.Pair; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface -{ - private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); - - - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException - { - QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); - if(table == null) - { - throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); - } - - List rs = new ArrayList<>(); - - Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); - - while(fileToRowsInterface.hasNext() && rs.size() < limit) - { - BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = new QRecord(); - - for(QFieldMetaData field : table.getFields().values()) - { - setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); - } - - processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); - - rs.add(record); - } - - ValueMapper.valueMapping(rs, mapping, table); - - return (rs); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException - { - for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) - { - String associationName = entry.getKey(); - BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); - - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isEmpty()) - { - throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); - } - - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - - String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; - - for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) - { - QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); - if(associatedRecord != null) - { - record.withAssociatedRecord(associationName, associatedRecord); - } - } - } - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException - { - Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); - - QRecord associatedRecord = new QRecord(); - boolean usedAnyValuesFromRow = false; - - for(QFieldMetaData field : table.getFields().values()) - { - boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); - usedAnyValuesFromRow |= valueFromRowWasUsed; - } - - if(usedAnyValuesFromRow) - { - processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); - return (associatedRecord); - } - else - { - return (null); - } - } - - // /*************************************************************************** - // ** - // ***************************************************************************/ - // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException - // { - // List rs = new ArrayList<>(); - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationName + ".")) - // { - // String fieldName = entry.getKey().substring(associationName.length() + 1); - - // ////////////////////////////////////////////////////////////////////////// - // // make sure the name here is for this table - not a sub-table under it // - // ////////////////////////////////////////////////////////////////////////// - // if(!fieldName.contains(".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); - // } - // } - // } - - // ///////////////////////////////////////////////////////////////////// - // // loop over the length of the record, building associated records // - // ///////////////////////////////////////////////////////////////////// - // QRecord associatedRecord = new QRecord(); - // Set processedFieldNames = new HashSet<>(); - // boolean gotAnyValues = false; - // int subStartIndex = -1; - - // for(int i = startIndex; i < endIndex; i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // /////////////////////////////////////////////// - // // ok - this is a value for this association // - // /////////////////////////////////////////////// - // if(subStartIndex == -1) - // { - // subStartIndex = i; - // } - - // String fieldName = entry.getKey(); - // if(processedFieldNames.contains(fieldName)) - // { - // ///////////////////////////////////////////////// - // // this means we're starting a new sub-record! // - // ///////////////////////////////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); - // rs.add(associatedRecord); - // } - - // associatedRecord = new QRecord(); - // processedFieldNames = new HashSet<>(); - // gotAnyValues = false; - // subStartIndex = i + 1; - // } - - // processedFieldNames.add(fieldName); - - // Serializable value = row.getValueElseNull(i); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); - // } - // } - // } - - // //////////////////////// - // // handle final value // - // //////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); - // rs.add(associatedRecord); - // } - - // return (rs); - // } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) - { - for(QFieldMetaData field : table.getFields().values()) - { - if(!processedFieldNames.contains(field.getName())) - { - setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index f87fa17b..7b215d2e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java similarity index 93% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index fef870b1..86c19775 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; @@ -34,7 +34,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -63,8 +66,7 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private Map wideLayoutMapping = new HashMap<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); private List mappedAssociations = new ArrayList<>(); @@ -79,7 +81,7 @@ public class BulkInsertMapping implements Serializable { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecordWithExplicitMapping::new); + WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new); /*************************************************************************** @@ -137,10 +139,21 @@ public class BulkInsertMapping implements Serializable ***************************************************************************/ @JsonIgnore public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + return getFieldIndexes(table, associationNameChain, headerRow, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException { if(hasHeaderRow && fieldNameToHeaderNameMap != null) { - return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes)); } else if(fieldNameToIndexMap != null) { @@ -208,7 +221,7 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) { Map rs = new HashMap<>(); @@ -222,13 +235,19 @@ public class BulkInsertMapping implements Serializable headerToIndexMap.put(headerValue, i); } + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// // loop over fields - finding what header name they are mapped to - then what index that header is at. // ///////////////////////////////////////////////////////////////////////////////////////////////////////// String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; for(QFieldMetaData field : table.getFields().values()) { - String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); if(headerName != null) { Integer headerIndex = headerToIndexMap.get(headerName); @@ -527,34 +546,4 @@ public class BulkInsertMapping implements Serializable } - - /******************************************************************************* - ** Getter for wideLayoutMapping - *******************************************************************************/ - public Map getWideLayoutMapping() - { - return (this.wideLayoutMapping); - } - - - - /******************************************************************************* - ** Setter for wideLayoutMapping - *******************************************************************************/ - public void setWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - } - - - - /******************************************************************************* - ** Fluent setter for wideLayoutMapping - *******************************************************************************/ - public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java index 6bedb18b..f7ce0f6c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -33,6 +33,7 @@ public class BulkLoadProfileField { private String fieldName; private Integer columnIndex; + private String headerName; private Serializable defaultValue; private Boolean doValueMapping; private Map valueMappings; @@ -192,4 +193,35 @@ public class BulkLoadProfileField return (this); } + + /******************************************************************************* + ** Getter for headerName + *******************************************************************************/ + public String getHeaderName() + { + return (this.headerName); + } + + + + /******************************************************************************* + ** Setter for headerName + *******************************************************************************/ + public void setHeaderName(String headerName) + { + this.headerName = headerName; + } + + + + /******************************************************************************* + ** Fluent setter for headerName + *******************************************************************************/ + public BulkLoadProfileField withHeaderName(String headerName) + { + this.headerName = headerName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java similarity index 92% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java index 8beaa891..7edbaafa 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -58,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertV2Test extends BaseTest +class BulkInsertV2FullProcessTest extends BaseTest { /******************************************************************************* @@ -124,7 +125,7 @@ class BulkInsertV2Test extends BaseTest QInstance qInstance = QContext.getQInstance(); String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); + new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // @@ -159,6 +160,16 @@ class BulkInsertV2Test extends BaseTest runProcessOutput = new RunProcessAction().execute(runProcessInput); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + + ////////////////////////////////////////////////////// + // assert about the suggested mapping that was done // + ////////////////////////////////////////////////////// + Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); + assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); + assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); //////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java new file mode 100644 index 00000000..a54e08ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for BulkLoadMappingSuggester + *******************************************************************************/ +class BulkLoadMappingSuggesterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleFlat() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); + List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("FLAT", bulkLoadProfile.getLayout()); + assertNull(getFieldByName(bulkLoadProfile, "id")); + assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleTall() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTallWithTableNamesOnAssociations() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testChallengingAddress1And2() + { + try + { + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + finally + { + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleWide() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("WIDE", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex()); + assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex()); + + ///////////////////////////////////////////////////////////////// + // assert that the order of fields matches the file's ordering // + ///////////////////////////////////////////////////////////////// + assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"), + bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName) + { + return (bulkLoadProfile.getFieldList().stream() + .filter(f -> f.getFieldName().equals(fieldName)) + .findFirst().orElse(null)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 435dcf92..7124e946 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 5c942fa2..1a0ef2ab 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 7317d963..4348cb2b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java new file mode 100644 index 00000000..6e59c527 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping + *******************************************************************************/ +class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1", + "orderLine.sku,1", "SKU 2", + "orderLine.quantity,1", "Quantity 2", + "orderLine.sku,2", "SKU 3", + "orderLine.quantity,2", "Quantity 3" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap()) + .with("orderNo", "orderNo") + .with("shipToName", "Ship To") + + .with("orderLine.sku,0", "SKU 1") + .with("orderLine.quantity,0", "Quantity 1") + .with("orderLine.sku,1", "SKU 2") + .with("orderLine.quantity,1", "Quantity 2") + .with("orderLine.sku,2", "SKU 3") + .with("orderLine.quantity,2", "Quantity 3") + + .with("extrinsics.key,0", "Extrinsic Key 1") + .with("extrinsics.value,0", "Extrinsic Value 1") + .with("extrinsics.key,1", "Extrinsic Key 2") + .with("extrinsics.value,1", "Extrinsic Value 2") + .build() + ) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.lineNumber,0", "1", + "orderLine.lineNumber,1", "2", + "orderLine.lineNumber,2", "3" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java deleted file mode 100644 index 4f8bd7a1..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -/******************************************************************************* - ** Unit test for WideRowsToRecord - *******************************************************************************/ -class WideRowsToRecordWithExplicitMappingTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderAndLinesWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )) - )) - .withMappedAssociations(List.of("orderLine")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - .withMappedAssociations(List.of("orderLine", "extrinsics")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 - 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, - 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 - """; - - Integer defaultStoreId = 42; - String defaultLineNo = "47"; - String defaultLineExtraValue = "bar"; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 1", "quantity", "Quantity 1"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 2", "quantity", "Quantity 2"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 3", "quantity", "Quantity 3"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) - )))) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - - .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) - .withFieldNameToDefaultValueMap(Map.of( - "storeId", defaultStoreId, - "orderLine.lineNumber", defaultLineNo, - "orderLine.extrinsics.value", defaultLineExtraValue - )) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(1); - assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(2); - assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - - lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private List getValues(List records, String fieldName) - { - return (records.stream().map(r -> r.getValue(fieldName)).toList()); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 61fc6d35..8bd49beb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; From bdbb2d2d002171aba5438cd2499e6b3a1fc322a4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 10:09:05 -0600 Subject: [PATCH 021/209] CE-1955 - Bulk load checkpoint - setting uploadFileArchiveTable in javalin metadata --- .../java/com/kingsrook/sampleapp/SampleJavalinServer.java | 4 ++++ .../kingsrook/sampleapp/metadata/SampleMetaDataProvider.java | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index 37f3e3d1..ad8166b0 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -25,6 +25,7 @@ package com.kingsrook.sampleapp; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; +import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; import com.kingsrook.sampleapp.metadata.SampleMetaDataProvider; import io.javalin.Javalin; import io.javalin.plugin.bundled.CorsPluginConfig; @@ -67,6 +68,9 @@ public class SampleJavalinServer SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); + qJavalinImplementation.setJavalinMetaData(new QJavalinMetaData() + .withUploadedFileArchiveTableName(SampleMetaDataProvider.UPLOAD_FILE_ARCHIVE_TABLE_NAME)); + javalinService = Javalin.create(config -> { // todo - not all? diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 37526837..77b05601 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -376,7 +376,7 @@ public class SampleMetaDataProvider .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name").withIsRequired(true)) .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name").withIsRequired(true)) .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("email", QFieldType.STRING).withIsRequired(true)) .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary").withDisplayFormat(DisplayFormat.CURRENCY)) .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked").withDisplayFormat(DisplayFormat.COMMAS)) @@ -769,7 +769,7 @@ public class SampleMetaDataProvider CAT(1, "Cat"); private final Integer id; - private final String label; + private final String label; public static final String NAME = "petSpecies"; From 3c06e0e589a8f2e68d63eb5339c5db2eb03116c3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:10:01 -0600 Subject: [PATCH 022/209] CE-1955 - Test fixes --- .../actions/reporting/GenerateReportActionTest.java | 6 ++++-- .../bulk/insert/BulkInsertV2FullProcessTest.java | 8 ++++++-- .../bulk/insert/filehandling/XlsxFileToRowsTest.java | 12 ++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 984a9aa2..0f84cc78 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -643,7 +643,7 @@ public class GenerateReportActionTest extends BaseTest Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); + assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name", "Birth Date"); } @@ -674,7 +674,9 @@ public class GenerateReportActionTest extends BaseTest .withColumns(List.of( new QReportField().withName("id"), new QReportField().withName("firstName"), - new QReportField().withName("lastName"))))); + new QReportField().withName("lastName"), + new QReportField().withName("birthDate") + )))); return report; } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java index 7edbaafa..b721d89d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java @@ -167,8 +167,12 @@ class BulkInsertV2FullProcessTest extends BaseTest Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); - assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); - assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("firstName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(3, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("lastName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getFieldName()); + assertEquals(4, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getColumnIndex()); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getColumnIndex()); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index c681f685..d8c89180 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -24,7 +24,10 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.Serializable; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.Month; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -54,7 +57,7 @@ class XlsxFileToRowsTest extends BaseTest ** *******************************************************************************/ @Test - void test() throws QException + void test() throws QException, IOException { byte[] byteArray = writeExcelBytes(); @@ -63,8 +66,8 @@ class XlsxFileToRowsTest extends BaseTest BulkLoadFileRow headerRow = fileToRowsInterface.next(); BulkLoadFileRow bodyRow = fileToRowsInterface.next(); - assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow); - assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow); + assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name", "Birth Date"}), headerRow); + assertEquals(new BulkLoadFileRow(new Serializable[] {1, "Darin", "Jonson", LocalDateTime.of(1980, Month.JANUARY, 31, 0, 0)}), bodyRow); /////////////////////////////////////////////////////////////////////////////////////// // make sure there's at least a limit (less than 20) to how many more rows there are // @@ -83,7 +86,7 @@ class XlsxFileToRowsTest extends BaseTest /*************************************************************************** ** ***************************************************************************/ - private static byte[] writeExcelBytes() throws QException + private static byte[] writeExcelBytes() throws QException, IOException { ReportFormat format = ReportFormat.XLSX; ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -100,6 +103,7 @@ class XlsxFileToRowsTest extends BaseTest new GenerateReportAction().execute(reportInput); byte[] byteArray = baos.toByteArray(); + // FileUtils.writeByteArrayToFile(new File("/tmp/xlsx.xlsx"), byteArray); return byteArray; } From c883749ba9e25bbdf33dce85fd4068852792aec3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:15:13 -0600 Subject: [PATCH 023/209] CE-1955 - Remove bulk-insert v1 test; rename bulkInsertV2 test --- ...st.java => BulkInsertFullProcessTest.java} | 2 +- .../bulk/insert/BulkInsertTest.java | 157 ------------------ 2 files changed, 1 insertion(+), 158 deletions(-) rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{BulkInsertV2FullProcessTest.java => BulkInsertFullProcessTest.java} (99%) delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java similarity index 99% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index b721d89d..7037662a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -59,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertV2FullProcessTest extends BaseTest +class BulkInsertFullProcessTest extends BaseTest { /******************************************************************************* diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java deleted file mode 100644 index 496ad0be..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - -/******************************************************************************* - ** Unit test for full bulk insert process - *******************************************************************************/ -class BulkInsertTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - @AfterEach - void beforeAndAfterEach() - { - MemoryRecordStore.getInstance().reset(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow1() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow2() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvHeaderUsingLabels() - { - return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// - assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); - String processUUID = runProcessOutput.getProcessUUID(); - - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - - runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); - - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); - assertThat(runProcessOutput.getException()).isEmpty(); - - //////////////////////////////////// - // query for the inserted records // - //////////////////////////////////// - List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNotNull(records.get(0).getValue("id")); - assertNotNull(records.get(1).getValue("id")); - } - -} \ No newline at end of file From 1c2638a5c48de401419e68315040f34d7cb0fc51 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:27:44 -0600 Subject: [PATCH 024/209] CE-1955 - Boosting test-coverage during bulk-load rollout --- .../core/instances/QInstanceEnricher.java | 4 +- .../bulk/insert/BulkInsertExtractStep.java | 134 ++++++++++-------- .../bulk/insert/BulkInsertV2ExtractStep.java | 127 ----------------- .../streamedwithfrontend/NoopLoadStep.java | 3 + .../insert/BulkInsertFullProcessTest.java | 3 +- .../NoopLoadStepTest.java | 50 +++++++ .../core/utils/aggregates/AggregatesTest.java | 51 +++++++ 7 files changed, 186 insertions(+), 186 deletions(-) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 9a48204a..4e8d0237 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -73,13 +73,13 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -819,7 +819,7 @@ public class QInstanceEnricher values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - BulkInsertV2ExtractStep.class, + BulkInsertExtractStep.class, BulkInsertTransformStep.class, BulkInsertLoadStep.class, values diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index c29d1648..694ae332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -22,84 +22,106 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.InputStream; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; -import com.kingsrook.qqq.backend.core.context.QContext; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; -import com.kingsrook.qqq.backend.core.state.AbstractStateKey; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; /******************************************************************************* ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. *******************************************************************************/ public class BulkInsertExtractStep extends AbstractExtractStep { + + /*************************************************************************** + ** + ***************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); - Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); - if(optionalUploadedFile.isEmpty()) + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) { - throw (new QException("Could not find uploaded file")); - } + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - byte[] bytes = optionalUploadedFile.get().getBytes(); - String fileName = optionalUploadedFile.get().getFilename(); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; - ///////////////////////////////////////////////////// - // let the user specify field labels instead names // - ///////////////////////////////////////////////////// - QTableMetaData table = runBackendStepInput.getTable(); - String tableName = runBackendStepInput.getTableName(); - QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); - for(Map.Entry entry : table.getFields().entrySet()) - { - mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); - } + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); - ////////////////////////////////////////////////////////////////////////// - // get the non-editable fields - they'll be blanked out in a customizer // - ////////////////////////////////////////////////////////////////////////// - List nonEditableFields = table.getFields().values().stream() - .filter(f -> !f.getIsEditable()) - .toList(); - - if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() - .withRecordPipe(getRecordPipe()) - .withLimit(getLimit()) - .withCsv(new String(bytes)) - .withDoCorrectValueTypes(true) - .withTable(QContext.getQInstance().getTable(tableName)) - .withMapping(mapping) - .withRecordCustomizer((record) -> + if(page.size() > remainingLimit) { - //////////////////////////////////////////// - // remove values from non-editable fields // - //////////////////////////////////////////// - for(QFieldMetaData nonEditableField : nonEditableFields) - { - record.setValue(nonEditableField.getName(), null); - } - })); + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } } - else + catch(QException qe) { - throw (new QUserFacingException("Unsupported file type.")); + throw qe; } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java deleted file mode 100644 index 2da27e7d..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.io.InputStream; -import java.util.List; -import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; - - -/******************************************************************************* - ** Extract step for generic table bulk-insert ETL process - ** - ** This step does a little bit of transforming, actually - taking rows from - ** an uploaded file, and potentially merging them (for child-table use-cases) - ** and applying the "Mapping" - to put fully built records into the pipe for the - ** Transform step. - *******************************************************************************/ -public class BulkInsertV2ExtractStep extends AbstractExtractStep -{ - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - int rowsAdded = 0; - int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); - - StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); - BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); - RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); - - try - ( - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - InputStream inputStream = new StorageAction().getInputStream(storageInput); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); - ) - { - /////////////////////////////////////////////////////////// - // read the header row (if this file & mapping uses one) // - /////////////////////////////////////////////////////////// - BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - - //////////////////////////////////////////////////////////////////////////////////////////////////////// - // while there are more rows in the file - and we're under the limit - get more records form the file // - //////////////////////////////////////////////////////////////////////////////////////////////////////// - while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) - { - int remainingLimit = originalLimit - rowsAdded; - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - int pageLimit = Math.min(remainingLimit, getMaxPageSize()); - List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); - - if(page.size() > remainingLimit) - { - ///////////////////////////////////////////////////////////// - // in case we got back more than we asked for, sub-list it // - ///////////////////////////////////////////////////////////// - page = page.subList(0, remainingLimit); - } - - ///////////////////////////////////////////// - // send this page of records into the pipe // - ///////////////////////////////////////////// - getRecordPipe().addRecords(page); - rowsAdded += page.size(); - } - } - catch(QException qe) - { - throw qe; - } - catch(Exception e) - { - throw new QException("Unhandled error in bulk insert extract step", e); - } - - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private int getMaxPageSize() - { - return (1000); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java index 0fcd5abf..f675f128 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -33,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp *******************************************************************************/ public class NoopLoadStep extends AbstractLoadStep { + private static final QLogger LOG = QLogger.getLogger(NoopLoadStep.class); /******************************************************************************* @@ -45,6 +47,7 @@ public class NoopLoadStep extends AbstractLoadStep /////////// // noop. // /////////// + LOG.trace("noop"); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 7037662a..b819760e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -188,7 +188,7 @@ class BulkInsertFullProcessTest extends BaseTest new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), - new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) ))); }; @@ -204,6 +204,7 @@ class BulkInsertFullProcessTest extends BaseTest assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(Map.of(1, "IL"), runProcessOutput.getValue("mappedValueLabels")); assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java new file mode 100644 index 00000000..59a25481 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java @@ -0,0 +1,50 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for NoopLoadStep + *******************************************************************************/ +class NoopLoadStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////////////// + // sorry, just here for coverage... // + ////////////////////////////////////// + new NoopLoadStep().runOnePage(new RunBackendStepInput(), new RunBackendStepOutput()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 4dff56c6..daddaaf2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -91,6 +91,57 @@ class AggregatesTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLong() + { + LongAggregates aggregates = new LongAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + aggregates.add(5L); + assertEquals(1, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(5, aggregates.getMax()); + assertEquals(5, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(10L); + assertEquals(2, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(10, aggregates.getMax()); + assertEquals(15, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("7.5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(15L); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(null); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); + } + + + /******************************************************************************* ** *******************************************************************************/ From 17fc976877bdc1350c1c8425b5b962a8e532b21f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 11:46:24 -0600 Subject: [PATCH 025/209] CE-1955 - Add rowNo to BulkLoadFileRow, set by FileToRowsInterface objects --- .../AbstractIteratorBasedFileToRows.java | 14 +++++++ .../insert/filehandling/CsvFileToRows.java | 2 +- .../filehandling/FileToRowsInterface.java | 6 +++ .../insert/filehandling/XlsxFileToRows.java | 2 +- .../bulk/insert/model/BulkLoadFileRow.java | 42 +++++++++++++++++-- .../filehandling/CsvFileToRowsTest.java | 4 +- .../insert/filehandling/TestFileToRows.java | 2 +- .../filehandling/XlsxFileToRowsTest.java | 4 +- 8 files changed, 65 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java index be80c82a..88437f1b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -36,6 +36,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn private boolean useLast = false; private BulkLoadFileRow last; + int rowNo = 0; /*************************************************************************** @@ -65,6 +66,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn @Override public BulkLoadFileRow next() { + rowNo++; if(iterator == null) { throw new IllegalStateException("Object was not init'ed"); @@ -99,6 +101,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn @Override public void unNext() { + rowNo--; useLast = true; } @@ -122,4 +125,15 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn this.iterator = iterator; } + + + /******************************************************************************* + ** Getter for rowNo + ** + *******************************************************************************/ + @Override + public int getRowNo() + { + return rowNo; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java index 7a58ddef..332c8722 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -92,7 +92,7 @@ public class CsvFileToRows extends AbstractIteratorBasedFileToRows im values[i++] = s; } - return (new BulkLoadFileRow(values)); + return (new BulkLoadFileRow(values, getRowNo())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java index 9d02ee0f..9ab6cce3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -66,6 +66,12 @@ public interface FileToRowsInterface extends AutoCloseable, Iterator Date: Wed, 27 Nov 2024 12:13:15 -0600 Subject: [PATCH 026/209] CE-1955 - Put rows & rowNos in backend details during bulk-load. assert about those. also add tests (and fixes to mapping) for no-header use-cases --- .../insert/mapping/BulkLoadRecordUtils.java | 112 +++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 1 + .../bulk/insert/mapping/TallRowsToRecord.java | 25 ++- ...licitFieldNameSuffixIndexBasedMapping.java | 4 +- .../bulk/insert/model/BulkInsertMapping.java | 34 +++- .../insert/mapping/FlatRowsToRecordTest.java | 65 +++++++ .../insert/mapping/TallRowsToRecordTest.java | 174 ++++++++++++++++++ ...tFieldNameSuffixIndexBasedMappingTest.java | 60 ++++++ 8 files changed, 464 insertions(+), 11 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java new file mode 100644 index 00000000..0c700fe0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Utility methods for working with records in a bulk-load. + ** + ** Originally added for working with backendDetails around the source rows. + *******************************************************************************/ +public class BulkLoadRecordUtils +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, BulkLoadFileRow fileRow) + { + return (addBackendDetailsAboutFileRows(record, new ArrayList<>(List.of(fileRow)))); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, ArrayList fileRows) + { + if(CollectionUtils.nullSafeHasContents(fileRows)) + { + Integer firstRowNo = fileRows.get(0).getRowNo(); + Integer lastRowNo = fileRows.get(fileRows.size() - 1).getRowNo(); + + if(Objects.equals(firstRowNo, lastRowNo)) + { + record.addBackendDetail("rowNos", "Row " + firstRowNo); + } + else + { + record.addBackendDetail("rowNos", "Rows " + firstRowNo + "-" + lastRowNo); + } + } + else + { + record.addBackendDetail("rowNos", "Rows ?"); + } + + record.addBackendDetail("fileRows", fileRows); + return (record); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getRowNosString(QRecord record) + { + return (record.getBackendDetailString("rowNos")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @SuppressWarnings("unchecked") + public static ArrayList getFileRows(QRecord record) + { + return (ArrayList) record.getBackendDetail("fileRows"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static List getFileRowNos(QRecord record) + { + return (getFileRows(record).stream().map(row -> row.getRowNo()).toList()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index d1a5d4c0..243496ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -60,6 +60,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 4587336c..8e0443d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -67,8 +68,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List rs = new ArrayList<>(); - List rowsForCurrentRecord = new ArrayList<>(); - List recordGroupByValues = null; + ArrayList rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; String associationNameChain = ""; @@ -82,6 +83,9 @@ public class TallRowsToRecord implements RowsToRecordInterface groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); } + //////////////////////// + // this is suspect... // + //////////////////////// List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -108,18 +112,19 @@ public class TallRowsToRecord implements RowsToRecordInterface ////////////////////////////////////////////////////////////// // not first, and not a match, so we can finish this record // ////////////////////////////////////////////////////////////// - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); //////////////////////////////////////// // reset these record-specific values // //////////////////////////////////////// rowsForCurrentRecord = new ArrayList<>(); recordGroupByValues = null; - - ////////////////////////////////////////////////////////////////////////////////////////////////////// - // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // - ////////////////////////////////////////////////////////////////////////////////////////////////////// - fileToRowsInterface.unNext(); } } @@ -129,7 +134,8 @@ public class TallRowsToRecord implements RowsToRecordInterface ///////////////////////////////////////////////////////////////////////////////////////////////// if(!rowsForCurrentRecord.isEmpty()) { - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); } ValueMapper.valueMapping(rs, mapping, table); @@ -156,6 +162,7 @@ public class TallRowsToRecord implements RowsToRecordInterface private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException { QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken>() {})); Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index 9c85647e..e79dd2c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -90,7 +90,9 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem ////////////////////////////////////////////////////// // start by building the record with its own fields // ////////////////////////////////////////////////////// - QRecord record = new QRecord(); + QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index 86c19775..d0164b3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -157,7 +157,7 @@ public class BulkInsertMapping implements Serializable } else if(fieldNameToIndexMap != null) { - return (fieldNameToIndexMap); + return (getFieldIndexesForNoHeaderUseCase(table, associationNameChain, wideAssociationIndexes)); } throw (new QException("Mapping was not properly configured.")); @@ -165,6 +165,38 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + private Map getFieldIndexesForNoHeaderUseCase(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Integer index = fieldNameToIndexMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(index != null) + { + rs.put(field.getName(), index); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 7124e946..fd1f70c4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -76,21 +76,86 @@ class FlatRowsToRecordTest extends BaseTest .withHasHeaderRow(true); List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(1, records.size()); assertEquals(List.of("Homer"), getValues(records, "firstName")); assertEquals(List.of("Simpson"), getValues(records, "lastName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 3", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 4", records.get(1).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); assertEquals(List.of("Ned"), getValues(records, "firstName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToColumnIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + // 0, 1, 2, 3, 4 + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2, + "cost", 4 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, null, mapping, 1); + assertEquals(1, records.size()); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 1", records.get(0).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, 2); + assertEquals(2, records.size()); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", records.get(1).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 4", records.get(0).getBackendDetail("rowNos")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 1a0ef2ab..126f500b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -89,6 +89,11 @@ class TallRowsToRecordTest extends BaseTest assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 2-4", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); @@ -96,6 +101,68 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4 + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = null; + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku", 3, + "orderLine.quantity", 4 + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 1-3", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 4-5", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } @@ -338,6 +405,113 @@ class TallRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSingleLine() throws QException + { + Integer defaultStoreId = 101; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPagination() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + 2, Ned, Flanders, BIBLE, 7 + 2, Ned, Flanders, LAWNMOWER, 1 + 3, Bart, Simpson, SKATEBOARD,1 + 3, Bart, Simpson, SLINGSHOT, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 3-4", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(1, records.size()); + order = records.get(0); + assertEquals(3, order.getValueInteger("orderNo")); + assertEquals("Bart", order.getValueString("shipToName")); + assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 6e59c527..38488683 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -86,12 +86,72 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4, 5, 6, 7, 8 + String csv = """ + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = null; + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku,0", 3, + "orderLine.quantity,0", 4, + "orderLine.sku,1", 5, + "orderLine.quantity,1", 6, + "orderLine.sku,2", 7, + "orderLine.quantity,2", 8 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); } From c88fd5b7d41389c293a45aef749f125dae974ab2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 12:35:54 -0600 Subject: [PATCH 027/209] CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors --- .../model/metadata/fields/QFieldType.java | 11 + .../bulk/insert/BulkInsertTransformStep.java | 169 +++++++++++++- .../mapping/BulkLoadValueTypeError.java | 76 +++++++ .../bulk/insert/mapping/ValueMapper.java | 6 +- ...ProcessSummaryWarningsAndErrorsRollup.java | 9 + .../processes/ProcessSummaryAssert.java | 208 ++++++++++++++++++ .../ProcessSummaryLineInterfaceAssert.java | 189 ++++++++++++++++ .../insert/BulkInsertTransformStepTest.java | 190 +++++++++++++--- .../qqq/backend/core/utils/TestUtils.java | 3 +- 9 files changed, 823 insertions(+), 38 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index b0e8f8e7..a8313116 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -100,6 +101,16 @@ public enum QFieldType + /*************************************************************************** + ** + ***************************************************************************/ + public String getMixedCaseLabel() + { + return StringUtils.allCapsToMixedCase(name()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 09d9ca8d..f2d87767 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; @@ -47,16 +48,25 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -66,7 +76,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + .withDoReplaceSingletonCountLinesWithSuffixOnly(false); + + private ListingHash errorToExampleRowValueMap = new ListingHash<>(); + private ListingHash errorToExampleRowsMap = new ListingHash<>(); private Map ukErrorSummaries = new HashMap<>(); private Map associationsToInsertSummaries = new HashMap<>(); @@ -77,6 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; + private final int EXAMPLE_ROW_LIMIT = 10; /******************************************************************************* @@ -118,6 +133,44 @@ public class BulkInsertTransformStep extends AbstractTransformStep // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // /////////////////////////////////////////////////////////////////////////////////////////////////////////////// BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the validationReview widget to render preview records using the table layout, and including the associations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + if(bulkInsertMapping != null) + { + ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); + ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); + ArrayList previewRecordAssociationNames = new ArrayList<>(); + + for(String mappedAssociation : bulkInsertMapping.getMappedAssociations()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst(); + if(association.isPresent()) + { + for(QFieldSection section : table.getSections()) + { + QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName()); + if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType())) + { + Serializable widgetJoinName = widget.getDefaultValues().get("joinName"); + if(Objects.equals(widgetJoinName, association.get().getJoinName())) + { + previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName()); + previewRecordAssociatedWidgetNames.add(widget.getName()); + previewRecordAssociationNames.add(association.get().getName()); + } + } + } + } + } + runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames); + runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames); + runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames); + } } @@ -131,7 +184,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep int recordsInThisPage = runBackendStepInput.getRecords().size(); QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); - // split the records w/o UK errors into those w/ e + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // split the records into 2 lists: those w/ errors (e.g., from the bulk-load mapping), and those that are okay // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// List recordsWithoutAnyErrors = new ArrayList<>(); List recordsWithSomeErrors = new ArrayList<>(); for(QRecord record : runBackendStepInput.getRecords()) @@ -153,16 +208,26 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QRecord record : recordsWithSomeErrors) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof BulkLoadValueTypeError blvte) + { + processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(blvte, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + } + } } } if(recordsWithoutAnyErrors.isEmpty()) { - //////////////////////////////////////////////////////////////////////////////// - // skip th rest of this method if there aren't any records w/o errors in them // - //////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////// + // skip the rest of this method if there aren't any records w/o errors in them // + ///////////////////////////////////////////////////////////////////////////////// this.rowsProcessed += recordsInThisPage; } @@ -248,8 +313,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { @@ -277,6 +345,37 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowValues.size() < EXAMPLE_ROW_LIMIT) + { + rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowMap(String message, QRecord record) + { + List rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowNos.size() < EXAMPLE_ROW_LIMIT) + { + rowNos.add(BulkLoadRecordUtils.getRowNosString(record)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,9 +510,61 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary.addSelfToListIfAnyCount(rs); } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet()) + { + String message = entry.getKey(); + if(errorToExampleRowValueMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); + line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); + } + else if(errorToExampleRowsMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); + line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); + } + } + processSummaryWarningsAndErrorsRollup.addToList(rs); return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + private record RowValue(String row, String value) + { + + /*************************************************************************** + ** + ***************************************************************************/ + public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + return row + " [" + value + "]"; + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java new file mode 100644 index 00000000..a7e6f371 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadValueTypeError extends BadInputStatusMessage +{ + private final String fieldLabel; + private final String fieldName; + private final Serializable value; + private final QFieldType type; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); + this.fieldName = fieldName; + this.value = value; + this.type = type; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index fdf1b6bd..44a37f6a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -70,6 +69,9 @@ public class ValueMapper return; } + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { @@ -102,7 +104,7 @@ public class ValueMapper } catch(Exception e) { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 606a026b..6296f31c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup } + + /******************************************************************************* + ** Getter for errorSummaries + ** + *******************************************************************************/ + public Map getErrorSummaries() + { + return errorSummaries; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java new file mode 100644 index 00000000..a0bf6a36 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's + *******************************************************************************/ +public class ProcessSummaryAssert extends AbstractAssert> +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryAssert(List actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput) + { + List processResults = (List) runProcessOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runProcessOutput.getValue("validationSummary"); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput) + { + List processResults = (List) runBackendStepOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runBackendStepOutput.getValue("validationSummary"); + } + + if(processResults == null) + { + fail("Could not find process results in backend step output."); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryAssert assertThat(List actual) + { + return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasSize(int expectedSize) + { + Assertions.assertThat(actual).hasSize(expectedSize); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status) + { + List foundStatuses = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus())); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasNoLineWithStatus(Status status) + { + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen."); + return (null); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java new file mode 100644 index 00000000..861e21a7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -0,0 +1,189 @@ + +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummaryLine. + *******************************************************************************/ +public class ProcessSummaryLineInterfaceAssert extends AbstractAssert +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual) + { + return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasCount(Integer count) + { + if(actual instanceof ProcessSummaryLine psl) + { + assertEquals(count, psl.getCount(), "Expected count in process summary line"); + } + else + { + failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")"); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasStatus(Status status) + { + assertEquals(status, actual.getStatus(), "Expected status in process summary line"); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).matches(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).contains(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotContain(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + Assertions.assertThat(psl.getBulletsOfText()) + .isNotNull() + .anyMatch(s -> s.contains(substring)); + } + else + { + Assertions.fail("Process Summary Line was not the expected type."); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + if(psl.getBulletsOfText() != null) + { + Assertions.assertThat(psl.getBulletsOfText()) + .noneMatch(s -> s.contains(substring)); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index b87b6799..5a8ae6d3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -22,10 +22,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -38,8 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,9 +95,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -102,13 +110,13 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); @@ -171,9 +179,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -186,20 +194,20 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); - /////////////////////////////////////////////////////// - // assert that all records pass. - /////////////////////////////////////////////////////// + /////////////////////////////////// + // assert that all records pass. // + /////////////////////////////////// assertEquals(7, output.getRecords().size()); } @@ -211,8 +219,8 @@ class BulkInsertTransformStepTest extends BaseTest private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId) { return (record.getValue("uuid").equals(uuid) - && record.getValue("sku").equals(sku) - && record.getValue("storeId").equals(storeId)); + && record.getValue("sku").equals(sku) + && record.getValue("storeId").equals(storeId)); } @@ -220,7 +228,7 @@ class BulkInsertTransformStepTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private QRecord newQRecord(String uuid, String sku, int storeId) + private QRecord newUkTestQRecord(String uuid, String sku, int storeId) { return new QRecord() .withValue("uuid", uuid) @@ -229,4 +237,134 @@ class BulkInsertTransformStepTest extends BaseTest .withValue("name", "Some Item"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValueMappingTypeErrors() throws QException + { + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1)) + .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store")) + .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2)) + .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store")) + )); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // add 102 records with an error in the total field - which is more than the number of examples that should be given // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)) + .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total"))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [A]") + .hasAnyBulletsOfTextContaining("Row 2 [BCD]") + .hasStatus(Status.ERROR) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [47]") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]") + .hasMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]") + .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]") + .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]") + .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101") + .hasStatus(Status.ERROR) + .hasCount(102); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRollupOfValidationErrors() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + String tooLong = ".".repeat(201); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2)) + )); + + ///////////////////////////////////////////////////////////////////// + // add 102 records with no security key - which should be an error // + ///////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("value for Ship To Name is too long") + .hasMessageContaining("Records:") + .doesNotHaveMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("without a value in the field: Store Id") + .hasMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasAnyBulletsOfTextContaining("Row 3") + .hasAnyBulletsOfTextContaining("Row 4") + .doesNotHaveAnyBulletsOfTextContaining("Row 101") + .hasStatus(Status.ERROR) + .hasCount(103); // the 102, plus row 1. + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 89adac13..54502074 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; @@ -662,7 +663,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) - .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() From 9213b8987b9213a63910b95789d5df564ee62072 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 12:36:35 -0600 Subject: [PATCH 028/209] CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors --- .../actions/processes/ProcessSummaryLine.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 98697f80..d4974a07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -53,6 +53,7 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface ////////////////////////////////////////////////////////////////////////// private ArrayList primaryKeys; + private ArrayList bulletsOfText; /******************************************************************************* @@ -497,4 +498,35 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface return (this); } + + /******************************************************************************* + ** Getter for bulletsOfText + *******************************************************************************/ + public ArrayList getBulletsOfText() + { + return (this.bulletsOfText); + } + + + + /******************************************************************************* + ** Setter for bulletsOfText + *******************************************************************************/ + public void setBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + } + + + + /******************************************************************************* + ** Fluent setter for bulletsOfText + *******************************************************************************/ + public ProcessSummaryLine withBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + return (this); + } + + } From 8c6b4e686398d48c08ad9c305227100fe124e634 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 14:59:57 -0600 Subject: [PATCH 029/209] CE-1955 - Add back to processes --- docs/metaData/Processes.adoc | 65 +++++++- .../actions/processes/RunProcessAction.java | 58 ++++++- .../model/actions/processes/ProcessState.java | 66 ++++++++ .../processes/RunBackendStepInput.java | 11 ++ .../actions/processes/RunProcessInput.java | 32 ++++ .../processes/QFrontendStepMetaData.java | 32 ++++ .../processes/RunProcessActionTest.java | 154 ++++++++++++++++-- .../javalin/QJavalinProcessHandler.java | 44 ++++- .../executors/io/ProcessInitOrStepInput.java | 1 + ...cessInitOrStepOrStatusOutputInterface.java | 2 + 10 files changed, 437 insertions(+), 28 deletions(-) diff --git a/docs/metaData/Processes.adoc b/docs/metaData/Processes.adoc index 7985fb73..789bd6d6 100644 --- a/docs/metaData/Processes.adoc +++ b/docs/metaData/Processes.adoc @@ -38,6 +38,13 @@ See {link-permissionRules} for details. *** 1) by a single call to `.withStepList(List)`, which internally adds each step into the `steps` map. *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map. ** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. +* `stepFlow` - *enum, default LINEAR* - specifies the the flow-control logic between steps. Possible values are: +** `LINEAR` - steps are executed in-order, through the `stepList`. +A backend step _can_ customize the `nextStepName` or re-order the `stepList`, if needed. +In a frontend step, a user may be given the option to go _back_ to a previous step as well. +** `STATE_MACHINE` - steps are executed as a Fine State Machine, starting with the first step in `stepList`, +but then proceeding based on the `nextStepName` specified by the previous step. +Thus allowing much more flexible flows. * `schedule` - *<>* - set up the process to run automatically on the specified schedule. See below for details. * `minInputRecords` - *Integer* - #not used...# @@ -67,6 +74,11 @@ For processes with a user-interface, they must define one or more "screens" in t * `formFields` - *List of String* - list of field names used by the screen as form-inputs. * `viewFields` - *List of String* - list of field names used by the screen as visible outputs. * `recordListFields` - *List of String* - list of field names used by the screen in a record listing. +* `format` - *Optional String* - directive for a frontend to use specialized formatting for the display of the process. +** Consult frontend documentation for supported values and their meanings. +* `backStepName` - *Optional String* - For processes using `LINEAR` flow, if this value is given, +then the frontend should offer a control that the user can take (e.g., a button) to move back to an +earlier step in the process. ==== QFrontendComponentMetaData @@ -90,10 +102,13 @@ Expects a process value named `html`. Expects process values named `downloadFileName` and `serverFilePath`. ** `GOOGLE_DRIVE_SELECT_FOLDER` - Special form that presents a UI from Google Drive, where the user can select a folder (e.g., as a target for uploading files in a subsequent backend step). ** `BULK_EDIT_FORM` - For use by the standard QQQ Bulk Edit process. +** `BULK_LOAD_FILE_MAPPING_FORM`, `BULK_LOAD_VALUE_MAPPING_FORM`, or `BULK_LOAD_PROFILE_FORM` - For use by the standard QQQ Bulk Load process. ** `VALIDATION_REVIEW_SCREEN` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays a component prompting the user to run full validation or to skip it, or, if full validation has been ran, then showing the results of that validation. ** `PROCESS_SUMMARY_RESULTS` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays the summary results of running the process. +** `WIDGET` - Render a QQQ Widget. +Requires that `widgetName` be given as a value for the component. ** `RECORD_LIST` - _Deprecated. Showed a grid with a list of records as populated by the process._ * `values` - *Map of String → Serializable* - Key=value pairs, with different expectations based on the component's `type`. @@ -116,6 +131,27 @@ It can be used, however, for example, to cause a `defaultValue` to be applied to It can also be used to cause the process to throw an error, if a field is marked as `isRequired`, but a value is not present. ** `recordListMetaData` - *RecordListMetaData object* - _Not used at this time._ +==== QStateMachineStep + +Processes that use `flow = STATE_MACHINE` should use process steps of type `QStateMachineStep`. + +A common pattern seen in state-machine processes, is that they will present a frontend-step to a user, +then always run a given backend-step in response to that screen which the user submitted. +Inside that backend-step, custom application logic will determine the next state to go to, +which is typically another frontend-step (which would then submit data to its corresponding backend-step, +and continue the FSM). + +To help facilitate this pattern, factory methods exist on `QStateMachineStep`, +for constructing the commonly-expected types of state-machine steps: + +* `frontendThenBackend(name, frontendStep, backendStep)` - for the frontend-then-backend pattern described above. +* `backendOnly(name, backendStep)` - for a state that only has a backend step. +This might be useful as a “reset†step, to run before restarting a state-loop. +* `frontendOnly(name, frontendStep)` - for a state that only has a frontend step, +which would always be followed by another state, which must be specified as the `defaultNextStepName` +on the `QStateMachineStep`. + + ==== BasepullConfiguration A "Basepull" process is a common pattern where an application needs to perform some action on all new (or updated) records from a particular data source. @@ -218,12 +254,10 @@ But for some cases, doing page-level transactions can reduce long-transactions a * `withSchedule(QScheduleMetaData schedule)` - Add a <> to the process. [#_custom_process_flow] -==== Custom Process Flow -As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process -will execute each of its steps in-order, as defined in the `stepList` property. -However, a Backend Step can customize this flow #todo - write more clearly here... - -There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow: +==== How to customize a Linear process flow +As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, +(with `flow = LINEAR`) a process will execute each of its steps in-order, as defined in the `stepList` property. +However, a Backend Step can customize this flow as follows: * `RunBackendStepOutput.setOverrideLastStepName(String stepName)` ** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next. @@ -239,7 +273,7 @@ does need to be found in the new `stepNameList` - otherwise, the framework will for figuring out where to go next. [source,java] -.Example of a defining process that can use a flexible flow: +.Example of a defining process that can use a customized linear flow: ---- // for a case like this, it would be recommended to define all step names in constants: public final static String STEP_START = "start"; @@ -324,4 +358,21 @@ public static class StartStep implements BackendStep } ---- +[#_process_back] +==== How to allow a process to go back + +The simplest option to allow a process to present a "Back" button to users, +thus allowing them to move backward through a process +(e.g., from a review screen back to an earlier input screen), is to set the property `backStepName` +on a `QFrontendStepMetaData`. + +If the step that is executed after the user hits "Back" is a backend step, then within that +step, `runBackendStepInput.getIsStepBack()` will return `true` (but ONLY within that first step after +the user hits "Back"). It may be necessary within individual processes to be aware that the user +has chosen to go back, to reset certain values in the process's state. + +Alternatively, if a frontend step's "Back" behavior needs to be dynamic (e.g., sometimes not available, +or sometimes targeting different steps in the process), then in a backend step that runs before the +frontend step, a call to `runBackendStepOutput.getProcessState().setBackStepName()` can be made, +to customize the value which would otherwise come from the `QFrontendStepMetaData`. diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 481201e3..a595b4e9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -122,6 +122,12 @@ public class RunProcessAction UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); ProcessState processState = primeProcessState(runProcessInput, stateKey, process); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // these should always be clear when we're starting a run - so make sure they haven't leaked from previous // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.clearNextStepName(); + processState.clearBackStepName(); + ///////////////////////////////////////////////////////// // if process is 'basepull' style, keep track of 'now' // ///////////////////////////////////////////////////////// @@ -188,14 +194,35 @@ public class RunProcessAction private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception { String lastStepName = runProcessInput.getStartAfterStep(); + String startAtStep = runProcessInput.getStartAtStep(); while(true) { /////////////////////////////////////////////////////////////////////////////////////////////////////// // always refresh the step list - as any step that runs can modify it (in the process state). // // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + // deal with if we were told, from the input, to start After a step, or start At a step. // /////////////////////////////////////////////////////////////////////////////////////////////////////// - List stepList = getAvailableStepList(processState, process, lastStepName); + List stepList; + if(startAtStep == null) + { + stepList = getAvailableStepList(processState, process, lastStepName, false); + } + else + { + stepList = getAvailableStepList(processState, process, startAtStep, true); + + /////////////////////////////////////////////////////////////////////////////////// + // clear this field - so after we run a step, we'll then loop in last-step mode. // + /////////////////////////////////////////////////////////////////////////////////// + startAtStep = null; + + /////////////////////////////////////////////////////////////////////////////////// + // if we're going to run a backend step now, let it see that this is a step-back // + /////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(true); + } + if(stepList.isEmpty()) { break; @@ -232,7 +259,18 @@ public class RunProcessAction ////////////////////////////////////////////////// throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); } + + //////////////////////////////////////////////////////////////////////////////////////// + // only let this value be set for the original back step - don't let it stick around. // + // if a process wants to keep track of this itself, it can, but in a different slot. // + //////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // in case we broke from the loop above (e.g., by going directly into a frontend step), once again make sure to lower this flag. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } @@ -264,6 +302,12 @@ public class RunProcessAction processFrontendStepFieldDefaultValues(processState, step); processFrontendComponents(processState, step); processState.setNextStepName(step.getName()); + + if(StringUtils.hasContent(step.getBackStepName()) && processState.getBackStepName().isEmpty()) + { + processState.setBackStepName(step.getBackStepName()); + } + return LoopTodo.BREAK; } case SKIP -> @@ -317,6 +361,7 @@ public class RunProcessAction // else run the given lastStepName // ///////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); step = process.getStep(lastStepName); if(step == null) { @@ -398,6 +443,7 @@ public class RunProcessAction // its sub-steps, or, to fall out of the loop and end the process. // ////////////////////////////////////////////////////////////////////////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1); return; } @@ -621,8 +667,10 @@ public class RunProcessAction /******************************************************************************* ** Get the list of steps which are eligible to run. + ** + ** lastStep will be included in the list, or not, based on includeLastStep. *******************************************************************************/ - private List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException + static List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep, boolean includeLastStep) throws QException { if(lastStep == null) { @@ -649,6 +697,10 @@ public class RunProcessAction if(stepName.equals(lastStep)) { foundLastStep = true; + if(includeLastStep) + { + validStepNames.add(stepName); + } } } return (stepNamesToSteps(process, validStepNames)); @@ -660,7 +712,7 @@ public class RunProcessAction /******************************************************************************* ** *******************************************************************************/ - private List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException + private static List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException { List result = new ArrayList<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index ad7a0827..c6d07011 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -40,6 +40,8 @@ public class ProcessState implements Serializable private Map values = new HashMap<>(); private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); + private Optional backStepName = Optional.empty(); + private boolean isStepBack = false; private ProcessMetaDataAdjustment processMetaDataAdjustment = null; @@ -122,6 +124,39 @@ public class ProcessState implements Serializable + /******************************************************************************* + ** Getter for backStepName + ** + *******************************************************************************/ + public Optional getBackStepName() + { + return backStepName; + } + + + + /******************************************************************************* + ** Setter for backStepName + ** + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = Optional.of(backStepName); + } + + + + /******************************************************************************* + ** clear out the value of backStepName (set the Optional to empty) + ** + *******************************************************************************/ + public void clearBackStepName() + { + this.backStepName = Optional.empty(); + } + + + /******************************************************************************* ** Getter for stepList ** @@ -176,4 +211,35 @@ public class ProcessState implements Serializable } + + /******************************************************************************* + ** Getter for isStepBack + *******************************************************************************/ + public boolean getIsStepBack() + { + return (this.isStepBack); + } + + + + /******************************************************************************* + ** Setter for isStepBack + *******************************************************************************/ + public void setIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + } + + + + /******************************************************************************* + ** Fluent setter for isStepBack + *******************************************************************************/ + public ProcessState withIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index bfaad833..81ff1d77 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Accessor for processState's isStepBack attribute + ** + *******************************************************************************/ + public boolean getIsStepBack() + { + return processState.getIsStepBack(); + } + + + /******************************************************************************* ** Accessor for processState - protected, because we generally want to access ** its members through wrapper methods, we think diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java index a099caaf..c9d500f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java @@ -49,6 +49,7 @@ public class RunProcessInput extends AbstractActionInput private ProcessState processState; private FrontendStepBehavior frontendStepBehavior = FrontendStepBehavior.BREAK; private String startAfterStep; + private String startAtStep; private String processUUID; private AsyncJobCallback asyncJobCallback; @@ -451,4 +452,35 @@ public class RunProcessInput extends AbstractActionInput { return asyncJobCallback; } + + /******************************************************************************* + ** Getter for startAtStep + *******************************************************************************/ + public String getStartAtStep() + { + return (this.startAtStep); + } + + + + /******************************************************************************* + ** Setter for startAtStep + *******************************************************************************/ + public void setStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + } + + + + /******************************************************************************* + ** Fluent setter for startAtStep + *******************************************************************************/ + public RunProcessInput withStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + return (this); + } + + } \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java index 0c07043e..514e4fae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java @@ -48,6 +48,7 @@ public class QFrontendStepMetaData extends QStepMetaData private Map formFieldMap; private String format; + private String backStepName; private List helpContents; @@ -436,4 +437,35 @@ public class QFrontendStepMetaData extends QStepMetaData } + + /******************************************************************************* + ** Getter for backStepName + *******************************************************************************/ + public String getBackStepName() + { + return (this.backStepName); + } + + + + /******************************************************************************* + ** Setter for backStepName + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = backStepName; + } + + + + /******************************************************************************* + ** Fluent setter for backStepName + *******************************************************************************/ + public QFrontendStepMetaData withBackStepName(String backStepName) + { + this.backStepName = backStepName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java index 1d989f96..bf7eeb0a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java @@ -23,10 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda; @@ -35,6 +40,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -73,14 +80,14 @@ class RunProcessActionTest extends BaseTest ///////////////////////////////////////////////////////////////// // two-steps - a, points at b; b has no next-step, so it exits // ///////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -109,8 +116,8 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) - .addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) + .withStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) + .withStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -150,7 +157,7 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); @@ -193,14 +200,14 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -238,7 +245,7 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendThenBackend("a", + .withStep(QStateMachineStep.frontendThenBackend("a", new QFrontendStepMetaData().withName("aFrontend"), new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -247,7 +254,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.frontendThenBackend("b", + .withStep(QStateMachineStep.frontendThenBackend("b", new QFrontendStepMetaData().withName("bFrontend"), new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -256,7 +263,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("c"); })))) - .addStep(QStateMachineStep.frontendThenBackend("c", + .withStep(QStateMachineStep.frontendThenBackend("c", new QFrontendStepMetaData().withName("cFrontend"), new QBackendStepMetaData().withName("cBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -265,7 +272,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("d"); })))) - .addStep(QStateMachineStep.frontendOnly("d", + .withStep(QStateMachineStep.frontendOnly("d", new QFrontendStepMetaData().withName("dFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -321,7 +328,132 @@ class RunProcessActionTest extends BaseTest runProcessOutput = new RunProcessAction().execute(input); assertEquals(List.of("in StepA", "in StepB", "in StepC"), log); assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingBack() throws QException + { + AtomicInteger backCount = new AtomicInteger(0); + Map stepRunCounts = new HashMap<>(); + + BackendStep backendStep = (runBackendStepInput, runBackendStepOutput) -> + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // shared backend-step lambda, that will do the same thing for both - but using step name to count how many times each is executed. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + MultiLevelMapHelper.getOrPutAndIncrement(stepRunCounts, runBackendStepInput.getStepName()); + if(runBackendStepInput.getIsStepBack()) + { + backCount.incrementAndGet(); + } + }; + + /////////////////////////////////////////////////////////// + // normal flow here: a -> b -> c // + // but, b can go back to a, as in: a -> b -> a -> b -> c // + /////////////////////////////////////////////////////////// + QProcessMetaData process = new QProcessMetaData().withName("test") + .withStep(new QBackendStepMetaData() + .withName("a") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStep(new QFrontendStepMetaData() + .withName("b") + .withBackStepName("a")) + .withStep(new QBackendStepMetaData() + .withName("c") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStepFlow(ProcessStepFlow.LINEAR); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + /////////////////////////////////////////////////////////// + // start the process - we should be sent to b (frontend) // + /////////////////////////////////////////////////////////// + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(0, backCount.get()); + assertEquals(Map.of("a", 1), stepRunCounts); + + //////////////////////////////////////////////////////////////// + // resume after b, but in back-mode - should end up back at b // + //////////////////////////////////////////////////////////////// + input.setStartAfterStep(null); + input.setStartAtStep("a"); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2), stepRunCounts); + + //////////////////////////////////////////////////////////////////////////// + // resume after b, in regular (forward) mode - should wrap up the process // + //////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("b"); + input.setStartAtStep(null); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isEmpty(); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2, "c", 1), stepRunCounts); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAvailableStepList() throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withStep(new QBackendStepMetaData().withName("A")) + .withStep(new QBackendStepMetaData().withName("B")) + .withStep(new QBackendStepMetaData().withName("C")) + .withStep(new QBackendStepMetaData().withName("D")) + .withStep(new QBackendStepMetaData().withName("E")); + + ProcessState processState = new ProcessState(); + processState.setStepList(process.getStepList().stream().map(s -> s.getName()).toList()); + + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, true)); + + assertStepListNames(List.of("B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", true)); + + assertStepListNames(List.of("D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", false)); + assertStepListNames(List.of("C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "E", false)); + assertStepListNames(List.of("E"), RunProcessAction.getAvailableStepList(processState, process, "E", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", false)); + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", true)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertStepListNames(List expectedNames, List actualSteps) + { + assertEquals(expectedNames, actualSteps.stream().map(s -> s.getName()).toList()); } } \ No newline at end of file diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index ec1c5bbf..1c0bd7b7 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -89,6 +89,7 @@ import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.Context; import io.javalin.http.UploadedFile; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.BooleanUtils; import org.eclipse.jetty.http.HttpStatus; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; import static io.javalin.apibuilder.ApiBuilder.get; @@ -321,7 +322,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processInit(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.BREAK); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.BREAK); } @@ -335,7 +336,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processRun(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.SKIP); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.SKIP); } @@ -343,7 +344,7 @@ public class QJavalinProcessHandler /******************************************************************************* ** *******************************************************************************/ - private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) + private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, String startAtStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) { Map resultForCaller = new HashMap<>(); Exception returningException = null; @@ -357,8 +358,23 @@ public class QJavalinProcessHandler resultForCaller.put("processUUID", processUUID); String processName = context.pathParam("processName"); - LOG.info(startAfterStep == null ? "Initiating process [" + processName + "] [" + processUUID + "]" - : "Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + + if(startAfterStep == null && startAtStep == null) + { + LOG.info("Initiating process [" + processName + "] [" + processUUID + "]"); + } + else if(startAfterStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + } + else if(startAtStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] at step [" + startAtStep + "]"); + } + else + { + LOG.warn("A logical impossibility was reached, regarding the nullity of startAfterStep and startAtStep, at least given how this code was originally written."); + } RunProcessInput runProcessInput = new RunProcessInput(); QJavalinImplementation.setupSession(context, runProcessInput); @@ -367,11 +383,13 @@ public class QJavalinProcessHandler runProcessInput.setFrontendStepBehavior(frontendStepBehavior); runProcessInput.setProcessUUID(processUUID); runProcessInput.setStartAfterStep(startAfterStep); + runProcessInput.setStartAtStep(startAtStep); populateRunProcessRequestWithValuesFromContext(context, runProcessInput); String reportName = ValueUtils.getValueAsString(runProcessInput.getValue("reportName")); QJavalinAccessLogger.logStart(startAfterStep == null ? "processInit" : "processStep", logPair("processName", processName), logPair("processUUID", processUUID), StringUtils.hasContent(startAfterStep) ? logPair("startAfterStep", startAfterStep) : null, + StringUtils.hasContent(startAtStep) ? logPair("startAtStep", startAfterStep) : null, StringUtils.hasContent(reportName) ? logPair("reportName", reportName) : null); ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -460,6 +478,7 @@ public class QJavalinProcessHandler } resultForCaller.put("values", runProcessOutput.getValues()); runProcessOutput.getProcessState().getNextStepName().ifPresent(nextStep -> resultForCaller.put("nextStep", nextStep)); + runProcessOutput.getProcessState().getBackStepName().ifPresent(backStep -> resultForCaller.put("backStep", backStep)); ///////////////////////////////////////////////////////////////////////////////////////////////////////////// // todo - delete after all frontends look for processMetaDataAdjustment instead of updatedFrontendStepList // @@ -660,8 +679,19 @@ public class QJavalinProcessHandler public static void processStep(Context context) { String processUUID = context.pathParam("processUUID"); - String lastStep = context.pathParam("step"); - doProcessInitOrStep(context, processUUID, lastStep, RunProcessInput.FrontendStepBehavior.BREAK); + + String startAfterStep = null; + String startAtStep = null; + if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(context.queryParam("isStepBack")))) + { + startAtStep = context.pathParam("step"); + } + else + { + startAfterStep = context.pathParam("step"); + } + + doProcessInitOrStep(context, processUUID, startAfterStep, startAtStep, RunProcessInput.FrontendStepBehavior.BREAK); } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java index c138b0bc..372b9aac 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java @@ -45,6 +45,7 @@ public class ProcessInitOrStepInput extends AbstractMiddlewareInput ///////////////////////////////////// private String processUUID; private String startAfterStep; + // todo - add (in next version?) startAtStep (for back) private RunProcessInput.FrontendStepBehavior frontendStepBehavior = RunProcessInput.FrontendStepBehavior.BREAK; diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java index f7d0d4a5..aef6ab44 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java @@ -58,6 +58,8 @@ public interface ProcessInitOrStepOrStatusOutputInterface extends AbstractMiddle *******************************************************************************/ void setNextStep(String nextStep); + // todo - add (in next version?) backStep + /******************************************************************************* ** Setter for values *******************************************************************************/ From 61582680f37a906e1c204c579c102740d40cd620 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:01:35 -0600 Subject: [PATCH 030/209] CE-1955 - Add support for back to bulk-load process --- .../qqq/backend/core/instances/QInstanceEnricher.java | 4 ++++ .../bulk/insert/BulkInsertReceiveFileMappingStep.java | 1 + 2 files changed, 5 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 7bf3c306..87be37bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -898,6 +898,7 @@ public class QInstanceEnricher QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() .withName("fileMapping") .withLabel("File Mapping") + .withBackStepName("upload") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() @@ -911,6 +912,7 @@ public class QInstanceEnricher QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() .withName("valueMapping") .withLabel("Value Mapping") + .withBackStepName("prepareFileMapping") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() @@ -934,7 +936,9 @@ public class QInstanceEnricher // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // ////////////////////////////////////////////////////////////////////////////////////////// process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withBackStepName("prepareFileMapping") .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index bc6ae239..afe3c9f1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -190,6 +190,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep // it's also where the value-mapping loop of steps points. // // and, this will actually be the default (e.g., the step after this one). // ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); } else { From 8ea16db1fc6d7f3480c607c3ee654ae045ce0417 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:10:24 -0600 Subject: [PATCH 031/209] CE-1955 - Checkstyle --- .../implementations/bulk/insert/BulkInsertTransformStep.java | 2 +- .../actions/processes/ProcessSummaryLineInterfaceAssert.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f2d87767..73d35070 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -91,7 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; - private final int EXAMPLE_ROW_LIMIT = 10; + private static final int EXAMPLE_ROW_LIMIT = 10; /******************************************************************************* diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java index 861e21a7..60d4561c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -1,4 +1,3 @@ - /* * QQQ - Low-code Application Framework for Engineers. * Copyright (C) 2021-2024. Kingsrook, LLC From a439bffc698a51fd50f1308fe6e90b06383be75a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:34:37 -0600 Subject: [PATCH 032/209] Add support for OpenAPIEnumSubSet --- .../javalin/schemabuilder/SchemaBuilder.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java index 9b0de98c..a97a7019 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java @@ -38,6 +38,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIHasAdditionalProperties; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIIncludeProperties; @@ -123,7 +124,25 @@ public class SchemaBuilder if(c.isEnum()) { schema.withType(Type.STRING); - schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + + if(element.isAnnotationPresent(OpenAPIEnumSubSet.class)) + { + try + { + OpenAPIEnumSubSet enumSubSetAnnotation = element.getAnnotation(OpenAPIEnumSubSet.class); + Class> enumSubSetClass = enumSubSetAnnotation.value(); + OpenAPIEnumSubSet.EnumSubSet enumSubSetContainer = enumSubSetClass.getConstructor().newInstance(); + schema.withEnumValues(enumSubSetContainer.getSubSet().stream().map(e -> String.valueOf(e)).collect(Collectors.toList())); + } + catch(Exception e) + { + throw new QRuntimeException("Error processing OpenAPIEnumSubSet on element: " + element, e); + } + } + else + { + schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + } } else if(c.equals(String.class)) { From 53ca77cde691a7e17abb849f67d2133dfb5e9a84 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:35:10 -0600 Subject: [PATCH 033/209] CE-1955 Update to use an enum-subset (excluding new BULK_LOAD components) --- .../components/FrontendComponent.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java index e28af1ce..f6c15758 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java @@ -23,11 +23,13 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.io.Serializable; +import java.util.EnumSet; import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIMapKnownEntries; @@ -63,10 +65,39 @@ public class FrontendComponent implements ToSchema + /*************************************************************************** + ** + ***************************************************************************/ + public static class QComponentTypeSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(QComponentType.class); + subSet.remove(QComponentType.BULK_LOAD_FILE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_PROFILE_FORM); + QComponentTypeSubSet.subSet = subSet; + } + + return(subSet); + } + } + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("The type of this component. e.g., what kind of UI element(s) should be presented to the user.") + @OpenAPIEnumSubSet(QComponentTypeSubSet.class) public QComponentType getType() { return (this.wrapped.getType()); From 8ec6ccd691462d7e15ac06f57e22e417dc0b4aa0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:36:36 -0600 Subject: [PATCH 034/209] CE-1955 added an icon for bulk-load process in example (since it has one now) --- .../src/main/resources/openapi/v1/openapi.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index f89ba784..beda0fc6 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -1483,6 +1483,8 @@ paths: processes: person.bulkInsert: hasPermission: true + icon: + name: "library_add" isHidden: true label: "Person Bulk Insert" name: "person.bulkInsert" From 0e93b90270eed8365d679878f6604aa25f5ddd54 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:50:05 -0600 Subject: [PATCH 035/209] CE-1955 Add mapping and validation of possible-values; refactor error classes some for rollup possible value errors --- .../bulk/insert/BulkInsertTransformStep.java | 18 +- .../AbstractBulkLoadRollableValueError.java | 55 ++++ .../mapping/BulkLoadPossibleValueError.java | 72 ++++++ .../insert/mapping/BulkLoadValueMapper.java | 238 ++++++++++++++++++ .../mapping/BulkLoadValueTypeError.java | 7 +- .../bulk/insert/mapping/FlatRowsToRecord.java | 2 +- .../bulk/insert/mapping/TallRowsToRecord.java | 2 +- .../bulk/insert/mapping/ValueMapper.java | 134 ---------- ...licitFieldNameSuffixIndexBasedMapping.java | 2 +- .../WideRowsToRecordWithSpreadMapping.java | 2 +- ...Test.java => BulkLoadValueMapperTest.java} | 4 +- .../insert/mapping/FlatRowsToRecordTest.java | 45 ++++ 12 files changed, 428 insertions(+), 153 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{ValueMapperTest.java => BulkLoadValueMapperTest.java} (96%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 73d35070..ad531967 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -56,8 +56,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; @@ -210,10 +210,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QErrorMessage error : record.getErrors()) { - if(error instanceof BulkLoadValueTypeError blvte) + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) { - processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); - addToErrorToExampleRowValueMap(blvte, record); + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); } else { @@ -348,14 +348,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + String message = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey(); List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); if(rowValues.size() < EXAMPLE_ROW_LIMIT) { - rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + rowValues.add(new RowValue(bulkLoadRollableValueError, record)); } } @@ -550,9 +550,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java new file mode 100644 index 00000000..74082fd7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AbstractBulkLoadRollableValueError(String message) + { + super(message); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract String getMessageToUseAsProcessSummaryRollupKey(); + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract Serializable getValue(); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java new file mode 100644 index 00000000..199d775f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError +{ + private final String fieldLabel; + private final Serializable value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option"); + this.value = value; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Unrecognized value for field [" + fieldLabel + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + @Override + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java new file mode 100644 index 00000000..cc44e027 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -0,0 +1,238 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadValueMapper +{ + private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException + { + valueMapping(records, mapping, table, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + + Map> possibleValueToRecordMap = new HashMap<>(); + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry valueEntry : record.getValues().entrySet()) + { + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) + { + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } + } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null && !"".equals(value)) + { + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>()); + fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record); + } + else + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); + } + } + } + + record.setValue(field.getName(), value); + } + + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + } + + ////////////////////////////////////////// + // look up and validate possible values // + ////////////////////////////////////////// + for(Map.Entry> entry : possibleValueToRecordMap.entrySet()) + { + String fieldName = entry.getKey(); + QFieldMetaData field = table.getField(fieldName); + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName); + + handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException + { + Set values = fieldPossibleValueToRecordMap.keySet(); + Map> valuesFound = new HashMap<>(); + Set valuesNotFound = new HashSet<>(values); + + //////////////////////////////////////////////////////// + // do a search, trying to use all given values as ids // + //////////////////////////////////////////////////////// + SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList idList = new ArrayList<>(values); + searchPossibleValueSourceInput.setIdList(idList); + searchPossibleValueSourceInput.setLimit(values.size()); + LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // for each possible value found, remove it from the set of ones not-found, and store it as a hit // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); + valuesFound.put(valueAsString, possibleValue); + valuesNotFound.remove(valueAsString); + } + + /////////////////////////////////////////////////////////////////////////// + // if there are any that weren't found, try to look them up now by label // + /////////////////////////////////////////////////////////////////////////// + if(!valuesNotFound.isEmpty()) + { + searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList labelList = new ArrayList<>(valuesNotFound); + searchPossibleValueSourceInput.setLabelList(labelList); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); + + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0))); + searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + valuesFound.put(possibleValue.getLabel(), possibleValue); + valuesNotFound.remove(possibleValue.getLabel()); + } + } + + //////////////////////////////////////////////////////////////////////////////// + // for each record now, either set a usable value (e.g., a PV.id) or an error // + //////////////////////////////////////////////////////////////////////////////// + for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) + { + String value = entry.getKey(); + for(QRecord record : entry.getValue()) + { + if(valuesFound.containsKey(value)) + { + record.setValue(field.getName(), valuesFound.get(value).getId()); + } + else + { + record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel())); + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java index a7e6f371..6bee0449 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -24,17 +24,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; /******************************************************************************* ** Specialized error for records, for bulk-load use-cases, where we want to ** report back info to the user about the field & value. *******************************************************************************/ -public class BulkLoadValueTypeError extends BadInputStatusMessage +public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError { private final String fieldLabel; - private final String fieldName; private final Serializable value; private final QFieldType type; @@ -47,7 +45,6 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) { super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); - this.fieldName = fieldName; this.value = value; this.type = type; this.fieldLabel = fieldLabel; @@ -58,6 +55,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage /*************************************************************************** ** ***************************************************************************/ + @Override public String getMessageToUseAsProcessSummaryRollupKey() { return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); @@ -69,6 +67,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage ** Getter for value ** *******************************************************************************/ + @Override public Serializable getValue() { return value; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 243496ef..310fd8db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -70,7 +70,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 8e0443d3..84ab1fa7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -138,7 +138,7 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java deleted file mode 100644 index 44a37f6a..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class ValueMapper -{ - private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException - { - valueMapping(records, mapping, table, null); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException - { - if(CollectionUtils.nullSafeIsEmpty(records)) - { - return; - } - - String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; - String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; - - Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); - for(QRecord record : records) - { - for(Map.Entry valueEntry : record.getValues().entrySet()) - { - QFieldMetaData field = table.getField(valueEntry.getKey()); - Serializable value = valueEntry.getValue(); - - /////////////////// - // value mappin' // - /////////////////// - if(mappingForTable.containsKey(field.getName()) && value != null) - { - Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); - if(mappedValue != null) - { - value = mappedValue; - } - } - - ///////////////////// - // type convertin' // - ///////////////////// - if(value != null) - { - QFieldType type = field.getType(); - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); - } - } - - record.setValue(field.getName(), value); - } - - ////////////////////////////////////// - // recursively process associations // - ////////////////////////////////////// - for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) - { - String associationName = entry.getKey(); - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isPresent()) - { - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); - } - else - { - throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); - } - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index e79dd2c2..c4231ea7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -75,7 +75,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 7b215d2e..e39e95c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -86,7 +86,7 @@ public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java similarity index 96% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 4348cb2b..687f1f62 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -41,7 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* ** Unit test for ValueMapper *******************************************************************************/ -class ValueMapperTest extends BaseTest +class BulkLoadValueMapperTest extends BaseTest { /******************************************************************************* @@ -98,7 +98,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index fd1f70c4..5237fcf5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -102,6 +103,8 @@ class FlatRowsToRecordTest extends BaseTest assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); } + + /******************************************************************************* ** *******************************************************************************/ @@ -205,6 +208,48 @@ class FlatRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueMappings() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Home State" }, + new Serializable[] { 1, "Homer", "Simpson", 1 }, + new Serializable[] { 2, "Marge", "Simpson", "MO" }, + new Serializable[] { 3, "Bart", "Simpson", null }, + new Serializable[] { 4, "Ned", "Flanders", "Not a state" }, + new Serializable[] { 5, "Mr.", "Burns", 5 } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "homeStateId", "Home State" + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(5, records.size()); + assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName")); + assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId")); + + assertThat(records.get(0).getErrors()).isNullOrEmpty(); + assertThat(records.get(1).getErrors()).isNullOrEmpty(); + assertThat(records.get(2).getErrors()).isNullOrEmpty(); + assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + } + + + /*************************************************************************** ** ***************************************************************************/ From b055913fc8aa1a08456104b9a19cdc8ed3bb42b0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:56:28 -0600 Subject: [PATCH 036/209] CE-1955 Initial checkin --- .../BulkInsertPrepareFileUploadStep.java | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java new file mode 100644 index 00000000..c021dcb1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -0,0 +1,292 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** step before the upload screen, to prepare dynamic help-text for user. + *******************************************************************************/ +public class BulkInsertPrepareFileUploadStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + List requiredFields = new ArrayList<>(); + List additionalFields = new ArrayList<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + if(field.getIsRequired()) + { + requiredFields.add(field); + } + else + { + additionalFields.add(field); + } + } + + StringBuilder html; + String childTableLabels = ""; + + StringBuilder tallCSV = new StringBuilder(); + StringBuilder wideCSV = new StringBuilder(); + StringBuilder flatCSV = new StringBuilder(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // potentially this could be a parameter - for now, hard-code false, but keep the code around that did this // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean listFieldsInHelpText = false; + + if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + { + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to + insert in the ${tableLabel} table.


+ +

Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional (though encouraged) whether you include + a header row in your file. For Excel files, only the first sheet in the workbook will be used.


+ """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + html.append(""" + Template: ${tableLabel}.csv"""); + } + else + { + html.append(""" +

You can download a template file to see the full list of available fields: + ${tableLabel}.csv +

+ """); + } + } + else + { + childTableLabels = StringUtils.joinWithCommasAndAnd(tableStructure.getAssociations().stream().map(a -> a.getLabel()).toList()) + " table" + StringUtils.plural(table.getAssociations()); + + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file. Your file can be in one of three layouts:

+ ${openUL} +

  • Flat: Each row in the file will create one record in the ${tableLabel} table.
    • +
  • Wide: Each row in the file will create one record in the ${tableLabel} table, + and optionally one or more records in the ${childTableLabels}, by supplying additional columns + for each sub-record that you want to create.
    • +
  • Tall: Rows with matching values in the fields being used for the ${tableLabel} + table will be used to create one ${tableLabel} record. One or more records will also be built + in the ${childTableLabels} by providing unique values in each row for the sub-records.
    • +
    + +

    Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional (though encouraged) whether you include + a header row in your file. For Excel files, only the first sheet in the workbook will be used.


    + """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + } + + addCsvFields(tallCSV, requiredFields, additionalFields); + addCsvFields(wideCSV, requiredFields, additionalFields); + + for(BulkLoadTableStructure association : tableStructure.getAssociations()) + { + if(listFieldsInHelpText) + { + html.append(""" +

    You can also add values for these ${childLabel} fields:

    + """.replace("${childLabel}", association.getLabel())); + appendFieldsAsUlToHtml(html, association.getFields()); + } + + addCsvFields(tallCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", ""); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 1"); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 2"); + } + + finishCSV(tallCSV); + finishCSV(wideCSV); + + if(listFieldsInHelpText) + { + html.append(""" + Templates: ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv + """); + } + else + { + html.append(""" +

    You can download a template file to see the full list of available fields: + ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv +

    + """); + } + } + + html.insert(0, """ +
    + File Upload Instructions +
    + """); + html.append("
    "); + + addCsvFields(flatCSV, requiredFields, additionalFields); + finishCSV(flatCSV); + + String htmlString = html.toString() + .replace("${tableLabel}", table.getLabel()) + .replace("${childTableLabels}", childTableLabels) + .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${tallCSV}", Base64.getEncoder().encodeToString(tallCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${wideCSV}", Base64.getEncoder().encodeToString(wideCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${openUL}", "
      "); + + runBackendStepOutput.addValue("upload.html", htmlString); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void finishCSV(StringBuilder flatCSV) + { + flatCSV.deleteCharAt(flatCSV.length() - 1); + flatCSV.append("\n"); + flatCSV.append(flatCSV.toString().replaceAll("[^,]", "")); + flatCSV.append("\n"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields) + { + addCsvFields(csv, requiredFields, additionalFields, "", ""); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields, String fieldLabelPrefix, String fieldLabelSuffix) + { + for(QFieldMetaData field : requiredFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + + for(QFieldMetaData field : additionalFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendTableRequiredAndAdditionalFields(StringBuilder html, List requiredFields, List additionalFields) + { + if(!requiredFields.isEmpty()) + { + html.append(""" +

      You will be required to supply values (either in a column in the file, or by + choosing a default value on the next screen) for the following ${tableLabel} fields:

      + """); + appendFieldsAsUlToHtml(html, requiredFields); + } + + if(!additionalFields.isEmpty()) + { + if(requiredFields.isEmpty()) + { + html.append(""" +

      You can supply values (either in a column in the file, or by choosing a + default value on the next screen) for the following ${tableLabel} fields:

      + """); + } + else + { + html.append("

      You can also add values for these fields:

      "); + } + + appendFieldsAsUlToHtml(html, additionalFields); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendFieldsAsUlToHtml(StringBuilder html, List additionalFields) + { + html.append("${openUL}"); + for(QFieldMetaData field : additionalFields) + { + html.append("
    • ").append(field.getLabel()).append("
      • "); + } + html.append("

    "); + } + +} From b0cc93cbb79a4dc05263141f1c784990d6600d0d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:56:45 -0600 Subject: [PATCH 037/209] CE-1955 Add FILE_UPLOAD adornment type --- .../model/metadata/fields/AdornmentType.java | 62 +++++++++++++++++++ .../model/metadata/fields/FieldAdornment.java | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index ba78ceef..1b1941f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -41,6 +41,7 @@ public enum AdornmentType RENDER_HTML, REVEAL, FILE_DOWNLOAD, + FILE_UPLOAD, ERROR; ////////////////////////////////////////////////////////////////////////// // keep these values in sync with AdornmentType.ts in qqq-frontend-core // @@ -164,4 +165,65 @@ public enum AdornmentType } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class FileUploadAdornment + { + public static String FORMAT = "format"; + public static String WIDTH = "width"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static FieldAdornment newFieldAdornment() + { + return (new FieldAdornment(AdornmentType.FILE_UPLOAD)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatDragAndDrop() + { + return (Pair.of(FORMAT, "dragAndDrop")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatButton() + { + return (Pair.of(FORMAT, "button")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthFull() + { + return (Pair.of(WIDTH, "full")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthHalf() + { + return (Pair.of(WIDTH, "half")); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java index b8aa82c5..74cc9db7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java @@ -177,7 +177,7 @@ public class FieldAdornment ** Fluent setter for values ** *******************************************************************************/ - public FieldAdornment withValue(Pair value) + public FieldAdornment withValue(Pair value) { return (withValue(value.getA(), value.getB())); } From 86f8e24d5fc24395d32bf19adc9d07d6083e861d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:57:10 -0600 Subject: [PATCH 038/209] CE-1955 Handle back better; put suggested mapping profile into process value under a dedicated key --- .../BulkInsertPrepareFileMappingStep.java | 19 ++++++++++++++++--- .../BulkInsertPrepareValueMappingStep.java | 6 ++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index f939df3a..80cb49d1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapp import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -59,9 +60,20 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); runBackendStepOutput.addValue("tableStructure", tableStructure); - @SuppressWarnings("unchecked") - List headerValues = (List) runBackendStepOutput.getValue("headerValues"); - buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + boolean needSuggestedMapping = true; + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + needSuggestedMapping = false; + + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + } + + if(needSuggestedMapping) + { + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } } @@ -74,6 +86,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 19e696e5..d24856c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -75,6 +76,11 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep { try { + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + } + ///////////////////////////////////////////////////////////// // prep the frontend for what field we're going to map now // ///////////////////////////////////////////////////////////// From 7cd3105ee6a1a2e52ed96215c4dba54764986b81 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:59:05 -0600 Subject: [PATCH 039/209] CE-1955 Add search-by labels - e.g., exact-matches on a single-field used as the PVS's label... definitely not perfect, but a passable first-version for bulk-load to do PVS mapping --- .../SearchPossibleValueSourceAction.java | 130 ++++++++++++++---- .../SearchPossibleValueSourceInput.java | 34 ++++- .../SearchPossibleValueSourceOutput.java | 32 +++++ .../SearchPossibleValueSourceActionTest.java | 71 ++++++++++ 4 files changed, 242 insertions(+), 25 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 9cb17a0c..97bcae1c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -26,8 +26,12 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -50,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; -import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -61,6 +65,9 @@ public class SearchPossibleValueSourceAction { private static final QLogger LOG = QLogger.getLogger(SearchPossibleValueSourceAction.class); + private static final Set warnedAboutUnexpectedValueField = Collections.synchronizedSet(new HashSet<>()); + private static final Set warnedAboutUnexpectedNoOfFieldsToSearchByLabel = Collections.synchronizedSet(new HashSet<>()); + private QPossibleValueTranslator possibleValueTranslator; @@ -110,6 +117,7 @@ public class SearchPossibleValueSourceAction List matchingIds = new ArrayList<>(); List inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList()); + Set labels = null; for(QPossibleValue possibleValue : possibleValueSource.getEnumValues()) { @@ -122,12 +130,24 @@ public class SearchPossibleValueSourceAction match = true; } } + else if(input.getLabelList() != null) + { + if(labels == null) + { + labels = input.getLabelList().stream().filter(Objects::nonNull).map(l -> l.toLowerCase()).collect(Collectors.toSet()); + } + + if(labels.contains(possibleValue.getLabel().toLowerCase())) + { + match = true; + } + } else { if(StringUtils.hasContent(input.getSearchTerm())) { match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase()) - || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); + || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); } else { @@ -168,21 +188,37 @@ public class SearchPossibleValueSourceAction Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId(); - if(anIdFromTheEnum instanceof Integer) + for(Serializable inputId : inputIdList) { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsInteger(id))); - } - else if(anIdFromTheEnum instanceof String) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsString(id))); - } - else if(anIdFromTheEnum instanceof Boolean) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsBoolean(id))); - } - else - { - LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + Object properlyTypedId = null; + try + { + if(anIdFromTheEnum instanceof Integer) + { + properlyTypedId = ValueUtils.getValueAsInteger(inputId); + } + else if(anIdFromTheEnum instanceof String) + { + properlyTypedId = ValueUtils.getValueAsString(inputId); + } + else if(anIdFromTheEnum instanceof Boolean) + { + properlyTypedId = ValueUtils.getValueAsBoolean(inputId); + } + else + { + LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + } + } + catch(Exception e) + { + LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId)); + } + + if (properlyTypedId != null) + { + rs.add(properlyTypedId); + } } return (rs); @@ -209,6 +245,53 @@ public class SearchPossibleValueSourceAction { queryFilter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getIdList())); } + else if(input.getLabelList() != null) + { + List fieldNames = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the 'value fields' will either be 'id' or 'label' (which means, use the fields from the tableMetaData's label fields) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String valueField : possibleValueSource.getValueFields()) + { + if("id".equals(valueField)) + { + fieldNames.add(table.getPrimaryKeyField()); + } + else if("label".equals(valueField)) + { + if(table.getRecordLabelFields() != null) + { + fieldNames.addAll(table.getRecordLabelFields()); + } + } + else + { + String message = "Unexpected valueField defined in possibleValueSource when searching possibleValueSource by label (required: 'id' or 'label')"; + if(!warnedAboutUnexpectedValueField.contains(possibleValueSource.getName())) + { + LOG.warn(message, logPair("valueField", valueField), logPair("possibleValueSource", possibleValueSource.getName())); + warnedAboutUnexpectedValueField.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } + + if(fieldNames.size() == 1) + { + queryFilter.addCriteria(new QFilterCriteria(fieldNames.get(0), QCriteriaOperator.IN, input.getLabelList())); + } + else + { + String message = "Unexpected number of fields found for searching possibleValueSource by label (required: 1, found: " + fieldNames.size() + ")"; + if(!warnedAboutUnexpectedNoOfFieldsToSearchByLabel.contains(possibleValueSource.getName())) + { + LOG.warn(message); + warnedAboutUnexpectedNoOfFieldsToSearchByLabel.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } else { String searchTerm = input.getSearchTerm(); @@ -269,8 +352,8 @@ public class SearchPossibleValueSourceAction queryFilter = input.getDefaultQueryFilter(); } - // todo - skip & limit as params - queryFilter.setLimit(250); + queryFilter.setLimit(input.getLimit()); + queryFilter.setSkip(input.getSkip()); queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); @@ -288,7 +371,7 @@ public class SearchPossibleValueSourceAction fieldName = table.getPrimaryKeyField(); } - List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); + List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids); output.setResults(qPossibleValues); @@ -301,7 +384,7 @@ public class SearchPossibleValueSourceAction ** *******************************************************************************/ @SuppressWarnings({ "rawtypes", "unchecked" }) - private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) + private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) throws QException { try { @@ -314,11 +397,10 @@ public class SearchPossibleValueSourceAction } catch(Exception e) { - // LOG.warn("Error sending [" + value + "] for field [" + field + "] through custom code for PVS [" + field.getPossibleValueSourceName() + "]", e); + String message = "Error sending searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; + LOG.warn(message, e); + throw (new QException(message)); } - - throw new NotImplementedException("Not impleemnted"); - // return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java index 65b7c2bc..8976a176 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java @@ -38,9 +38,10 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen private QQueryFilter defaultQueryFilter; private String searchTerm; private List idList; + private List labelList; private Integer skip = 0; - private Integer limit = 100; + private Integer limit = 250; @@ -281,4 +282,35 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen this.limit = limit; return (this); } + + + /******************************************************************************* + ** Getter for labelList + *******************************************************************************/ + public List getLabelList() + { + return (this.labelList); + } + + + + /******************************************************************************* + ** Setter for labelList + *******************************************************************************/ + public void setLabelList(List labelList) + { + this.labelList = labelList; + } + + + + /******************************************************************************* + ** Fluent setter for labelList + *******************************************************************************/ + public SearchPossibleValueSourceInput withLabelList(List labelList) + { + this.labelList = labelList; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java index e7186614..e60ed262 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java @@ -35,6 +35,7 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput { private List> results = new ArrayList<>(); + private String warning; /******************************************************************************* @@ -88,4 +89,35 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput return (this); } + + /******************************************************************************* + ** Getter for warning + *******************************************************************************/ + public String getWarning() + { + return (this.warning); + } + + + + /******************************************************************************* + ** Setter for warning + *******************************************************************************/ + public void setWarning(String warning) + { + this.warning = warning; + } + + + + /******************************************************************************* + ** Fluent setter for warning + *******************************************************************************/ + public SearchPossibleValueSourceOutput withWarning(String warning) + { + this.warning = warning; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java index ee2bb104..d71f6335 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java @@ -217,6 +217,63 @@ class SearchPossibleValueSourceActionTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_tableByLabels() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Square", "Circle"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("Square")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(3) && pv.getLabel().equals("Circle")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("notFound"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_enumByLabel() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("IL", "MO", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Il", "mo", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("not-found"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + } + + /******************************************************************************* ** @@ -414,4 +471,18 @@ class SearchPossibleValueSourceActionTest extends BaseTest return output; } + + + /******************************************************************************* + ** + *******************************************************************************/ + private SearchPossibleValueSourceOutput getSearchPossibleValueSourceOutputByLabels(List labels, String possibleValueSourceName) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setLabelList(labels); + input.setPossibleValueSourceName(possibleValueSourceName); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + return output; + } + } \ No newline at end of file From a7247b59700c950141042674139d911ef36f8fb4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:59:48 -0600 Subject: [PATCH 040/209] CE-1955 Add method resetValidationFields - to help processes that go 'back' --- .../StreamedETLWithFrontendProcess.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 8d3e8287..d05bb12c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -193,6 +194,17 @@ public class StreamedETLWithFrontendProcess } + /*************************************************************************** + ** useful for a process step to call upon 'back' + ***************************************************************************/ + public static void resetValidationFields(RunBackendStepInput runBackendStepInput) + { + runBackendStepInput.addValue(FIELD_DO_FULL_VALIDATION, null); + runBackendStepInput.addValue(FIELD_VALIDATION_SUMMARY, null); + runBackendStepInput.addValue(FIELD_PROCESS_SUMMARY, null); + } + + /******************************************************************************* ** From 11db820196db273ccf4d8d7406453a2d5ad5057b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:03:02 -0600 Subject: [PATCH 041/209] CE-1955 Bulk insert updates: Add prepareFileUploadStep; make theFile field use drag&drop adornment --- .../core/instances/QInstanceEnricher.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 87be37bc..da9c9a91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType.FileUploadAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -77,6 +78,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileUploadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; @@ -881,14 +883,20 @@ public class QInstanceEnricher .map(QFieldMetaData::getLabel) .collect(Collectors.joining(", ")); + QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData() + .withName("prepareFileUpload") + .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class)); + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() .withName("upload") .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV or Excel (.xlsx) file with the following columns:\n" + fieldsForHelpText)) + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB) + .withFieldAdornment(FileUploadAdornment.newFieldAdornment() + .withValue(FileUploadAdornment.formatDragAndDrop()) + .withValue(FileUploadAdornment.widthFull())) + .withLabel(table.getLabel() + " File") + .withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() @@ -920,6 +928,7 @@ public class QInstanceEnricher .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); int i = 0; + process.addStep(i++, prepareFileUploadStep); process.addStep(i++, uploadScreen); process.addStep(i++, prepareFileMappingStep); From 21069e231086659bf22f6fb916d1942404bba1b9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:10:00 -0600 Subject: [PATCH 042/209] CE-1955 Checkstyle! --- .../bulk/insert/BulkInsertPrepareFileUploadStep.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index c021dcb1..5fe61f41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -161,10 +161,10 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep if(listFieldsInHelpText) { html.append(""" - Templates: ${tableLabel} - Flat.csv - | ${tableLabel} - Tall.csv - | ${tableLabel} - Wide.csv - """); + Templates: ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv + """); } else { From 7e3592628a4bcf273c86e13c76a996f36e8024b3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:27:35 -0600 Subject: [PATCH 043/209] CE-1955 Don't put empty-string values into records (in setValueOrDefault) - in general, we might get an empty-string from a file, but let's treat it like a non-value, null. --- .../bulk/insert/mapping/RowsToRecordInterface.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 740781c2..6d10b0bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -92,7 +92,7 @@ public interface RowsToRecordInterface value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); } - if(value != null) + if(value != null && !"".equals(value)) { record.setValue(fieldName, value); } From 2bf12158be0e8b7691495fbf788ddee46774c912 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:27:50 -0600 Subject: [PATCH 044/209] CE-1955 Fix to set tableName before preUpload step --- .../implementations/bulk/insert/BulkInsertFullProcessTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index b819760e..2ab31159 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -132,6 +132,7 @@ class BulkInsertFullProcessTest extends BaseTest ///////////////////////////////////////////////////////// RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setProcessName(processName); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); String processUUID = runProcessOutput.getProcessUUID(); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); @@ -155,7 +156,6 @@ class BulkInsertFullProcessTest extends BaseTest ////////////////////////// runProcessInput.setProcessUUID(processUUID); runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); runProcessOutput = new RunProcessAction().execute(runProcessInput); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); From 21aeac2def4debbf4440f8b851849b2cf173205d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:51:44 -0600 Subject: [PATCH 045/209] CE-1955 Switch fieldMetaData to use a type from in here for FieldAdornment, to include some better docs, but also to exclude new FILE_UPLOAD adornment type enum value --- .../responses/components/FieldAdornment.java | 115 ++++++++++++++++++ .../responses/components/FieldMetaData.java | 18 ++- .../main/resources/openapi/v1/openapi.yaml | 51 ++++---- 3 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java new file mode 100644 index 00000000..2f59ee63 --- /dev/null +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; + + +import java.io.Serializable; +import java.util.EnumSet; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FieldAdornment implements ToSchema +{ + @OpenAPIExclude() + private com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment(com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped) + { + this.wrapped = wrapped; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment() + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class FieldAdornmentSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(AdornmentType.class); + subSet.remove(AdornmentType.FILE_UPLOAD); // todo - remove for next version! + FieldAdornmentSubSet.subSet = subSet; + } + + return (subSet); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Type of this adornment") + @OpenAPIEnumSubSet(FieldAdornmentSubSet.class) + public AdornmentType getType() + { + return (this.wrapped == null || this.wrapped.getType() == null ? null : this.wrapped.getType()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Values associated with this adornment. Keys and the meanings of their values will differ by type.") + public Map getValues() + { + return (this.wrapped.getValues()); + } + +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java index 5c4505ac..b8a75d5f 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java @@ -23,7 +23,6 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.util.List; -import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; @@ -61,6 +60,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -82,6 +82,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -92,6 +93,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +104,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -112,6 +115,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -122,6 +126,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -143,6 +148,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -153,6 +159,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -166,6 +173,8 @@ public class FieldMetaData implements ToSchema // todo - inline PVS + + /*************************************************************************** ** ***************************************************************************/ @@ -177,14 +186,17 @@ public class FieldMetaData implements ToSchema // todo behaviors? + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("Special UI dressings to add to the field.") - @OpenAPIListItems(value = FieldAdornment.class) // todo! + @OpenAPIListItems(value = FieldAdornment.class, useRef = true) public List getAdornments() { - return (this.wrapped.getAdornments()); + return (this.wrapped.getAdornments() == null ? null : this.wrapped.getAdornments().stream().map(a -> new FieldAdornment(a)).toList()); } // todo help content diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index beda0fc6..dd802596 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -130,26 +130,31 @@ components: description: "Description of the error" type: "string" type: "object" + FieldAdornment: + properties: + type: + description: "Type of this adornment" + enum: + - "LINK" + - "CHIP" + - "SIZE" + - "CODE_EDITOR" + - "RENDER_HTML" + - "REVEAL" + - "FILE_DOWNLOAD" + - "ERROR" + type: "string" + values: + description: "Values associated with this adornment. Keys and the meanings\ + \ of their values will differ by type." + type: "object" + type: "object" FieldMetaData: properties: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." @@ -973,21 +978,7 @@ components: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." From 1911e27cc0eade0b775d97a379166fec02b8ffe0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:38:28 -0600 Subject: [PATCH 046/209] CE-1955 clear out uploaded file if user goes back to this step --- .../bulk/insert/BulkInsertPrepareFileUploadStep.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index 5fe61f41..26ebc729 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -52,6 +52,14 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + runBackendStepOutput.addValue("theFile", null); + } + String tableName = runBackendStepInput.getValueString("tableName"); QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); From b5eae02fa4f96479c3a46d060dc1636ea868f751 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:39:18 -0600 Subject: [PATCH 047/209] CE-1955 populate association structures for record preview validation screen based on table structure associations, not actual mapping (e.g., so lines always appear on orders, even if not being used - to make that clear to user that they aren't being used) --- .../bulk/insert/BulkInsertTransformStep.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index ad531967..bd1fb40c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -58,7 +58,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; @@ -94,6 +95,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private static final int EXAMPLE_ROW_LIMIT = 10; + /******************************************************************************* ** extension of ProcessSummaryLine for lines where a UniqueKey was violated, ** where we'll collect a sample (or maybe all) of the values that broke the UK. @@ -139,16 +141,20 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); - BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); - if(bulkInsertMapping != null) + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(table.getName()); + if(CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) { ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); ArrayList previewRecordAssociationNames = new ArrayList<>(); - for(String mappedAssociation : bulkInsertMapping.getMappedAssociations()) + //////////////////////////////////////////////////////////// + // note - not recursively processing associations here... // + //////////////////////////////////////////////////////////// + for(BulkLoadTableStructure associatedStructure : tableStructure.getAssociations()) { - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst(); + String associationName = associatedStructure.getAssociationPath(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); if(association.isPresent()) { for(QFieldSection section : table.getSections()) @@ -518,17 +524,17 @@ public class BulkInsertTransformStep extends AbstractTransformStep String message = entry.getKey(); if(errorToExampleRowValueMap.containsKey(message)) { - ProcessSummaryLine line = entry.getValue(); - List rowValues = errorToExampleRowValueMap.get(message); - String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); } else if(errorToExampleRowsMap.containsKey(message)) { - ProcessSummaryLine line = entry.getValue(); - List rowDescriptions = errorToExampleRowsMap.get(message); - String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); } From 8157510c04d5a8eb83e037e1f4050273441a28bf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:39:38 -0600 Subject: [PATCH 048/209] CE-1955 Add fields to bulkLoad fileMapping screen, for helpContent to be associated with --- .../qqq/backend/core/instances/QInstanceEnricher.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index da9c9a91..1d88cb7e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -906,8 +906,10 @@ public class QInstanceEnricher QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() .withName("fileMapping") .withLabel("File Mapping") - .withBackStepName("upload") - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); + .withBackStepName("prepareFileUpload") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)) + .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN)) + .withFormField(new QFieldMetaData("layout", QFieldType.STRING)); // is actually PVS, but, this field is only added to help support helpContent, so :shrug: QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() .withName("receiveFileMapping") From 7bab11ea7ed875cb660a21b94948b01c6c05e9f9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:41:12 -0600 Subject: [PATCH 049/209] CE-1955 Add support for wildcard (at start of) process names - e.g., to support bulkLoad etc processes; update to apply all helpContent to the qInstance that came in as a parameter, rather than the one in context (to work correctly for hot-swaps). --- .../QInstanceHelpContentManager.java | 50 +++++++++++++------ .../QInstanceHelpContentManagerTest.java | 47 ++++++++++++++++- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java index 02376ff3..e46fb80a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java @@ -30,7 +30,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -111,7 +110,7 @@ public class QInstanceHelpContentManager } else { - LOG.info("Discarding help content with key that does not contain name:value format", logPair("key", key), logPair("id", record.getValue("id"))); + LOG.info("Discarding help content with key-part that does not contain name:value format", logPair("key", key), logPair("part", part), logPair("id", record.getValue("id"))); } } @@ -150,19 +149,19 @@ public class QInstanceHelpContentManager /////////////////////////////////////////////////////////////////////////////////// if(StringUtils.hasContent(tableName)) { - processHelpContentForTable(key, tableName, sectionName, fieldName, slotName, roles, helpContent); + processHelpContentForTable(qInstance, key, tableName, sectionName, fieldName, slotName, roles, helpContent); } else if(StringUtils.hasContent(processName)) { - processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent); + processHelpContentForProcess(qInstance, key, processName, fieldName, stepName, roles, helpContent); } else if(StringUtils.hasContent(widgetName)) { - processHelpContentForWidget(key, widgetName, slotName, roles, helpContent); + processHelpContentForWidget(qInstance, key, widgetName, slotName, roles, helpContent); } else if(nameValuePairs.containsKey("instanceLevel")) { - processHelpContentForInstance(key, slotName, roles, helpContent); + processHelpContentForInstance(qInstance, key, slotName, roles, helpContent); } } catch(Exception e) @@ -176,9 +175,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForTable(String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForTable(QInstance qInstance, String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) { - QTableMetaData table = QContext.getQInstance().getTable(tableName); + QTableMetaData table = qInstance.getTable(tableName); if(table == null) { LOG.info("Unrecognized table in help content", logPair("key", key)); @@ -246,9 +245,30 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) + private static void processHelpContentForProcess(QInstance qInstance, String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) { - QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(processName.startsWith("*") && processName.length() > 1) + { + boolean anyMatched = false; + String subName = processName.substring(1); + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getName().endsWith(subName)) + { + anyMatched = true; + processHelpContentForProcess(qInstance, key, process.getName(), fieldName, stepName, roles, helpContent); + } + } + + if(!anyMatched) + { + LOG.info("Wildcard process name did not match any processes in help content", logPair("key", key)); + } + + return; + } + + QProcessMetaData process = qInstance.getProcess(processName); if(process == null) { LOG.info("Unrecognized process in help content", logPair("key", key)); @@ -306,9 +326,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForWidget(QInstance qInstance, String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) { - QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); + QWidgetMetaDataInterface widget = qInstance.getWidget(widgetName); if(!StringUtils.hasContent(slotName)) { LOG.info("Missing slot name in help content", logPair("key", key)); @@ -335,7 +355,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForInstance(String key, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForInstance(QInstance qInstance, String key, String slotName, Set roles, QHelpContent helpContent) { if(!StringUtils.hasContent(slotName)) { @@ -345,11 +365,11 @@ public class QInstanceHelpContentManager { if(helpContent != null) { - QContext.getQInstance().withHelpContent(slotName, helpContent); + qInstance.withHelpContent(slotName, helpContent); } else { - QContext.getQInstance().removeHelpContent(slotName, roles); + qInstance.removeHelpContent(slotName, roles); } } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java index 51af9eaa..3511326f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java @@ -47,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataIn import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -292,6 +294,49 @@ class QInstanceHelpContentManagerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWildcardProcessField() throws QException + { + ///////////////////////////////////// + // get the instance from base test // + ///////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + + HelpContent recordEntity = new HelpContent() + .withId(1) + .withKey("process:*.bulkInsert;step:upload") + .withContent("v1") + .withRole(HelpContentRole.PROCESS_SCREEN.getId()); + new InsertAction().execute(new InsertInput(HelpContent.TABLE_NAME).withRecordEntity(recordEntity)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now - post-insert customizer should have automatically added help content to the instance - to all bulkInsert processes // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int hitCount = 0; + for(QTableMetaData table : qInstance.getTables().values()) + { + QProcessMetaData process = qInstance.getProcess(table.getName() + ".bulkInsert"); + if(process == null) + { + return; + } + + List helpContents = process.getFrontendStep("upload").getHelpContents(); + assertEquals(1, helpContents.size()); + assertEquals("v1", helpContents.get(0).getContent()); + assertEquals(Set.of(QHelpRole.PROCESS_SCREEN), helpContents.get(0).getRoles()); + hitCount++; + } + + assertThat(hitCount).isGreaterThanOrEqualTo(3); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,7 +456,7 @@ class QInstanceHelpContentManagerTest extends BaseTest QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply("foo;bar:baz")); assertThat(collectingLogger.getCollectedMessages()).hasSize(1); - assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key that does not contain name:value format"); + assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key-part that does not contain name:value format"); collectingLogger.clear(); QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply(null)); From 8d37ce3c5426d56742bac34cb7fa6b001167af83 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:43:10 -0600 Subject: [PATCH 050/209] CE-1955 add checks for material-dashboard resources before trying to blindly serve them; add field for QJavalinMetaData; --- .../javalin/QApplicationJavalinServer.java | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java index d18a6a03..3dfd3507 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; +import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; import com.kingsrook.qqq.middleware.javalin.specs.AbstractMiddlewareVersion; import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; import io.javalin.Javalin; @@ -69,6 +70,7 @@ public class QApplicationJavalinServer private List middlewareVersionList = List.of(new MiddlewareVersionV1()); private List additionalRouteProviders = null; private Consumer javalinConfigurationCustomizer = null; + private QJavalinMetaData javalinMetaData = null; private long lastQInstanceHotSwapMillis; private long millisBetweenHotSwaps = 2500; @@ -100,6 +102,11 @@ public class QApplicationJavalinServer { if(serveFrontendMaterialDashboard) { + if(getClass().getResource("/material-dashboard/index.html") == null) + { + LOG.warn("/material-dashboard/index.html resource was not found. This might happen if you're using a local (e.g., within-IDE) snapshot version... Try updating pom.xml to reference a released version of qfmd?"); + } + //////////////////////////////////////////////////////////////////////////////////////// // If you have any assets to add to the web server (e.g., logos, icons) place them at // // src/main/resources/material-dashboard-overlay (or a directory of your choice // @@ -108,7 +115,10 @@ public class QApplicationJavalinServer // material-dashboard directory, so in case the same file exists in both (e.g., // // favicon.png), the app-specific one will be used. // //////////////////////////////////////////////////////////////////////////////////////// - config.staticFiles.add("/material-dashboard-overlay"); + if(getClass().getResource("/material-dashboard-overlay") != null) + { + config.staticFiles.add("/material-dashboard-overlay"); + } ///////////////////////////////////////////////////////////////////// // tell javalin where to find material-dashboard static web assets // @@ -128,7 +138,7 @@ public class QApplicationJavalinServer { try { - QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); + QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance, javalinMetaData); config.router.apiBuilder(qJavalinImplementation.getRoutes()); } catch(QInstanceValidationException e) @@ -526,4 +536,35 @@ public class QApplicationJavalinServer return (this); } + + /******************************************************************************* + ** Getter for javalinMetaData + *******************************************************************************/ + public QJavalinMetaData getJavalinMetaData() + { + return (this.javalinMetaData); + } + + + + /******************************************************************************* + ** Setter for javalinMetaData + *******************************************************************************/ + public void setJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + } + + + + /******************************************************************************* + ** Fluent setter for javalinMetaData + *******************************************************************************/ + public QApplicationJavalinServer withJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + return (this); + } + + } From 76d7a8a858399d7ad196d78d45efa29970e35b79 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:43:33 -0600 Subject: [PATCH 051/209] CE-1955 Initial checkin --- .../selenium/BaseSampleSeleniumTest.java | 85 +++++++++++++++++ .../selenium/BulkLoadSeleniumTest.java | 95 +++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java new file mode 100644 index 00000000..a0a904fc --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + */ + +package com.kingsrook.sampleapp.selenium; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.frontend.materialdashboard.selenium.lib.QBaseSeleniumTest; +import com.kingsrook.sampleapp.SampleJavalinServer; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseSampleSeleniumTest extends QBaseSeleniumTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseSampleSeleniumTest.class); + + public static final Integer DEFAULT_WAIT_SECONDS = 10; + + private int port = 8011; + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @BeforeEach + public void beforeEach() + { + super.beforeEach(); + qSeleniumLib.withBaseUrl("http://localhost:" + port); + qSeleniumLib.withWaitSeconds(DEFAULT_WAIT_SECONDS); + + new SampleJavalinServer().startJavalinServer(port); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected boolean useInternalJavalin() + { + return (false); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clickLeftNavMenuItem(String text) + { + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clickLeftNavMenuItemThenSubItem(String text, String subItemText) + { + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiCollapse-vertical.MuiCollapse-entered .MuiListItem-root", subItemText).click(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void goToPathAndWaitForSelectorContaining(String path, String selector, String text) + { + driver.get(qSeleniumLib.getBaseUrl() + path); + qSeleniumLib.waitForSelectorContaining(selector, text); + } + +} + diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java new file mode 100644 index 00000000..02f2785d --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.sampleapp.selenium; + + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadSeleniumTest extends BaseSampleSeleniumTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimple() throws IOException + { + String email = "jtkirk@starfleet.com"; + String tablePath = "/peopleApp/greetingsApp/person"; + + //////////////////////////////////// + // write a file to be bulk-loaded // + //////////////////////////////////// + String path = "/tmp/" + UUID.randomUUID() + ".csv"; + String csv = String.format(""" + email,firstName,lastName + %s,James T.,Kirk + """, email); + FileUtils.writeStringToFile(new File(path), csv, StandardCharsets.UTF_8); + + goToPathAndWaitForSelectorContaining(tablePath + "/person.bulkInsert", ".MuiTypography-h5", "Person Bulk Insert: Upload File"); + + ////////////////////////////// + // complete the upload form // + ////////////////////////////// + qSeleniumLib.waitForSelector("input[type=file]").sendKeys(path); + qSeleniumLib.waitForSelectorContaining("button", "next").click(); + + ///////////////////////////////////////// + // proceed through file-mapping screen // + ///////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("button", "next").click(); + + ////////////////////////////////////////////////// + // confirm data on preview screen, then proceed // + ////////////////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", email); + qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", "Preview 1 of 1"); + qSeleniumLib.waitForSelectorContaining("button", "arrow_forward").click(); // to avoid the record-preview 'next' button + + /////////////////////////////////////// + // proceed through validation screen // + /////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("button", "submit").click(); + + //////////////////////////////////////// + // confirm result screen and close it // + //////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining(".MuiListItemText-root", "1 Person record was inserted"); + qSeleniumLib.waitForSelectorContaining("button", "close").click(); + + //////////////////////////////////////////// + // go to the order that was just inserted // + // bonus - also test record-view-by-key // + //////////////////////////////////////////// + goToPathAndWaitForSelectorContaining(tablePath + "/key?email=" + email, "h5", "Viewing Person"); + } + +} From f7bd049b8140dea67a2d3957556675c787301e30 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:44:29 -0600 Subject: [PATCH 052/209] CE-1955 Update qfmd to feature-bulk-upload-v2; add test-dep for qfmd; add slf4j simple and selenium and webdriver. --- qqq-sample-project/pom.xml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index 2ff7aee7..4db98cd3 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,14 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - 0.20.0 + feature-bulk-upload-v2-20241203.163149-2 + + + com.kingsrook.qqq + qqq-frontend-material-dashboard + tests + feature-bulk-upload-v2-20241203.163149-2 + test com.h2database @@ -77,7 +84,27 @@ - + + + + org.slf4j + slf4j-simple + 2.0.6 + + + + + org.seleniumhq.selenium + selenium-java + 4.19.1 + test + + + io.github.bonigarcia + webdrivermanager + 5.6.2 + test + From 131da68a38325f50f251d30b966bb5cdc10ba577 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:46:37 -0600 Subject: [PATCH 053/209] CE-1955 Update to use new AbstractQQQApplication and QApplicationJavalinServer --- .../sampleapp/SampleJavalinServer.java | 58 +++++++------------ .../metadata/SampleMetaDataProvider.java | 14 ++++- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index e78227fd..ead70bfc 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -23,11 +23,9 @@ package com.kingsrook.sampleapp; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; +import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer; import com.kingsrook.sampleapp.metadata.SampleMetaDataProvider; -import io.javalin.Javalin; /******************************************************************************* @@ -39,9 +37,7 @@ public class SampleJavalinServer private static final int PORT = 8000; - private QInstance qInstance; - - private Javalin javalinService; + private QApplicationJavalinServer qApplicationJavalinServer; @@ -59,42 +55,28 @@ public class SampleJavalinServer ** *******************************************************************************/ public void startJavalinServer() + { + startJavalinServer(PORT); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void startJavalinServer(int port) { try { - qInstance = SampleMetaDataProvider.defineInstance(); - SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); - QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); - qJavalinImplementation.setJavalinMetaData(new QJavalinMetaData() + qApplicationJavalinServer = new QApplicationJavalinServer(new SampleMetaDataProvider()); + qApplicationJavalinServer.setServeFrontendMaterialDashboard(true); + qApplicationJavalinServer.setServeLegacyUnversionedMiddlewareAPI(true); + qApplicationJavalinServer.setPort(port); + qApplicationJavalinServer.setJavalinMetaData(new QJavalinMetaData() .withUploadedFileArchiveTableName(SampleMetaDataProvider.UPLOAD_FILE_ARCHIVE_TABLE_NAME)); - - javalinService = Javalin.create(config -> - { - config.router.apiBuilder(qJavalinImplementation.getRoutes()); - // todo - not all? - config.bundledPlugins.enableCors(cors -> cors.addRule(corsRule -> corsRule.anyHost())); - }).start(PORT); - - ///////////////////////////////////////////////////////////////// - // set the server to hot-swap the q instance before all routes // - ///////////////////////////////////////////////////////////////// - QJavalinImplementation.setQInstanceHotSwapSupplier(() -> - { - try - { - return (SampleMetaDataProvider.defineInstance()); - } - catch(Exception e) - { - LOG.warn("Error hot-swapping meta data", e); - return (null); - } - }); - javalinService.before(QJavalinImplementation::hotSwapQInstance); - - javalinService.after(ctx -> ctx.res().setHeader("Access-Control-Allow-Origin", "http://localhost:3000")); + qApplicationJavalinServer.start(); } catch(Exception e) { @@ -109,9 +91,9 @@ public class SampleJavalinServer *******************************************************************************/ public void stopJavalinServer() { - if(javalinService != null) + if(qApplicationJavalinServer != null) { - javalinService.stop(); + qApplicationJavalinServer.stop(); } } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 77b05601..062d8721 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.QuickSightChartR import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; +import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -94,7 +95,7 @@ import org.apache.commons.io.IOUtils; /******************************************************************************* ** *******************************************************************************/ -public class SampleMetaDataProvider +public class SampleMetaDataProvider extends AbstractQQQApplication { public static boolean USE_MYSQL = false; @@ -128,6 +129,17 @@ public class SampleMetaDataProvider + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QInstance defineQInstance() throws QException + { + return defineInstance(); + } + + + /******************************************************************************* ** *******************************************************************************/ From 164d9e1de50186b72366d4bdf61f5daef499dcb1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 21:46:49 -0600 Subject: [PATCH 054/209] CE-1955 Checkstyle --- .../selenium/BaseSampleSeleniumTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java index a0a904fc..5c45c3de 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java @@ -1,5 +1,22 @@ /* - * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ package com.kingsrook.sampleapp.selenium; From eec1924113ff19c645c501b5885bf47ad374d835 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 22:03:03 -0600 Subject: [PATCH 055/209] CE-1955 add browser-tools orb, to try to fix selenium/chrome version mismatch --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fffc467..a3e27fbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: localstack: localstack/platform@2.1 + browser-tools: circleci/browser-tools@1.4.7 commands: store_jacoco_site: @@ -38,6 +39,8 @@ commands: - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} + - browser-tools/install-chrome + - browser-tools/install-chromedriver - run: name: Write .env command: | From 434d1587761416e2f61195506ddc71f205f21d79 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 07:12:10 -0600 Subject: [PATCH 056/209] CE-1955 disable until ci selenium fixed --- .../com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java index 02f2785d..e97055f5 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.UUID; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -39,6 +40,7 @@ public class BulkLoadSeleniumTest extends BaseSampleSeleniumTest ** *******************************************************************************/ @Test + @Disabled("selenium not working in circleci at this time...") void testSimple() throws IOException { String email = "jtkirk@starfleet.com"; From c4583f16a94b92ffa996798034f3bf3a6ffa29ee Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 14:58:34 -0600 Subject: [PATCH 057/209] CE-1955 Fix to re-set the position of the review step, upon going back --- .../BulkInsertPrepareFileMappingStep.java | 2 +- .../BulkInsertPrepareValueMappingStep.java | 2 +- .../StreamedETLWithFrontendProcess.java | 18 +++++++++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index 80cb49d1..c72df5fd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -65,7 +65,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { needSuggestedMapping = false; - StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); } if(needSuggestedMapping) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index d24856c0..471f0661 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -78,7 +78,7 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep { if(runBackendStepOutput.getProcessState().getIsStepBack()) { - StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); } ///////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index d05bb12c..09e621ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -70,6 +73,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.basepull.Basepul *******************************************************************************/ public class StreamedETLWithFrontendProcess { + private static final QLogger LOG = QLogger.getLogger(StreamedETLWithFrontendProcess.class); + public static final String STEP_NAME_PREVIEW = "preview"; public static final String STEP_NAME_REVIEW = "review"; public static final String STEP_NAME_VALIDATE = "validate"; @@ -197,11 +202,22 @@ public class StreamedETLWithFrontendProcess /*************************************************************************** ** useful for a process step to call upon 'back' ***************************************************************************/ - public static void resetValidationFields(RunBackendStepInput runBackendStepInput) + public static void resetValidationFields(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) { runBackendStepInput.addValue(FIELD_DO_FULL_VALIDATION, null); runBackendStepInput.addValue(FIELD_VALIDATION_SUMMARY, null); runBackendStepInput.addValue(FIELD_PROCESS_SUMMARY, null); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // in case, on the first time forward, the review step got moved after the validation step // + // (see BaseStreamedETLStep.moveReviewStepAfterValidateStep) - then un-do that upon going back. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + LOG.debug("Resetting step list. It was:" + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + LOG.debug("... and now step list is: " + stepList); } From 271f2dc25be8fcbbdddf933211b34b154e06e5bb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 14:59:53 -0600 Subject: [PATCH 058/209] CE-1955 Add a display-value for the mappingJSON in saved bulk-load-profiles --- ...ProfileJsonFieldDisplayValueFormatter.java | 141 ++++++++++++++++++ .../SavedBulkLoadProfileMetaDataProvider.java | 7 +- 2 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..843a1063 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static SavedBulkLoadProfileJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SavedBulkLoadProfileJsonFieldDisplayValueFormatter() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SavedBulkLoadProfileJsonFieldDisplayValueFormatter getInstance() + { + if(savedReportJsonFieldDisplayValueFormatter == null) + { + savedReportJsonFieldDisplayValueFormatter = new SavedBulkLoadProfileJsonFieldDisplayValueFormatter(); + } + return (savedReportJsonFieldDisplayValueFormatter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SavedBulkLoadProfileJsonFieldDisplayValueFormatter getDefault() + { + return getInstance(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(field.getName().equals("mappingJson")) + { + String mappingJson = record.getValueString("mappingJson"); + if(StringUtils.hasContent(mappingJson)) + { + try + { + record.setDisplayValue("mappingJson", jsonToDisplayValue(mappingJson)); + } + catch(Exception e) + { + record.setDisplayValue("mappingJson", "Invalid Mapping..."); + } + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String jsonToDisplayValue(String mappingJson) + { + JSONObject jsonObject = new JSONObject(mappingJson); + + List parts = new ArrayList<>(); + + if(jsonObject.has("fieldList")) + { + JSONArray fieldListArray = jsonObject.getJSONArray("fieldList"); + parts.add(fieldListArray.length() + " field" + StringUtils.plural(fieldListArray.length())); + } + + if(jsonObject.has("hasHeaderRow")) + { + boolean hasHeaderRow = jsonObject.getBoolean("hasHeaderRow"); + parts.add((hasHeaderRow ? "With" : "Without") + " header row"); + } + + if(jsonObject.has("layout")) + { + String layout = jsonObject.getString("layout"); + parts.add("Layout: " + StringUtils.allCapsToMixedCase(layout)); + } + + return StringUtils.join("; ", parts); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index 23826835..1438dd37 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -113,12 +113,11 @@ public class SavedBulkLoadProfileMetaDataProvider .withFieldsFromEntity(SavedBulkLoadProfile.class) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) - .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("mappingJson")).withIsHidden(true)) - .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId")).withIsHidden(true)) + .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); - // todo - want one of these? - // table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); + table.getField("mappingJson").setLabel("Mapping"); table.withShareableTableMetaData(new ShareableTableMetaData() .withSharedRecordTableName(SharedSavedBulkLoadProfile.TABLE_NAME) From edf248c851e341b16d83ebfcb0960e8e88df0281 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 19 Dec 2024 16:03:39 -0600 Subject: [PATCH 059/209] Add methods to ProcessLockUtils to work in bulk (both for creating and releasing locks); fix ProcessLock join to type table (had wrong joinOn field) --- .../locks/ProcessLockMetaDataProducer.java | 2 +- .../locks/ProcessLockOrException.java | 53 ++ .../processes/locks/ProcessLockUtils.java | 496 ++++++++++++++---- .../processes/locks/ProcessLockUtilsTest.java | 147 +++++- 4 files changed, 584 insertions(+), 114 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockOrException.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java index 4e9b214a..c338fa97 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockMetaDataProducer.java @@ -95,7 +95,7 @@ public class ProcessLockMetaDataProducer implements MetaDataProducerInterface. + */ + +package com.kingsrook.qqq.backend.core.processes.locks; + + +/*************************************************************************** + ** Record to hold either a processLock, or an unableToObtainProcessLockException. + ** Used as return value from bulk-methods in ProcessLockUtils (where some + ** requested keys may succeed and return a lock, and others may fail + ** and return the exception). + ***************************************************************************/ +public record ProcessLockOrException(ProcessLock processLock, UnableToObtainProcessLockException unableToObtainProcessLockException) +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessLockOrException(ProcessLock processLock) + { + this(processLock, null); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ProcessLockOrException(UnableToObtainProcessLockException unableToObtainProcessLockException) + { + this(null, unableToObtainProcessLockException); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java index ac3a05e8..0d062ce1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtils.java @@ -22,14 +22,24 @@ package com.kingsrook.qqq.backend.core.processes.locks; +import java.io.Serializable; import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; @@ -39,6 +49,10 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -70,10 +84,79 @@ public class ProcessLockUtils /******************************************************************************* + ** try to create a process lock, of a given key & type - but immediately fail + ** if the lock already exists. ** + ** @param key along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. *******************************************************************************/ public static ProcessLock create(String key, String typeName, String details) throws UnableToObtainProcessLockException, QException { + Map locks = createMany(List.of(key), typeName, details); + return getProcessLockOrThrow(key, locks); + } + + + + /******************************************************************************* + ** try to create a process lock, of a given key & type - and re-try if it failed. + ** (e.g., wait until existing lock holder releases the lock). + ** + ** @param key along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + ** @param sleepBetweenTries how long to sleep between retries. + ** @param maxWait max amount of that will be waited between call to this method + * and an eventual UnableToObtainProcessLockException (plus or minus + * one sleepBetweenTries (actually probably just plus that). + ** + *******************************************************************************/ + public static ProcessLock create(String key, String typeName, String details, Duration sleepBetweenTries, Duration maxWait) throws UnableToObtainProcessLockException, QException + { + Map locks = createMany(List.of(key), typeName, details, sleepBetweenTries, maxWait); + return getProcessLockOrThrow(key, locks); + } + + + + /*************************************************************************** + ** For the single-lock versions of create, either return the lock identified by + ** key, or throw. + ***************************************************************************/ + private static ProcessLock getProcessLockOrThrow(String key, Map locks) throws UnableToObtainProcessLockException + { + if(locks.get(key) != null && locks.get(key).processLock() != null) + { + return (locks.get(key).processLock()); + } + else if(locks.get(key) != null && locks.get(key).unableToObtainProcessLockException() != null) + { + throw (locks.get(key).unableToObtainProcessLockException()); + } + else + { + throw (new UnableToObtainProcessLockException("Missing key [" + key + "] in response from request to create lock. Lock not created.")); + } + } + + + + /******************************************************************************* + ** try to create many process locks, of list of keys & a type - but immediately + ** fail (on a one-by-one basis) if the lock already exists. + ** + ** @param keys along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + *******************************************************************************/ + public static Map createMany(List keys, String typeName, String details) throws QException + { + Map rs = new HashMap<>(); + ProcessLockType lockType = getProcessLockTypeByName(typeName); if(lockType == null) { @@ -82,137 +165,305 @@ public class ProcessLockUtils QSession qSession = QContext.getQSession(); - Instant now = Instant.now(); - ProcessLock processLock = new ProcessLock() - .withKey(key) - .withProcessLockTypeId(lockType.getId()) - .withSessionUUID(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUuid(), null)) - .withUserId(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUser().getIdReference(), null)) - .withDetails(details) - .withCheckInTimestamp(now); + Instant now = Instant.now(); + Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); + List processLocksToInsert = new ArrayList<>(); - Integer defaultExpirationSeconds = lockType.getDefaultExpirationSeconds(); - if(defaultExpirationSeconds != null) + Function constructProcessLockFromKey = (key) -> { - processLock.setExpiresAtTimestamp(now.plusSeconds(defaultExpirationSeconds)); + ProcessLock processLock = new ProcessLock() + .withKey(key) + .withProcessLockTypeId(lockType.getId()) + .withSessionUUID(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUuid(), null)) + .withUserId(ObjectUtils.tryAndRequireNonNullElse(() -> qSession.getUser().getIdReference(), null)) + .withDetails(details) + .withCheckInTimestamp(now); + + if(defaultExpirationSeconds != null) + { + processLock.setExpiresAtTimestamp(now.plusSeconds(defaultExpirationSeconds)); + } + + return (processLock); + }; + + for(String key : keys) + { + processLocksToInsert.add(constructProcessLockFromKey.apply(key)); } - QRecord insertOutputRecord = tryToInsert(processLock); + Map insertResultMap = tryToInsertMany(processLocksToInsert); - //////////////////////////////////////////////////////////// - // if inserting failed... see if we can get existing lock // - //////////////////////////////////////////////////////////// - StringBuilder existingLockDetails = new StringBuilder(); - ProcessLock existingLock = null; - if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + //////////////////////////////////////// + // look at which (if any) keys failed // + //////////////////////////////////////// + Set failedKeys = new HashSet<>(); + for(Map.Entry entry : insertResultMap.entrySet()) { - QRecord existingLockRecord = new GetAction().executeForRecord(new GetInput(ProcessLock.TABLE_NAME).withUniqueKey(Map.of("key", key, "processLockTypeId", lockType.getId()))); - if(existingLockRecord != null) + if(entry.getValue().unableToObtainProcessLockException() != null) { - existingLock = new ProcessLock(existingLockRecord); - if(StringUtils.hasContent(existingLock.getUserId())) - { - existingLockDetails.append("Held by: ").append(existingLock.getUserId()); - } + failedKeys.add(entry.getKey()); + } + } - if(StringUtils.hasContent(existingLock.getDetails())) - { - existingLockDetails.append("; with details: ").append(existingLock.getDetails()); - } + ////////////////////////////////////////////////////////////////////// + // if any keys failed, try to get the existing locks for those keys // + ////////////////////////////////////////////////////////////////////// + Map existingLockRecords = new HashMap<>(); + if(CollectionUtils.nullSafeHasContents(failedKeys)) + { + QueryOutput queryOutput = new QueryAction().execute(new QueryInput(ProcessLock.TABLE_NAME).withFilter(new QQueryFilter() + .withCriteria("processLockTypeId", QCriteriaOperator.EQUALS, lockType.getId()) + .withCriteria("key", QCriteriaOperator.IN, failedKeys))); + for(QRecord record : queryOutput.getRecords()) + { + existingLockRecords.put(record.getValueString("key"), record); + } + } - Instant expiresAtTimestamp = existingLock.getExpiresAtTimestamp(); - if(expiresAtTimestamp != null) - { - ZonedDateTime zonedExpiresAt = expiresAtTimestamp.atZone(ValueUtils.getSessionOrInstanceZoneId()); - existingLockDetails.append("; expiring at: ").append(QValueFormatter.formatDateTimeWithZone(zonedExpiresAt)); - } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over results from insert call - either adding successes to the output structure, or adding details about failures, // + // OR - deleting expired locks and trying a second insert! // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + List deleteIdList = new ArrayList<>(); + List tryAgainList = new ArrayList<>(); + Map existingLockDetailsMap = new HashMap<>(); + Map existingLockMap = new HashMap<>(); + for(Map.Entry entry : insertResultMap.entrySet()) + { + String key = entry.getKey(); + ProcessLock processLock = entry.getValue().processLock(); - if(expiresAtTimestamp != null && expiresAtTimestamp.isBefore(now)) - { - ///////////////////////////////////////////////////////////////////////////////// - // if existing lock has expired, then we can delete it and try to insert again // - ///////////////////////////////////////////////////////////////////////////////// - LOG.info("Existing lock has expired - deleting it and trying again.", logPair("id", existingLock.getId()), - logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", expiresAtTimestamp)); - new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(existingLock.getId())); - insertOutputRecord = tryToInsert(processLock); - } + ////////////////////////////////////////////////////////////////////////// + // if inserting failed... see if we found an existing lock for this key // + ////////////////////////////////////////////////////////////////////////// + StringBuilder existingLockDetails = new StringBuilder(); + ProcessLock existingLock = null; + + if(processLock != null) + { + rs.put(key, new ProcessLockOrException(processLock)); } else { - ///////////////////////////////////////////////////////// - // if existing lock doesn't exist, try to insert again // - ///////////////////////////////////////////////////////// - insertOutputRecord = tryToInsert(processLock); + QRecord existingLockRecord = existingLockRecords.get(key); + if(existingLockRecord != null) + { + existingLock = new ProcessLock(existingLockRecord); + if(StringUtils.hasContent(existingLock.getUserId())) + { + existingLockDetails.append("Held by: ").append(existingLock.getUserId()); + } + + if(StringUtils.hasContent(existingLock.getDetails())) + { + existingLockDetails.append("; with details: ").append(existingLock.getDetails()); + } + + Instant expiresAtTimestamp = existingLock.getExpiresAtTimestamp(); + if(expiresAtTimestamp != null) + { + ZonedDateTime zonedExpiresAt = expiresAtTimestamp.atZone(ValueUtils.getSessionOrInstanceZoneId()); + existingLockDetails.append("; expiring at: ").append(QValueFormatter.formatDateTimeWithZone(zonedExpiresAt)); + } + + existingLockDetailsMap.put(key, existingLockDetails.toString()); + existingLockMap.put(key, existingLock); + + if(expiresAtTimestamp != null && expiresAtTimestamp.isBefore(now)) + { + ///////////////////////////////////////////////////////////////////////////////// + // if existing lock has expired, then we can delete it and try to insert again // + ///////////////////////////////////////////////////////////////////////////////// + LOG.info("Existing lock has expired - deleting it and trying again.", logPair("id", existingLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", expiresAtTimestamp)); + deleteIdList.add(existingLock.getId()); + tryAgainList.add(constructProcessLockFromKey.apply(key)); + } + } + else + { + /////////////////////////////////////////////////////////////////////////////// + // if existing lock doesn't exist now (e.g., it was deleted before the UC // + // check failed and when we looked for it), then just try to insert it again // + /////////////////////////////////////////////////////////////////////////////// + tryAgainList.add(constructProcessLockFromKey.apply(key)); + } } } - if(CollectionUtils.nullSafeHasContents(insertOutputRecord.getErrors())) + ///////////////////////////////////////////////////// + // if there are expired locks to delete, do so now // + ///////////////////////////////////////////////////// + if(!deleteIdList.isEmpty()) { - ///////////////////////////////////////////////////////////////////////////////// - // if at this point, we have errors on the last attempted insert, then give up // - ///////////////////////////////////////////////////////////////////////////////// - LOG.info("Errors in process lock record after attempted insert", logPair("errors", insertOutputRecord.getErrors()), - logPair("key", key), logPair("type", typeName), logPair("details", details)); - throw (new UnableToObtainProcessLockException("A Process Lock already exists for key [" + key + "] of type [" + typeName + "], " + existingLockDetails) - .withExistingLock(existingLock)); + new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKeys(deleteIdList)); } - LOG.info("Created process lock", logPair("id", processLock.getId()), - logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", processLock.getExpiresAtTimestamp())); - return new ProcessLock(insertOutputRecord); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are any to try again (either because we just deleted their now-expired locks, or because we otherwise couldn't find their locks, do so now // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!tryAgainList.isEmpty()) + { + Map tryAgainResult = tryToInsertMany(tryAgainList); + for(Map.Entry entry : tryAgainResult.entrySet()) + { + String key = entry.getKey(); + ProcessLock processLock = entry.getValue().processLock(); + UnableToObtainProcessLockException unableToObtainProcessLockException = entry.getValue().unableToObtainProcessLockException(); + + if(processLock != null) + { + rs.put(key, new ProcessLockOrException(processLock)); + } + else + { + rs.put(key, new ProcessLockOrException(Objects.requireNonNullElseGet(unableToObtainProcessLockException, () -> new UnableToObtainProcessLockException("Process lock not created, but no details available.")))); + } + } + } + + //////////////////////////////////////////////////////////////////// + // put anything not successfully created into result map as error // + //////////////////////////////////////////////////////////////////// + for(ProcessLock processLock : processLocksToInsert) + { + String key = processLock.getKey(); + if(rs.containsKey(key)) + { + LOG.info("Created process lock", logPair("id", processLock.getId()), + logPair("key", key), logPair("type", typeName), logPair("details", details), logPair("expiresAtTimestamp", processLock.getExpiresAtTimestamp())); + } + else + { + if(existingLockDetailsMap.containsKey(key)) + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException("A Process Lock already exists for key [" + key + "] of type [" + typeName + "], " + existingLockDetailsMap.get(key)) + .withExistingLock(existingLockMap.get(key)))); + } + else + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException("Process lock for key [" + key + "] of type [" + typeName + "] was not created..."))); + } + } + } + + return (rs); } /******************************************************************************* - ** + ** Try to do an insert - noting that an exception from the InsertAction will be + ** caught in here, and placed in the records as an Error! *******************************************************************************/ - private static QRecord tryToInsert(ProcessLock processLock) throws QException + private static Map tryToInsertMany(List processLocks) { - return new InsertAction().execute(new InsertInput(ProcessLock.TABLE_NAME).withRecordEntity(processLock)).getRecords().get(0); + Map rs = new HashMap<>(); + + try + { + List insertedRecords = new InsertAction().execute(new InsertInput(ProcessLock.TABLE_NAME).withRecordEntities(processLocks)).getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + String key = insertedRecord.getValueString("key"); + if(CollectionUtils.nullSafeHasContents(insertedRecord.getErrors())) + { + rs.put(key, new ProcessLockOrException(new UnableToObtainProcessLockException(insertedRecord.getErrors().get(0).getMessage()))); + } + else + { + rs.put(key, new ProcessLockOrException(new ProcessLock(insertedRecord))); + } + } + } + catch(Exception e) + { + for(ProcessLock processLock : processLocks) + { + rs.put(processLock.getKey(), new ProcessLockOrException(new UnableToObtainProcessLockException("Error attempting to insert process lock: " + e.getMessage()))); + } + } + + return (rs); } /******************************************************************************* + ** try to create many process locks, of a given list of key & a type - and re-try + ** upon failures (e.g., wait until existing lock holder releases the lock). + ** + ** @param keys along with typeName, part of Unique Key for the lock. + ** @param typeName along with key, part of Unique Key for the lock. Must be a + * defined lock type, from which we derive defaultExpirationSeconds. + ** @param details advice to show users re: who/what created the lock. + ** @param sleepBetweenTries how long to sleep between retries. + ** @param maxWait max amount of that will be waited between call to this method + * and an eventual UnableToObtainProcessLockException (plus or minus + * one sleepBetweenTries (actually probably just plus that). ** *******************************************************************************/ - public static ProcessLock create(String key, String type, String holderId, Duration sleepBetweenTries, Duration maxWait) throws UnableToObtainProcessLockException, QException + public static Map createMany(List keys, String typeName, String details, Duration sleepBetweenTries, Duration maxWait) throws QException { + Map rs = new HashMap<>(); + Map lastExceptionsPerKey = new HashMap<>(); + Set stillNeedCreated = new HashSet<>(keys); + Instant giveUpTime = Instant.now().plus(maxWait); UnableToObtainProcessLockException lastCaughtUnableToObtainProcessLockException = null; while(true) { - try + Map createManyResult = createMany(stillNeedCreated.size() == keys.size() ? keys : new ArrayList<>(stillNeedCreated), typeName, details); + for(Map.Entry entry : createManyResult.entrySet()) { - ProcessLock processLock = create(key, type, holderId); - return (processLock); + String key = entry.getKey(); + ProcessLockOrException processLockOrException = entry.getValue(); + if(processLockOrException.processLock() != null) + { + rs.put(key, processLockOrException); + stillNeedCreated.remove(key); + } + else if(processLockOrException.unableToObtainProcessLockException() != null) + { + lastExceptionsPerKey.put(key, processLockOrException.unableToObtainProcessLockException()); + } } - catch(UnableToObtainProcessLockException e) + + if(stillNeedCreated.isEmpty()) { - lastCaughtUnableToObtainProcessLockException = e; - if(Instant.now().plus(sleepBetweenTries).isBefore(giveUpTime)) - { - SleepUtils.sleep(sleepBetweenTries); - } - else - { - break; - } + ////////////////////////////////////////////////////////// + // if they've all been created now, great, return them! // + ////////////////////////////////////////////////////////// + return (rs); + } + + ///////////////////////////////////////////////////////////////////////////// + // oops, let's sleep (if we're before the give up time) and then try again // + ///////////////////////////////////////////////////////////////////////////// + if(Instant.now().plus(sleepBetweenTries).isBefore(giveUpTime)) + { + SleepUtils.sleep(sleepBetweenTries); + } + else + { + ///////////////////////////////// + // else, break if out of time! // + ///////////////////////////////// + break; } } - /////////////////////////////////////////////////////////////////////////////////////////////////// - // this variable can never be null with current code-path, but prefer to be defensive regardless // - /////////////////////////////////////////////////////////////////////////////////////////////////// - @SuppressWarnings("ConstantValue") - String suffix = lastCaughtUnableToObtainProcessLockException == null ? "" : ": " + lastCaughtUnableToObtainProcessLockException.getMessage(); + //////////////////////////////////////////////////////////////////////////////////////////// + // any that didn't get created, they need their last error (or a new error) put in the rs // + //////////////////////////////////////////////////////////////////////////////////////////// + for(String key : stillNeedCreated) + { + rs.put(key, new ProcessLockOrException(lastExceptionsPerKey.getOrDefault(key, new UnableToObtainProcessLockException("Missing key [" + key + "] in response from request to create lock. Lock not created.")))); + } - //noinspection ConstantValue - throw (new UnableToObtainProcessLockException("Unable to obtain process lock for key [" + key + "] in type [" + type + "] after [" + maxWait + "]" + suffix) - .withExistingLock(lastCaughtUnableToObtainProcessLockException == null ? null : lastCaughtUnableToObtainProcessLockException.getExistingLock())); + return (rs); } @@ -389,27 +640,41 @@ public class ProcessLockUtils { if(id == null) { - LOG.debug("No id passed in to releaseById - returning with noop"); + LOG.debug("No ids passed in to releaseById - returning with noop"); + return; + } + + releaseByIds(List.of(id)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void releaseByIds(List ids) + { + List nonNullIds = ids == null ? Collections.emptyList() : ids.stream().filter(Objects::nonNull).map(o -> (Serializable) o).toList(); + + if(CollectionUtils.nullSafeIsEmpty(nonNullIds)) + { + LOG.debug("No ids passed in to releaseById - returning with noop"); return; } - ProcessLock processLock = null; try { - processLock = ProcessLockUtils.getById(id); - if(processLock == null) + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKeys(nonNullIds)); + if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) { - LOG.info("Process lock not found in releaseById call", logPair("id", id)); + throw (new QException("Error deleting processLocks: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString())); } + + LOG.info("Released process locks", logPair("ids", nonNullIds)); } catch(QException e) { - LOG.warn("Exception releasing processLock byId", e, logPair("id", id)); - } - - if(processLock != null) - { - release(processLock); + LOG.warn("Exception releasing processLocks byId", e, logPair("ids", ids)); } } @@ -420,26 +685,33 @@ public class ProcessLockUtils *******************************************************************************/ public static void release(ProcessLock processLock) { - try + if(processLock == null) { - if(processLock == null) - { - LOG.debug("No process lock passed in to release - returning with noop"); - return; - } - - DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(ProcessLock.TABLE_NAME).withPrimaryKey(processLock.getId())); - if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors())) - { - throw (new QException("Error deleting processLock record: " + deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString())); - } - - LOG.info("Released process lock", logPair("id", processLock.getId()), logPair("key", processLock.getKey()), logPair("typeId", processLock.getProcessLockTypeId()), logPair("details", processLock.getDetails())); + LOG.debug("No process lock passed in to release - returning with noop"); + return; } - catch(QException e) + + releaseMany(List.of(processLock)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void releaseMany(List processLocks) + { + if(CollectionUtils.nullSafeIsEmpty(processLocks)) { - LOG.warn("Exception releasing processLock", e, logPair("processLockId", () -> processLock.getId())); + LOG.debug("No process locks passed in to release - returning with noop"); + return; } + + List ids = processLocks.stream() + .filter(Objects::nonNull) + .map(pl -> (Serializable) pl.getId()) + .toList(); + releaseByIds(ids); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java index cbe594f6..b177b9bd 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/locks/ProcessLockUtilsTest.java @@ -25,13 +25,17 @@ package com.kingsrook.qqq.backend.core.processes.locks; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -39,6 +43,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.session.QUser; import com.kingsrook.qqq.backend.core.utils.SleepUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -227,6 +232,28 @@ class ProcessLockUtilsTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testReleaseBadInputs() + { + ////////////////////////////////////////////////////////// + // make sure we don't blow up, just noop in these cases // + ////////////////////////////////////////////////////////// + QCollectingLogger qCollectingLogger = QLogger.activateCollectingLoggerForClass(ProcessLockUtils.class); + ProcessLockUtils.releaseById(null); + ProcessLockUtils.release(null); + ProcessLockUtils.releaseMany(null); + ProcessLockUtils.releaseByIds(null); + ProcessLockUtils.releaseMany(ListBuilder.of(null)); + ProcessLockUtils.releaseByIds(ListBuilder.of(null)); + QLogger.deactivateCollectingLoggerForClass(ProcessLockUtils.class); + assertEquals(6, qCollectingLogger.getCollectedMessages().stream().filter(m -> m.getMessage().contains("noop")).count()); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -304,7 +331,7 @@ class ProcessLockUtilsTest extends BaseTest ////////////////////////////////////////////// // checkin w/ a time - sets it to that time // ////////////////////////////////////////////// - Instant specifiedTime = Instant.now(); + Instant specifiedTime = Instant.now().plusSeconds(47); ProcessLockUtils.checkIn(processLock, specifiedTime); processLock = ProcessLockUtils.getById(processLock.getId()); assertEquals(specifiedTime, processLock.getExpiresAtTimestamp()); @@ -380,4 +407,122 @@ class ProcessLockUtilsTest extends BaseTest assertNull(processLock.getExpiresAtTimestamp()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMany() throws QException + { + ///////////////////////////////////////////////// + // make sure that we can create multiple locks // + ///////////////////////////////////////////////// + List keys = List.of("1", "2", "3"); + List processLocks = new ArrayList<>(); + Map results = ProcessLockUtils.createMany(keys, "typeA", "me"); + for(String key : keys) + { + ProcessLock processLock = results.get(key).processLock(); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNull(processLock.getExpiresAtTimestamp()); + processLocks.add(processLock); + } + + ///////////////////////////////////////////////////////// + // make sure we can't create a second for the same key // + ///////////////////////////////////////////////////////// + assertThatThrownBy(() -> ProcessLockUtils.create("1", "typeA", "you")) + .isInstanceOf(UnableToObtainProcessLockException.class) + .hasMessageContaining("Held by: " + QContext.getQSession().getUser().getIdReference()) + .hasMessageContaining("with details: me") + .hasMessageNotContaining("expiring at: 20") + .matches(e -> ((UnableToObtainProcessLockException) e).getExistingLock() != null); + + ///////////////////////////////////////////////////////// + // make sure we can create another for a different key // + ///////////////////////////////////////////////////////// + ProcessLockUtils.create("4", "typeA", "him"); + + ///////////////////////////////////////////////////////////////////// + // make sure we can create another for a different type (same key) // + ///////////////////////////////////////////////////////////////////// + ProcessLockUtils.create("1", "typeB", "her"); + + //////////////////////////////////////////////////////////////////// + // now try to create some that will overlap, but one that'll work // + //////////////////////////////////////////////////////////////////// + keys = List.of("3", "4", "5"); + results = ProcessLockUtils.createMany(keys, "typeA", "me"); + for(String key : List.of("3", "4")) + { + UnableToObtainProcessLockException exception = results.get(key).unableToObtainProcessLockException(); + assertNotNull(exception); + } + + ProcessLock processLock = results.get("5").processLock(); + assertNotNull(processLock.getId()); + assertNotNull(processLock.getCheckInTimestamp()); + assertNull(processLock.getExpiresAtTimestamp()); + processLocks.add(processLock); + + ////////////////////////////// + // make sure we can release // + ////////////////////////////// + ProcessLockUtils.releaseMany(processLocks); + + /////////////////////////////////////////////////////// + // make sure can re-lock 1 now after it was released // + /////////////////////////////////////////////////////// + processLock = ProcessLockUtils.create("1", "typeA", "you"); + assertNotNull(processLock.getId()); + assertEquals("you", processLock.getDetails()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testManyWithSleep() throws QException + { + ///////////////////////////////////////////////// + // make sure that we can create multiple locks // + ///////////////////////////////////////////////// + List keys = List.of("1", "2", "3"); + Map results0 = ProcessLockUtils.createMany(keys, "typeB", "me"); + for(String key : keys) + { + assertNotNull(results0.get(key).processLock()); + } + + //////////////////////////////////////////////////////////// + // try again - and 2 and 3 should fail, if we don't sleep // + //////////////////////////////////////////////////////////// + keys = List.of("2", "3", "4"); + Map results1 = ProcessLockUtils.createMany(keys, "typeB", "you"); + assertNull(results1.get("2").processLock()); + assertNull(results1.get("3").processLock()); + assertNotNull(results1.get("4").processLock()); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try another insert, which should initially succeed for #5, then sleep, and eventually succeed on 3 & 4 as well // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + keys = List.of("3", "4", "5"); + Map results2 = ProcessLockUtils.createMany(keys, "typeB", "them", Duration.of(1, ChronoUnit.SECONDS), Duration.of(3, ChronoUnit.SECONDS)); + for(String key : keys) + { + assertNotNull(results2.get(key).processLock()); + } + + //////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that we have a different ids for some that expired and then succeeded post-sleep // + //////////////////////////////////////////////////////////////////////////////////////////////// + assertNotEquals(results0.get("3").processLock().getId(), results2.get("3").processLock().getId()); + assertNotEquals(results1.get("4").processLock().getId(), results2.get("4").processLock().getId()); + + } + } \ No newline at end of file From 000226c30a18d0a4c7e76763b838be2d9f1d5257 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 20 Dec 2024 09:38:20 -0600 Subject: [PATCH 060/209] Make unique id on pet species enum --- .../kingsrook/sampleapp/metadata/SampleMetaDataProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 062d8721..c416e358 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -778,7 +778,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication public enum PetSpecies implements PossibleValueEnum { DOG(1, "Dog"), - CAT(1, "Cat"); + CAT(2, "Cat"); private final Integer id; private final String label; From a6001af7b56062866c7b72de52adbcafa0c46a52 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 23 Dec 2024 14:59:28 -0600 Subject: [PATCH 061/209] Add overload of toQRecordOnlyChangedFields that allows primary keys to be included (more useful for the update use-case) --- .../core/model/data/QRecordEntity.java | 141 +++++++++------ .../model/metadata/tables/QTableMetaData.java | 14 +- .../core/model/data/QRecordEntityTest.java | 162 +++++++++++++++++- .../core/model/data/testentities/Item.java | 130 ++++++++++++++ 4 files changed, 387 insertions(+), 60 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java index e62757e1..6425893a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntity.java @@ -41,11 +41,14 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -61,6 +64,11 @@ public abstract class QRecordEntity private Map originalRecordValues; + //////////////////////////////////////////////////////////////////////////////// + // map of entity class names to QTableMetaData objects that they helped build // + //////////////////////////////////////////////////////////////////////////////// + private static Map tableReferences = new HashMap<>(); + /******************************************************************************* @@ -95,6 +103,19 @@ public abstract class QRecordEntity + /*************************************************************************** + ** register a mapping between an entity class and a table that it is associated with. + ***************************************************************************/ + public static void registerTable(Class entityClass, QTableMetaData table) + { + if(entityClass != null && table != null) + { + tableReferences.put(entityClass.getName(), table); + } + } + + + /******************************************************************************* ** Build an entity of this QRecord type from a QRecord ** @@ -176,7 +197,10 @@ public abstract class QRecordEntity /******************************************************************************* - ** Convert this entity to a QRecord. + ** Convert this entity to a QRecord. ALL fields in the entity will be set + ** in the QRecord. Note that, if you're using this for an input to the UpdateAction, + ** that this could cause values to be set to null, e.g., if you constructed + ** a entity from scratch, and didn't set all values in it!! ** *******************************************************************************/ public QRecord toQRecord() throws QRuntimeException @@ -190,25 +214,7 @@ public abstract class QRecordEntity qRecord.setValue(qRecordEntityField.getFieldName(), (Serializable) qRecordEntityField.getGetter().invoke(this)); } - for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) - { - @SuppressWarnings("unchecked") - List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); - String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); - - if(associatedEntities != null) - { - ///////////////////////////////////////////////////////////////////////////////// - // do this so an empty list in the entity becomes an empty list in the QRecord // - ///////////////////////////////////////////////////////////////////////////////// - qRecord.withAssociatedRecords(associationName, new ArrayList<>()); - } - - for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities)) - { - qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord()); - } - } + toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecord()); return (qRecord); } @@ -220,15 +226,65 @@ public abstract class QRecordEntity + /*************************************************************************** + * + ***************************************************************************/ + private void toQRecordProcessAssociations(QRecord outputRecord, Function toRecordFunction) throws Exception + { + for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) + { + @SuppressWarnings("unchecked") + List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); + String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); + + if(associatedEntities != null) + { + outputRecord.withAssociatedRecords(associationName, new ArrayList<>()); + for(QRecordEntity associatedEntity : associatedEntities) + { + outputRecord.withAssociatedRecord(associationName, toRecordFunction.apply(associatedEntity)); + } + } + } + } + + + /******************************************************************************* - ** + ** Overload of toQRecordOnlyChangedFields that preserves original behavior of + ** that method, which is, to NOT includePrimaryKey *******************************************************************************/ + @Deprecated(since = "includePrimaryKey param was added") public QRecord toQRecordOnlyChangedFields() + { + return toQRecordOnlyChangedFields(false); + } + + + + /******************************************************************************* + ** Useful for the use-case of: + ** - fetch a QRecord (e.g., QueryAction or GetAction) + ** - build a QRecordEntity out of it + ** - change a field (or two) in it + ** - want to pass it into an UpdateAction, and want to see only the fields that + ** you know you changed get passed in to UpdateAction (e.g., PATCH semantics). + ** + ** But also - per the includePrimaryKey param, include the primaryKey in the + ** records (e.g., to tell the Update which records to update). + ** + ** Also, useful for: + ** - construct new entity, calling setters to populate some fields + ** - pass that entity into + *******************************************************************************/ + public QRecord toQRecordOnlyChangedFields(boolean includePrimaryKey) { try { QRecord qRecord = new QRecord(); + String primaryKeyFieldName = ObjectUtils.tryElse(() -> tableReferences.get(getClass().getName()).getPrimaryKeyField(), null); + for(QRecordEntityField qRecordEntityField : getFieldList(this.getClass())) { Serializable thisValue = (Serializable) qRecordEntityField.getGetter().invoke(this); @@ -238,31 +294,16 @@ public abstract class QRecordEntity originalValue = originalRecordValues.get(qRecordEntityField.getFieldName()); } - if(!Objects.equals(thisValue, originalValue)) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if this value and the original value don't match - OR - this is the table's primary key field - then put the value in the record. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(!Objects.equals(thisValue, originalValue) || (includePrimaryKey && Objects.equals(primaryKeyFieldName, qRecordEntityField.getFieldName()))) { qRecord.setValue(qRecordEntityField.getFieldName(), thisValue); } } - for(QRecordEntityAssociation qRecordEntityAssociation : getAssociationList(this.getClass())) - { - @SuppressWarnings("unchecked") - List associatedEntities = (List) qRecordEntityAssociation.getGetter().invoke(this); - String associationName = qRecordEntityAssociation.getAssociationAnnotation().name(); - - if(associatedEntities != null) - { - ///////////////////////////////////////////////////////////////////////////////// - // do this so an empty list in the entity becomes an empty list in the QRecord // - ///////////////////////////////////////////////////////////////////////////////// - qRecord.withAssociatedRecords(associationName, new ArrayList<>()); - } - - for(QRecordEntity associatedEntity : CollectionUtils.nonNullList(associatedEntities)) - { - qRecord.withAssociatedRecord(associationName, associatedEntity.toQRecord()); - } - } + toQRecordProcessAssociations(qRecord, (entity) -> entity.toQRecordOnlyChangedFields(includePrimaryKey)); return (qRecord); } @@ -488,15 +529,15 @@ public abstract class QRecordEntity { // todo - more types!! return (returnType.equals(String.class) - || returnType.equals(Integer.class) - || returnType.equals(int.class) - || returnType.equals(Boolean.class) - || returnType.equals(boolean.class) - || returnType.equals(BigDecimal.class) - || returnType.equals(Instant.class) - || returnType.equals(LocalDate.class) - || returnType.equals(LocalTime.class) - || returnType.equals(byte[].class)); + || returnType.equals(Integer.class) + || returnType.equals(int.class) + || returnType.equals(Boolean.class) + || returnType.equals(boolean.class) + || returnType.equals(BigDecimal.class) + || returnType.equals(Instant.class) + || returnType.equals(LocalDate.class) + || returnType.equals(LocalTime.class) + || returnType.equals(byte[].class)); ///////////////////////////////////////////// // note - this list has implications upon: // // - QFieldType.fromClass // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index e6078562..93ae4f96 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -68,14 +68,6 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private String name; private String label; - // TODO: resolve confusion over: - // Is this name of what backend the table is stored in (yes) - // Or the "name" of the table WITHIN the backend (no) - // although that's how "backendName" is used in QFieldMetaData. - // Idea: - // rename "backendName" here to "backend" - // add "nameInBackend" (or similar) for the table name in the backend - // OR - add a whole "backendDetails" object, with different details per backend-type private String backendName; private String primaryKeyField; private boolean isHidden = false; @@ -184,6 +176,12 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData } } + /////////////////////////////////////////////////////////////////////////////////////////////////// + // stash a reference from this entityClass to this table in the QRecordEntity class // + // (used within that class later, if it wants to know about a table that an Entity helped build) // + /////////////////////////////////////////////////////////////////////////////////////////////////// + QRecordEntity.registerTable(entityClass, this); + return (this); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java index f038c27c..38beaae4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/QRecordEntityTest.java @@ -24,8 +24,10 @@ package com.kingsrook.qqq.backend.core.model.data; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.testentities.Item; import com.kingsrook.qqq.backend.core.model.data.testentities.ItemWithPrimitives; @@ -35,7 +37,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -49,6 +54,31 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class QRecordEntityTest extends BaseTest { + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws QException + { + QContext.getQInstance().addTable(new QTableMetaData() + .withName(Item.TABLE_NAME) + .withFieldsFromEntity(Item.class) + ); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QContext.getQInstance().getTables().remove(Item.TABLE_NAME); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -68,6 +98,19 @@ class QRecordEntityTest extends BaseTest assertEquals(47, qRecord.getValueInteger("quantity")); assertEquals(new BigDecimal("3.50"), qRecord.getValueBigDecimal("price")); assertTrue(qRecord.getValueBoolean("featured")); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that, if we had no lists of associations in the entity, that we also have none in the record // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + assertThat(qRecord.getAssociatedRecords()).isNullOrEmpty(); + + /////////////////////////////////////////////////////////////////////// + // now assert that an empty list translates through to an empty list // + /////////////////////////////////////////////////////////////////////// + item.setItemAlternates(Collections.emptyList()); + qRecord = item.toQRecord(); + assertTrue(qRecord.getAssociatedRecords().containsKey(Item.ASSOCIATION_ITEM_ALTERNATES_NAME)); + assertTrue(qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME).isEmpty()); } @@ -76,9 +119,40 @@ class QRecordEntityTest extends BaseTest ** *******************************************************************************/ @Test - void testItemToQRecordOnlyChangedFields() throws QException + void testItemToQRecordWithAssociations() throws QException + { + Item item = new Item(); + item.setSku("ABC-123"); + item.setQuantity(47); + item.setItemAlternates(List.of( + new Item().withSku("DEF"), + new Item().withSku("GHI").withQuantity(3) + )); + + QRecord qRecord = item.toQRecord(); + assertEquals("ABC-123", qRecord.getValueString("sku")); + assertEquals(47, qRecord.getValueInteger("quantity")); + + List associatedRecords = qRecord.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(2, associatedRecords.size()); + assertEquals("DEF", associatedRecords.get(0).getValue("sku")); + assertTrue(associatedRecords.get(0).getValues().containsKey("quantity")); + assertNull(associatedRecords.get(0).getValue("quantity")); + assertEquals("GHI", associatedRecords.get(1).getValue("sku")); + assertEquals(3, associatedRecords.get(1).getValue("quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsEntityThatCameFromQRecord() throws QException { Item item = new Item(new QRecord() + .withValue("id", 1701) .withValue("sku", "ABC-123") .withValue("description", null) .withValue("quantity", 47) @@ -88,11 +162,20 @@ class QRecordEntityTest extends BaseTest QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); assertTrue(qRecordOnlyChangedFields.getValues().isEmpty()); + QRecord qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true); + assertEquals(1, qRecordOnlyChangedFieldsIncludePKey.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id")); + item.setDescription("My Changed Item"); - qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false); assertEquals(1, qRecordOnlyChangedFields.getValues().size()); assertEquals("My Changed Item", qRecordOnlyChangedFields.getValueString("description")); + qRecordOnlyChangedFieldsIncludePKey = item.toQRecordOnlyChangedFields(true); + assertEquals(2, qRecordOnlyChangedFieldsIncludePKey.getValues().size()); + assertEquals("My Changed Item", qRecordOnlyChangedFieldsIncludePKey.getValueString("description")); + assertEquals(1701, qRecordOnlyChangedFieldsIncludePKey.getValue("id")); + item.setPrice(null); qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); assertEquals(2, qRecordOnlyChangedFields.getValues().size()); @@ -101,6 +184,81 @@ class QRecordEntityTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsFromNewEntity() throws QException + { + Item item = new Item() + .withId(1701) + .withSku("ABC-123"); + + QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(); + assertEquals(2, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + assertEquals("ABC-123", qRecordOnlyChangedFields.getValue("sku")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("deprecation") + @Test + void testItemToQRecordOnlyChangedFieldsWithAssociations() throws QException + { + Item item = new Item(new QRecord() + .withValue("id", 1701) + .withValue("sku", "ABC-123") + .withAssociatedRecord(Item.ASSOCIATION_ITEM_ALTERNATES_NAME, new Item(new QRecord() + .withValue("id", 1702) + .withValue("sku", "DEF") + .withValue("quantity", 3) + .withValue("price", new BigDecimal("3.50")) + ).toQRecord()) + ); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if no values were changed in the entities, from when they were constructed (from records), then value maps should be empty // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QRecord qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(false); + assertTrue(qRecordOnlyChangedFields.getValues().isEmpty()); + List associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertTrue(associatedRecords.get(0).getValues().isEmpty()); + + /////////////////////////////////////////////////////// + // but - if pkeys are requested, confirm we get them // + /////////////////////////////////////////////////////// + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true); + assertEquals(1, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(1, associatedRecords.get(0).getValues().size()); + assertEquals(1702, associatedRecords.get(0).getValue("id")); + + //////////////////////////////////////////// + // change some properties in the entities // + //////////////////////////////////////////// + item.setDescription("My Changed Item"); + item.getItemAlternates().get(0).setQuantity(4); + item.getItemAlternates().get(0).setPrice(null); + + qRecordOnlyChangedFields = item.toQRecordOnlyChangedFields(true); + assertEquals(2, qRecordOnlyChangedFields.getValues().size()); + assertEquals(1701, qRecordOnlyChangedFields.getValue("id")); + assertEquals("My Changed Item", qRecordOnlyChangedFields.getValue("description")); + associatedRecords = qRecordOnlyChangedFields.getAssociatedRecords().get(Item.ASSOCIATION_ITEM_ALTERNATES_NAME); + assertEquals(3, associatedRecords.get(0).getValues().size()); + assertEquals(1702, associatedRecords.get(0).getValue("id")); + assertEquals(4, associatedRecords.get(0).getValue("quantity")); + assertNull(associatedRecords.get(0).getValue("price")); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java index 51be0f3e..7efb8270 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/data/testentities/Item.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.model.data.testentities; import java.math.BigDecimal; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.data.QAssociation; import com.kingsrook.qqq.backend.core.model.data.QField; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; @@ -34,6 +36,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; *******************************************************************************/ public class Item extends QRecordEntity { + public static final String TABLE_NAME = "item"; + + public static final String ASSOCIATION_ITEM_ALTERNATES_NAME = "itemAlternates"; + + @QField(isPrimaryKey = true) + private Integer id; + @QField(isRequired = true, label = "SKU") private String sku; @@ -49,6 +58,9 @@ public class Item extends QRecordEntity @QField(backendName = "is_featured") private Boolean featured; + @QAssociation(name = ASSOCIATION_ITEM_ALTERNATES_NAME) + private List itemAlternates; + /******************************************************************************* @@ -179,4 +191,122 @@ public class Item extends QRecordEntity { this.featured = featured; } + + + + /******************************************************************************* + ** Fluent setter for sku + *******************************************************************************/ + public Item withSku(String sku) + { + this.sku = sku; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for description + *******************************************************************************/ + public Item withDescription(String description) + { + this.description = description; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for quantity + *******************************************************************************/ + public Item withQuantity(Integer quantity) + { + this.quantity = quantity; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for price + *******************************************************************************/ + public Item withPrice(BigDecimal price) + { + this.price = price; + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for featured + *******************************************************************************/ + public Item withFeatured(Boolean featured) + { + this.featured = featured; + return (this); + } + + + + /******************************************************************************* + ** Getter for itemAlternates + *******************************************************************************/ + public List getItemAlternates() + { + return (this.itemAlternates); + } + + + + /******************************************************************************* + ** Setter for itemAlternates + *******************************************************************************/ + public void setItemAlternates(List itemAlternates) + { + this.itemAlternates = itemAlternates; + } + + + + /******************************************************************************* + ** Fluent setter for itemAlternates + *******************************************************************************/ + public Item withItemAlternates(List itemAlternates) + { + this.itemAlternates = itemAlternates; + return (this); + } + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public Item withId(Integer id) + { + this.id = id; + return (this); + } + + } From db526009d2b1d353b97eb70dab00710f2dcfd534 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:27:21 -0600 Subject: [PATCH 062/209] CE-1955 - more flexible handling of inbound types for looking up possible values --- .../insert/mapping/BulkLoadValueMapper.java | 46 +++++++++++++++---- .../mapping/BulkLoadValueMapperTest.java | 43 +++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index cc44e027..4fa7a414 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; @@ -170,20 +171,44 @@ public class BulkLoadValueMapper ***************************************************************************/ private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException { - Set values = fieldPossibleValueToRecordMap.keySet(); - Map> valuesFound = new HashMap<>(); - Set valuesNotFound = new HashSet<>(values); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); + + Set values = fieldPossibleValueToRecordMap.keySet(); + Map valuesToValueInPvsIdTypeMap = new HashMap<>(); + Map> valuesFound = new HashMap<>(); + Set valuesNotFound = new HashSet<>(); //////////////////////////////////////////////////////// // do a search, trying to use all given values as ids // //////////////////////////////////////////////////////// SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList idList = new ArrayList<>(values); + + ArrayList idList = new ArrayList<>(); + for(String value : values) + { + Serializable valueInPvsIdType = value; + + try + { + valueInPvsIdType = ValueUtils.getValueAsFieldType(possibleValueSource.getIdType(), value); + } + catch(Exception e) + { + //////////////////////////// + // leave as original type // + //////////////////////////// + } + + valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType); + idList.add(valueInPvsIdType); + valuesNotFound.add(valueInPvsIdType); + } + searchPossibleValueSourceInput.setIdList(idList); searchPossibleValueSourceInput.setLimit(values.size()); LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); - SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); //////////////////////////////////////////////////////////////////////////////////////////////////// // for each possible value found, remove it from the set of ones not-found, and store it as a hit // @@ -202,7 +227,7 @@ public class BulkLoadValueMapper { searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList labelList = new ArrayList<>(valuesNotFound); + List labelList = valuesNotFound.stream().map(ValueUtils::getValueAsString).toList(); searchPossibleValueSourceInput.setLabelList(labelList); searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); @@ -220,12 +245,15 @@ public class BulkLoadValueMapper //////////////////////////////////////////////////////////////////////////////// for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) { - String value = entry.getKey(); + String value = entry.getKey(); + Serializable valueInPvsIdType = valuesToValueInPvsIdTypeMap.get(entry.getKey()); + String pvsIdAsString = ValueUtils.getValueAsString(valueInPvsIdType); + for(QRecord record : entry.getValue()) { - if(valuesFound.containsKey(value)) + if(valuesFound.containsKey(pvsIdAsString)) { - record.setValue(field.getName(), valuesFound.get(value).getId()); + record.setValue(field.getName(), valuesFound.get(pvsIdAsString).getId()); } else { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 687f1f62..26a4528f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -36,6 +37,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -113,6 +115,47 @@ class BulkLoadValueMapperTest extends BaseTest + /*************************************************************************** + ** + ***************************************************************************/ + void testPossibleValue(Serializable inputValue, Serializable expectedValue, boolean expectErrors) throws QException + { + QRecord inputRecord = new QRecord().withValue("homeStateId", inputValue); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), new BulkInsertMapping(), QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)); + assertEquals(expectedValue, inputRecord.getValue("homeStateId")); + + if(expectErrors) + { + assertThat(inputRecord.getErrors().get(0)).isInstanceOf(BulkLoadPossibleValueError.class); + } + else + { + assertThat(inputRecord.getErrors()).isNullOrEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValues() throws QException + { + testPossibleValue(1, 1, false); + testPossibleValue("1", 1, false); + testPossibleValue("1.0", 1, false); + testPossibleValue(new BigDecimal("1.0"), 1, false); + testPossibleValue("IL", 1, false); + + testPossibleValue(512, 512, true); // an id, but not in the PVS + testPossibleValue("USA", "USA", true); + testPossibleValue(true, true, true); + testPossibleValue(new BigDecimal("4.7"), new BigDecimal("4.7"), true); + } + + + /*************************************************************************** ** ***************************************************************************/ From 2b0b176ced8219614b8f2f2934fae8a6871723c9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:27:47 -0600 Subject: [PATCH 063/209] CE-1955 - only handle a single level deep of associations... --- .../mapping/BulkLoadTableStructureBuilder.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java index 0ac81f00..26575be3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -123,8 +123,18 @@ public class BulkLoadTableStructureBuilder for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) { - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); - tableStructure.addAssociation(associatedStructure); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at this time, we are not prepared to handle 3-level deep associations, so, only process them from the top level... // + // main challenge being, wide-mode. so, maybe we should just only support 3-level+ associations for tall? // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(association == null) + { + String nextLevelPath = + (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") + + (association != null ? association.getName() : ""); + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); + tableStructure.addAssociation(associatedStructure); + } } return (tableStructure); From 7e475e2c18e29eb6bc3be6f40bed151fa972586d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:29:22 -0600 Subject: [PATCH 064/209] CE-1955 - add idType to possibleValueSource - used by bulk load possible-value mapping --- .../core/instances/QInstanceEnricher.java | 60 +++++++++++++++++++ .../core/instances/QInstanceValidator.java | 2 + .../possiblevalues/QPossibleValueSource.java | 34 +++++++++++ 3 files changed, 96 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 1d88cb7e..a0d926c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -33,8 +36,10 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -89,6 +94,7 @@ import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -1347,6 +1353,60 @@ public class QInstanceEnricher } } + if(possibleValueSource.getIdType() == null) + { + QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName()); + if(table != null) + { + String primaryKeyField = table.getPrimaryKeyField(); + QFieldMetaData primaryKeyFieldMetaData = table.getFields().get(primaryKeyField); + if(primaryKeyFieldMetaData != null) + { + possibleValueSource.setIdType(primaryKeyFieldMetaData.getType()); + } + } + } + } + else if(QPossibleValueSourceType.ENUM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + if(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues())) + { + Object id = possibleValueSource.getEnumValues().get(0).getId(); + try + { + possibleValueSource.setIdType(QFieldType.fromClass(id.getClass())); + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first enum value", e, logPair("possibleValueSource", possibleValueSource.getName()), logPair("id", id)); + } + } + } + } + else if(QPossibleValueSourceType.CUSTOM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + try + { + QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); + + Method getPossibleValueMethod = customPossibleValueProvider.getClass().getDeclaredMethod("getPossibleValue", Serializable.class); + Type returnType = getPossibleValueMethod.getGenericReturnType(); + Type idType = ((ParameterizedType) returnType).getActualTypeArguments()[0]; + + if(idType instanceof Class c) + { + possibleValueSource.setIdType(QFieldType.fromClass(c)); + } + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first custom value", e, logPair("possibleValueSource", possibleValueSource.getName())); + } + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 58618c3e..a1edd5a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -2075,6 +2075,8 @@ public class QInstanceValidator default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType()); } + assertCondition(possibleValueSource.getIdType() != null, "possibleValueSource " + name + " is missing its idType."); + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java index e8fc860b..00b121b4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* @@ -42,6 +43,8 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface private String label; private QPossibleValueSourceType type; + private QFieldType idType; + private String valueFormat = PVSValueFormatAndFields.LABEL_ONLY.getFormat(); private List valueFields = PVSValueFormatAndFields.LABEL_ONLY.getFields(); private String valueFormatIfNotFound = null; @@ -662,4 +665,35 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface return (this); } + + /******************************************************************************* + ** Getter for idType + *******************************************************************************/ + public QFieldType getIdType() + { + return (this.idType); + } + + + + /******************************************************************************* + ** Setter for idType + *******************************************************************************/ + public void setIdType(QFieldType idType) + { + this.idType = idType; + } + + + + /******************************************************************************* + ** Fluent setter for idType + *******************************************************************************/ + public QPossibleValueSource withIdType(QFieldType idType) + { + this.idType = idType; + return (this); + } + + } From 6b7d3ac26da42f83440bf6e563d8b9f1776b6dae Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 18:53:27 -0600 Subject: [PATCH 065/209] CE-1955 propagate errors from child (association) records up to main record --- .../bulk/insert/BulkInsertTransformStep.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index bd1fb40c..f113f76f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -197,6 +197,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithSomeErrors = new ArrayList<>(); for(QRecord record : runBackendStepInput.getRecords()) { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + if(CollectionUtils.nullSafeHasContents(record.getErrors())) { recordsWithSomeErrors.add(record); @@ -351,6 +359,34 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + ** + ***************************************************************************/ + private List getErrorsFromAssociations(QRecord record) + { + List rs = null; + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + for(QRecord associatedRecord : CollectionUtils.nonNullList(entry.getValue())) + { + if(CollectionUtils.nullSafeHasContents(associatedRecord.getErrors())) + { + rs = Objects.requireNonNullElseGet(rs, () -> new ArrayList<>()); + rs.addAll(associatedRecord.getErrors()); + + List childErrors = getErrorsFromAssociations(associatedRecord); + if(CollectionUtils.nullSafeHasContents(childErrors)) + { + rs.addAll(childErrors); + } + } + } + } + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ From 9cfc7fafc12c29dceaafb466086822425ba72e0b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:08:01 -0600 Subject: [PATCH 066/209] CE-1955 case-insenitiveKey map, to help with bulk load possible value case-insensitvity --- .../collections/CaseInsensitiveKeyMap.java | 53 +++ .../utils/collections/TransformedKeyMap.java | 400 ++++++++++++++++++ .../CaseInsensitiveKeyMapTest.java | 52 +++ .../collections/TransformedKeyMapTest.java | 199 +++++++++ 4 files changed, 704 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java new file mode 100644 index 00000000..8cbcb54d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Map; +import java.util.function.Supplier; + + +/******************************************************************************* + ** Version of map where string keys are handled case-insensitively. e.g., + ** map.put("One", 1); map.get("ONE") == 1. + *******************************************************************************/ +public class CaseInsensitiveKeyMap extends TransformedKeyMap +{ + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap() + { + super(key -> key.toLowerCase()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap(Supplier> supplier) + { + super(key -> key.toLowerCase(), supplier); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java new file mode 100644 index 00000000..afe1116d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java @@ -0,0 +1,400 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +/******************************************************************************* + ** Version of a map that uses a transformation function on keys. The original + ** idea being, e.g., to support case-insensitive keys via a toLowerCase transform. + ** e.g., map.put("One", 1); map.get("ONE") == 1. + ** + ** But, implemented generically to support any transformation function. + ** + ** keySet() and entries() should give only the first version of a key that overlapped. + ** e.g., map.put("One", 1); map.put("one", 1); map.keySet() == Set.of("One"); + *******************************************************************************/ +public class TransformedKeyMap implements Map +{ + private Function keyTransformer; + private Map wrappedMap; + + private Map originalKeys = new HashMap<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = new HashMap<>(); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer, Supplier> supplier) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = supplier.get(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int size() + { + return (wrappedMap.size()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEmpty() + { + return (wrappedMap.isEmpty()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsKey(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.containsKey(transformed); + } + catch(Exception e) + { + return (false); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsValue(Object value) + { + return (wrappedMap.containsValue(value)); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V get(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.get(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @Nullable V put(OK key, V value) + { + TK transformed = keyTransformer.apply(key); + originalKeys.putIfAbsent(transformed, key); + return wrappedMap.put(transformed, value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V remove(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + originalKeys.remove(transformed); + return wrappedMap.remove(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void putAll(@NotNull Map m) + { + for(Entry entry : m.entrySet()) + { + put(entry.getKey(), entry.getValue()); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void clear() + { + wrappedMap.clear(); + originalKeys.clear(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set keySet() + { + return new HashSet<>(originalKeys.values()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Collection values() + { + return wrappedMap.values(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set> entrySet() + { + Set> wrappedEntries = wrappedMap.entrySet(); + Set> originalEntries; + try + { + originalEntries = wrappedEntries.getClass().getConstructor().newInstance(); + } + catch(Exception e) + { + originalEntries = new HashSet<>(); + } + + for(Entry wrappedEntry : wrappedEntries) + { + OK originalKey = originalKeys.get(wrappedEntry.getKey()); + originalEntries.add(new TransformedKeyMapEntry<>(originalKey, wrappedEntry.getValue())); + } + + return (originalEntries); + } + + // methods with a default implementation below here // + + + + /* + @Override + public V getOrDefault(Object key, V defaultValue) + { + return Map.super.getOrDefault(key, defaultValue); + } + + + + @Override + public void forEach(BiConsumer action) + { + Map.super.forEach(action); + } + + + + @Override + public void replaceAll(BiFunction function) + { + Map.super.replaceAll(function); + } + + + + @Override + public @Nullable V putIfAbsent(OK key, V value) + { + return Map.super.putIfAbsent(key, value); + } + + + + @Override + public boolean remove(Object key, Object value) + { + return Map.super.remove(key, value); + } + + + + @Override + public boolean replace(OK key, V oldValue, V newValue) + { + return Map.super.replace(key, oldValue, newValue); + } + + + + @Override + public @Nullable V replace(OK key, V value) + { + return Map.super.replace(key, value); + } + + + + @Override + public V computeIfAbsent(OK key, @NotNull Function mappingFunction) + { + return Map.super.computeIfAbsent(key, mappingFunction); + } + + + + @Override + public V computeIfPresent(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.computeIfPresent(key, remappingFunction); + } + + + + @Override + public V compute(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.compute(key, remappingFunction); + } + + + + @Override + public V merge(OK key, @NotNull V value, @NotNull BiFunction remappingFunction) + { + return Map.super.merge(key, value, remappingFunction); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public static class TransformedKeyMapEntry implements Map.Entry + { + private final EK key; + private EV value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMapEntry(EK key, EV value) + { + this.key = key; + this.value = value; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EK getKey() + { + return (key); + } + + + + @Override + public EV getValue() + { + return (value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public EV setValue(EV value) + { + throw (new UnsupportedOperationException("Setting value in an entry of a TransformedKeyMap is not supported.")); + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java new file mode 100644 index 00000000..63919551 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CaseInsensitiveKeyMap + *******************************************************************************/ +class CaseInsensitiveKeyMapTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + CaseInsensitiveKeyMap map = new CaseInsensitiveKeyMap<>(); + map.put("One", 1); + map.put("one", 1); + map.put("ONE", 1); + assertEquals(1, map.get("one")); + assertEquals(1, map.get("One")); + assertEquals(1, map.get("oNe")); + assertEquals(1, map.size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java new file mode 100644 index 00000000..f319706d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java @@ -0,0 +1,199 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.math.BigDecimal; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TransformedKeyMap + *******************************************************************************/ +@SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" }) +class TransformedKeyMapTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCaseInsensitiveKeyMap() + { + TransformedKeyMap caseInsensitiveKeys = new TransformedKeyMap<>(key -> key.toLowerCase()); + caseInsensitiveKeys.put("One", 1); + caseInsensitiveKeys.put("one", 1); + caseInsensitiveKeys.put("ONE", 1); + assertEquals(1, caseInsensitiveKeys.get("one")); + assertEquals(1, caseInsensitiveKeys.get("One")); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(1, caseInsensitiveKeys.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", caseInsensitiveKeys.entrySet().iterator().next().getKey()); + assertEquals("One", caseInsensitiveKeys.keySet().iterator().next()); + + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + + for(String key : caseInsensitiveKeys.keySet()) + { + assertEquals(1, caseInsensitiveKeys.get(key)); + } + + for(Map.Entry entry : caseInsensitiveKeys.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(1, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + caseInsensitiveKeys.put("Two", 2); + assertEquals(2, caseInsensitiveKeys.size()); + assertEquals(2, caseInsensitiveKeys.entrySet().size()); + assertEquals(2, caseInsensitiveKeys.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + caseInsensitiveKeys.remove("TWO"); + assertNull(caseInsensitiveKeys.get("Two")); + assertNull(caseInsensitiveKeys.get("two")); + assertEquals(1, caseInsensitiveKeys.size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + caseInsensitiveKeys.clear(); + assertNull(caseInsensitiveKeys.get("one")); + assertEquals(0, caseInsensitiveKeys.size()); + assertEquals(0, caseInsensitiveKeys.keySet().size()); + assertEquals(0, caseInsensitiveKeys.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + caseInsensitiveKeys.putAll(Map.of("One", 1, "one", 1, "ONE", 1, "TwO", 2, "tWo", 2, "three", 3)); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(2, caseInsensitiveKeys.get("two")); + assertEquals(3, caseInsensitiveKeys.get("Three")); + assertEquals(3, caseInsensitiveKeys.size()); + assertEquals(3, caseInsensitiveKeys.entrySet().size()); + assertEquals(3, caseInsensitiveKeys.keySet().size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStringToNumberMap() + { + BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); + BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); + + TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> + switch (key.toLowerCase()) + { + case "one", "uno", "eins" -> 1; + case "two", "dos", "zwei" -> 2; + case "three", "tres", "drei" -> 3; + default -> null; + }); + multiLingualWordToNumber.put("One", BigDecimal.ONE); + multiLingualWordToNumber.put("uno", BigDecimal.ONE); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("uno")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("eins")); + assertEquals(1, multiLingualWordToNumber.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", multiLingualWordToNumber.entrySet().iterator().next().getKey()); + assertEquals("One", multiLingualWordToNumber.keySet().iterator().next()); + + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + + for(String key : multiLingualWordToNumber.keySet()) + { + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get(key)); + } + + for(Map.Entry entry : multiLingualWordToNumber.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(BigDecimal.ONE, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + multiLingualWordToNumber.put("Two", BIG_DECIMAL_TWO); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("Dos")); + assertEquals(2, multiLingualWordToNumber.size()); + assertEquals(2, multiLingualWordToNumber.entrySet().size()); + assertEquals(2, multiLingualWordToNumber.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + multiLingualWordToNumber.remove("ZWEI"); + assertNull(multiLingualWordToNumber.get("Two")); + assertNull(multiLingualWordToNumber.get("Dos")); + assertEquals(1, multiLingualWordToNumber.size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + multiLingualWordToNumber.clear(); + assertNull(multiLingualWordToNumber.get("eins")); + assertNull(multiLingualWordToNumber.get("One")); + assertEquals(0, multiLingualWordToNumber.size()); + assertEquals(0, multiLingualWordToNumber.keySet().size()); + assertEquals(0, multiLingualWordToNumber.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei",BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe")); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos")); + assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei")); + assertEquals(3, multiLingualWordToNumber.size()); + assertEquals(3, multiLingualWordToNumber.entrySet().size()); + assertEquals(3, multiLingualWordToNumber.keySet().size()); + } + +} \ No newline at end of file From a4499219c8837e712356c722e036746555b3cad7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:09:41 -0600 Subject: [PATCH 067/209] CE-1955 Update fastexcel version; Update XlsxFileToRows to read formats, and then do a better job of handling numbers as date-time, date, int, or decimal (hopefully) --- qqq-backend-core/pom.xml | 2 +- .../insert/filehandling/XlsxFileToRows.java | 192 ++++++++++++++---- .../filehandling/XlsxFileToRowsTest.java | 59 +++++- 3 files changed, 210 insertions(+), 43 deletions(-) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 0944e1e9..21723b83 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -100,7 +100,7 @@ org.dhatim fastexcel - 0.12.15 + 0.18.4 org.dhatim diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index 206ca722..de0b1ccb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -26,14 +26,16 @@ import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.math.BigDecimal; -import java.math.MathContext; -import java.time.LocalDateTime; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.ReadingOptions; +import org.dhatim.fastexcel.reader.Row; import org.dhatim.fastexcel.reader.Sheet; @@ -42,6 +44,10 @@ import org.dhatim.fastexcel.reader.Sheet; *******************************************************************************/ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface { + private static final QLogger LOG = QLogger.getLogger(XlsxFileToRows.class); + + private static final Pattern DAY_PATTERN = Pattern.compile(".*\\b(d|dd)\\b.*"); + private ReadableWorkbook workbook; private Stream rows; @@ -55,7 +61,7 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - Optional dateTime = readerRow.getCellAsDate(i); - if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100) - { - yield dateTime.get(); - } - - Optional optionalBigDecimal = readerRow.getCellAsNumber(i); - if(optionalBigDecimal.isPresent()) - { - BigDecimal bigDecimal = optionalBigDecimal.get(); - if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0) - { - yield bigDecimal.intValue(); - } - - yield bigDecimal; - } - - yield (null); - } - case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null); - case STRING, FORMULA -> cell.getText(); - case EMPTY, ERROR -> null; - }; - } + values[i] = processCell(readerRow, i); } return new BulkLoadFileRow(values, getRowNo()); @@ -121,6 +93,150 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + { + ///////////////////////////////////////////////////////////////////////////////////// + // dates, date-times, integers, and decimals are all identified as type = "number" // + // so go through this process to try to identify what user means it as // + ///////////////////////////////////////////////////////////////////////////////////// + if(isDateTimeFormat(dataFormatString)) + { + //////////////////////////////////////////////////////////////////////////////////////// + // first - if it has a date-time looking format string, then treat it as a date-time. // + //////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate()); + } + else if(isDateFormat(dataFormatString)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // second, if it has a date looking format string (which is a sub-set of date-time), then treat as date. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate().toLocalDate()); + } + else + { + //////////////////////////////////////////////////////////////////////////////////////// + // now assume it's a number - but in case this optional is empty (why?) return a null // + //////////////////////////////////////////////////////////////////////////////////////// + Optional bigDecimal = readerRow.getCellAsNumber(columnIndex); + if(bigDecimal.isEmpty()) + { + return (null); + } + + try + { + //////////////////////////////////////////////////////////// + // now if the bigDecimal is an exact integer, return that // + //////////////////////////////////////////////////////////// + Integer i = bigDecimal.get().intValueExact(); + return (i); + } + catch(ArithmeticException e) + { + ///////////////////////////////// + // else, end up with a decimal // + ///////////////////////////////// + return (bigDecimal.get()); + } + } + } + case STRING -> + { + return cell.asString(); + } + case BOOLEAN -> + { + return cell.asBoolean(); + } + case EMPTY, ERROR, FORMULA -> + { + LOG.debug("cell type: " + cell.getType() + " had value string: " + cell.asString()); + return (null); + } + default -> + { + return (null); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateTimeFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString) && hasHour(dataFormatString)) + { + return (true); + } + + return false; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasHour(String dataFormatString) + { + return dataFormatString.contains("h"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasDay(String dataFormatString) + { + return DAY_PATTERN.matcher(dataFormatString).matches(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString)) + { + return (true); + } + + return false; + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index 95052d56..8d144ad6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.Month; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -45,6 +44,7 @@ import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -61,13 +61,14 @@ class XlsxFileToRowsTest extends BaseTest { byte[] byteArray = writeExcelBytes(); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray)); + FileToRowsInterface fileToRowsInterface = new XlsxFileToRows(); + fileToRowsInterface.init(new ByteArrayInputStream(byteArray)); BulkLoadFileRow headerRow = fileToRowsInterface.next(); BulkLoadFileRow bodyRow = fileToRowsInterface.next(); - assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name", "Birth Date"}, 1), headerRow); - assertEquals(new BulkLoadFileRow(new Serializable[] {1, "Darin", "Jonson", LocalDateTime.of(1980, Month.JANUARY, 31, 0, 0)}, 2), bodyRow); + assertEquals(new BulkLoadFileRow(new String[] { "Id", "First Name", "Last Name", "Birth Date" }, 1), headerRow); + assertEquals(new BulkLoadFileRow(new Serializable[] { 1, "Darin", "Jonson", LocalDate.of(1980, Month.JANUARY, 31) }, 2), bodyRow); /////////////////////////////////////////////////////////////////////////////////////// // make sure there's at least a limit (less than 20) to how many more rows there are // @@ -107,4 +108,54 @@ class XlsxFileToRowsTest extends BaseTest return byteArray; } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDateTimeFormats() + { + assertFormatDateAndOrDateTime(true, false, "dddd, m/d/yy at h:mm"); + assertFormatDateAndOrDateTime(true, false, "h PM, ddd mmm dd"); + assertFormatDateAndOrDateTime(true, false, "dd/mm/yyyy hh:mm"); + assertFormatDateAndOrDateTime(true, false, "yyyy-mm-dd hh:mm:ss.000"); + assertFormatDateAndOrDateTime(true, false, "hh:mm dd/mm/yyyy"); + + assertFormatDateAndOrDateTime(false, true, "yyyy-mm-dd"); + assertFormatDateAndOrDateTime(false, true, "mmmm d \\[dddd\\]"); + assertFormatDateAndOrDateTime(false, true, "mmm dd, yyyy"); + assertFormatDateAndOrDateTime(false, true, "d-mmm"); + assertFormatDateAndOrDateTime(false, true, "dd.mm.yyyy"); + + assertFormatDateAndOrDateTime(false, false, "yyyy"); + assertFormatDateAndOrDateTime(false, false, "mmm-yyyy"); + assertFormatDateAndOrDateTime(false, false, "hh"); + assertFormatDateAndOrDateTime(false, false, "hh:mm"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void assertFormatDateAndOrDateTime(boolean expectDateTime, boolean expectDate, String format) + { + if(XlsxFileToRows.isDateTimeFormat(format)) + { + assertTrue(expectDateTime, format + " was considered a dateTime, but wasn't expected to."); + assertFalse(expectDate, format + " was considered a dateTime, but was expected to be a date."); + } + else if(XlsxFileToRows.isDateFormat(format)) + { + assertFalse(expectDateTime, format + " was considered a date, but was expected to be a dateTime."); + assertTrue(expectDate, format + " was considered a date, but was expected to."); + } + else + { + assertFalse(expectDateTime, format + " was not considered a dateTime, but was expected to."); + assertFalse(expectDate, format + " was considered a date, but was expected to."); + } + } + } \ No newline at end of file From 7f67eda2e310bd3245640078e874ae74ee660e2f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:11:41 -0600 Subject: [PATCH 068/209] CE-1955 do case-insensitive lookups of possible values by label --- .../insert/mapping/BulkLoadValueMapper.java | 63 ++++++++++++++----- .../mapping/BulkLoadValueMapperTest.java | 4 ++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index 4fa7a414..f6b01561 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.collections.CaseInsensitiveKeyMap; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -167,24 +168,57 @@ public class BulkLoadValueMapper /*************************************************************************** + ** Given a listingHash of Strings from the bulk-load file, to QRecords, + ** we will either: + ** - make sure the value set for the field is valid in the PV type + ** - or put an error in the record (leaving the original value from the file in the field) ** + ** We'll do potentially 2 possible-value searches - the first "by id" - + ** type-converting the input strings to the PV's id type. Then, if any + ** values weren't found by id, a second search by "labels"... which might + ** be a bit suspicious, e.g., if the PV has a multi-field label... ***************************************************************************/ private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException { QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); - Set values = fieldPossibleValueToRecordMap.keySet(); - Map valuesToValueInPvsIdTypeMap = new HashMap<>(); - Map> valuesFound = new HashMap<>(); - Set valuesNotFound = new HashSet<>(); + //////////////////////////////////////////////////////////// + // String from file -> List that have that value // + //////////////////////////////////////////////////////////// + Set values = fieldPossibleValueToRecordMap.keySet(); + + ///////////////////////////////////////////////////////////////////////////////// + // String from file -> Integer (for example) - that is the type-converted // + // version of the PVS's idType (but before any lookups were done with that id) // + // e.g., "42" -> 42 // + // e.g., "SOME_CONST" -> "SOME_CONST" (for PVS w/ string ids) // + ///////////////////////////////////////////////////////////////////////////////// + Map valuesToValueInPvsIdTypeMap = new HashMap<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // String versions of EITHER ids or values found in searchPossibleValueSource call (depending on what was searched by) // + // e.g., "42" -> QPossibleValue(42, "Forty-two") (when searched by id) // + // e.g., "Forty-Two" -> QPossibleValue(42, "Forty-two") (when searched by label) // + // e.g., "SOME_CONST" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by id) // + // e.g., "Some Const" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by label) // + // goal being - file could have "42" or "Forty-Two" (or "forty two") and those would all map to QPossibleValue(42, "Forty-two") // + // or - file could have "SOME_CONST" or "Some Const" (or "some const") and those would all map to QPossibleValue("SOME_CONST", "Some Const") // + // this is also why using CaseInsensitiveKeyMap! // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + CaseInsensitiveKeyMap> valuesFoundAsStrings = new CaseInsensitiveKeyMap<>(); + + ///////////////////////////////////////////////////////// + // String values (from file) that still need looked up // + ///////////////////////////////////////////////////////// + Set valuesNotFound = new HashSet<>(); //////////////////////////////////////////////////////// // do a search, trying to use all given values as ids // //////////////////////////////////////////////////////// + ArrayList idList = new ArrayList<>(); SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList idList = new ArrayList<>(); for(String value : values) { Serializable valueInPvsIdType = value; @@ -202,7 +236,7 @@ public class BulkLoadValueMapper valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType); idList.add(valueInPvsIdType); - valuesNotFound.add(valueInPvsIdType); + valuesNotFound.add(value); } searchPossibleValueSourceInput.setIdList(idList); @@ -211,12 +245,12 @@ public class BulkLoadValueMapper SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); //////////////////////////////////////////////////////////////////////////////////////////////////// - // for each possible value found, remove it from the set of ones not-found, and store it as a hit // + // for each possible value found, store it as a hit, and remove it from the set of ones not-found // //////////////////////////////////////////////////////////////////////////////////////////////////// for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); - valuesFound.put(valueAsString, possibleValue); + valuesFoundAsStrings.put(valueAsString, possibleValue); valuesNotFound.remove(valueAsString); } @@ -227,15 +261,14 @@ public class BulkLoadValueMapper { searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - List labelList = valuesNotFound.stream().map(ValueUtils::getValueAsString).toList(); - searchPossibleValueSourceInput.setLabelList(labelList); - searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); + searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound)); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); // todo - a little sus... leaves no room for any dupes, which, can they happen? - LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0))); + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next())); searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { - valuesFound.put(possibleValue.getLabel(), possibleValue); + valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue); valuesNotFound.remove(possibleValue.getLabel()); } } @@ -251,9 +284,9 @@ public class BulkLoadValueMapper for(QRecord record : entry.getValue()) { - if(valuesFound.containsKey(pvsIdAsString)) + if(valuesFoundAsStrings.containsKey(pvsIdAsString)) { - record.setValue(field.getName(), valuesFound.get(pvsIdAsString).getId()); + record.setValue(field.getName(), valuesFoundAsStrings.get(pvsIdAsString).getId()); } else { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 26a4528f..2435a059 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -148,6 +149,9 @@ class BulkLoadValueMapperTest extends BaseTest testPossibleValue(new BigDecimal("1.0"), 1, false); testPossibleValue("IL", 1, false); + BackendQueryFilterUtils.setCaseSensitive(true); + testPossibleValue("il", 1, false); + testPossibleValue(512, 512, true); // an id, but not in the PVS testPossibleValue("USA", "USA", true); testPossibleValue(true, true, true); From f57df2be86c5dff1720dbcbea0c0cd127944c50c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:12:01 -0600 Subject: [PATCH 069/209] CE-1955 change type-argument to be extends-Serializable --- .../model/metadata/possiblevalues/PossibleValueEnum.java | 5 ++++- .../core/model/metadata/possiblevalues/QPossibleValue.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java index fc61b2df..97585230 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java @@ -22,11 +22,14 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; + + /******************************************************************************* ** Interface to be implemented by enums which can be used as a PossibleValueSource. ** *******************************************************************************/ -public interface PossibleValueEnum +public interface PossibleValueEnum { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java index a36fb981..929a7800 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; + + /******************************************************************************* ** An actual possible value - an id and label. ** ** Type parameter `T` is the type of the id (often Integer, maybe String) *******************************************************************************/ -public class QPossibleValue +public class QPossibleValue { private final T id; private final String label; From 8b00e8c877c1c72dd4072088c12bc4c4e8e2a962 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:58:09 -0600 Subject: [PATCH 070/209] checkstyle --- .../utils/collections/TransformedKeyMap.java | 1 + .../collections/TransformedKeyMapTest.java | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java index afe1116d..ae424e66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java @@ -344,6 +344,7 @@ public class TransformedKeyMap implements Map { return Map.super.merge(key, value, remappingFunction); } + */ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java index f319706d..594aeabc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java @@ -36,6 +36,10 @@ import static org.junit.jupiter.api.Assertions.assertNull; @SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" }) class TransformedKeyMapTest extends BaseTest { + private static final BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); + private static final BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); + + /******************************************************************************* ** @@ -111,23 +115,21 @@ class TransformedKeyMapTest extends BaseTest assertEquals(3, caseInsensitiveKeys.keySet().size()); } + + /******************************************************************************* ** *******************************************************************************/ @Test void testStringToNumberMap() { - BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); - BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); - - TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> - switch (key.toLowerCase()) - { - case "one", "uno", "eins" -> 1; - case "two", "dos", "zwei" -> 2; - case "three", "tres", "drei" -> 3; - default -> null; - }); + TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> switch(key.toLowerCase()) + { + case "one", "uno", "eins" -> 1; + case "two", "dos", "zwei" -> 2; + case "three", "tres", "drei" -> 3; + default -> null; + }); multiLingualWordToNumber.put("One", BigDecimal.ONE); multiLingualWordToNumber.put("uno", BigDecimal.ONE); assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one")); @@ -187,7 +189,7 @@ class TransformedKeyMapTest extends BaseTest ///////////////////////////////////////// // make sure put-all works as expected // ///////////////////////////////////////// - multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei",BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); + multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei", BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe")); assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos")); assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei")); From 21982e8f53571af6e906e0dc2d7f2019557a7d29 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 27 Dec 2024 08:54:22 -0600 Subject: [PATCH 071/209] Remove uncommitted BackendQueryFilterUtils.setCaseSensitive --- .../bulk/insert/mapping/BulkLoadValueMapperTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 2435a059..c73a2147 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -30,7 +30,6 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -148,8 +147,6 @@ class BulkLoadValueMapperTest extends BaseTest testPossibleValue("1.0", 1, false); testPossibleValue(new BigDecimal("1.0"), 1, false); testPossibleValue("IL", 1, false); - - BackendQueryFilterUtils.setCaseSensitive(true); testPossibleValue("il", 1, false); testPossibleValue(512, 512, true); // an id, but not in the PVS From 048ee2e332bfef04b32a8399adad099a7b428681 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 27 Dec 2024 09:08:13 -0600 Subject: [PATCH 072/209] Expand cases hit due to new idType requirement in possible values --- .../core/instances/QInstanceValidatorTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 89c69734..760a0991 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -1185,10 +1185,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have searchFields", "should not have orderByFields", "should not have a customCodeReference", - "is missing enum values"); + "is missing enum values", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_STATE).setEnumValues(new ArrayList<>()), - "is missing enum values"); + "is missing enum values", + "is missing its idType."); } @@ -1213,10 +1215,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have a customCodeReference", "is missing a tableName", "is missing searchFields", - "is missing orderByFields"); + "is missing orderByFields", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setTableName("Not a table"), - "Unrecognized table"); + "Unrecognized table", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setSearchFields(List.of("id", "notAField", "name")), "unrecognized searchField: notAField"); @@ -1244,11 +1248,13 @@ public class QInstanceValidatorTest extends BaseTest "should not have a tableName", "should not have searchFields", "should not have orderByFields", - "is missing a customCodeReference"); + "is missing a customCodeReference", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_CUSTOM).setCustomCodeReference(new QCodeReference()), "missing a code reference name", - "missing a code type"); + "missing a code type", + "is missing its idType."); } From 3fda1a1eda3400570fd66143ed7125ab3e1f3a01 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 12:57:49 -0600 Subject: [PATCH 073/209] CE-1955 Add handling for associations w/ some vs. all values coming from defaults instead of columns; --- .../bulk/insert/mapping/TallRowsToRecord.java | 21 ++++- ...licitFieldNameSuffixIndexBasedMapping.java | 80 +++++++++++++++++-- .../bulk/insert/model/BulkInsertMapping.java | 34 ++++++++ .../insert/mapping/TallRowsToRecordTest.java | 55 +++++++++++++ ...tFieldNameSuffixIndexBasedMappingTest.java | 53 ++++++++++++ 5 files changed, 233 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 84ab1fa7..bf67522c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -32,6 +32,7 @@ import java.util.Optional; import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; *******************************************************************************/ public class TallRowsToRecord implements RowsToRecordInterface { + private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class); + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); - ////////////////////////////////////////////////////// - // get all rows for the main table from the 0th row // - ////////////////////////////////////////////////////// + //////////////////////////////////////////////////////// + // get all values for the main table from the 0th row // + //////////////////////////////////////////////////////// BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { @@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. // + // but, if any fields for this association have a default value - then - make a row using just default values. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName)); + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row))); + break; + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index c4231ea7..86a5faf2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem while(fileToRowsInterface.hasNext() && rs.size() < limit) { BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false); rs.add(record); } @@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem /*************************************************************************** ** may return null, if there were no values in the row for this (sub-wide) record. + ** more specifically: + ** + ** the param `rowOfOnlyDefaultValues` - should be false for the header table, + ** and true for an association iff all mapped fields are using 'default values' + ** (e.g., not values from the file). + ** + ** So this method will return null, indicating "no child row to build" if: + ** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any + ** default values, which, probably never happens! + ** - else (doing a row with at least 1 value from the file) - then, null is + ** returned if there were NO values from the file. + ** + ** The goal here is to support these cases: + ** + ** Case A (a row of not only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = column: sku2 + ** - lineItem.qty,1 = column: qty2 + ** - lineItem.lineNo,1 = Default: 2 + ** then a file row with no values for sku2 & qty2 - we don't want a row + ** in that case (which would only have the default value of lineNo=2) + ** + ** Case B (a row of only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = Default: SUPPLEMENT + ** - lineItem.qty,1 = Default: 1 + ** - lineItem.lineNo,1 = Default: 2 + ** we want every parent (order) to include a 2nd line item - with 3 + ** default values (sku=SUPPLEMENT, qty=q, lineNo=2). + ** ***************************************************************************/ - private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException { ////////////////////////////////////////////////////// // start by building the record with its own fields // @@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem QRecord record = new QRecord(); BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRowFromFile = false; boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { - hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile; + + ///////////////////////////////////////////////////////////////////////////////////// + // for wide mode (different from tall) - allow a row that only has default values. // + // e.g., Line Item (2) might be a default to add to every order // + ///////////////////////////////////////////////////////////////////////////////////// + if(record.getValue(field.getName()) != null) + { + hadAnyValuesInRow = true; + } } - if(!hadAnyValuesInRow) + if(rowOfOnlyDefaultValues) { - return (null); + if(!hadAnyValuesInRow) + { + return (null); + } + } + else + { + if(!hadAnyValuesInRowFromFile) + { + return (null); + } } ///////////////////////////// @@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem // todo - doesn't support grand-children List wideAssociationIndexes = List.of(i); Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + + boolean rowOfOnlyDefaultValues = false; if(fieldIndexes.isEmpty()) { - break; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a // + // child record here if there are any default values - so check for them - and only if they are empty, then break the loop. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes); + if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues)) + { + break; + } + rowOfOnlyDefaultValues = true; } - QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues); if(record != null) { rs.add(record); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index d0164b3f..e391b5f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** get a map of default-values for fields in a given table (at the specified + ** association chain and wide-indexes). Will only include fields using a + ** default value. + ***************************************************************************/ + @JsonIgnore + public Map getFieldDefaultValues(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) throws QException + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + /////////////////////////////////////////////////////////////////////////// + // loop over fields - adding them to the rs if they have a default value // + /////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(defaultValue != null) + { + rs.put(field.getName(), defaultValue); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 126f500b..b59f3b1d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -105,6 +105,61 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } + + + /******************************************************************************* + ** test to show that we can do 1 default line item (child record) for each + ** header record. + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + 2, Ned, Flanders + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku", "NUCLEAR-ROD", + "orderLine.quantity", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 3", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 38488683..62eeb607 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -98,6 +98,59 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1 + 1, Homer, Simpson, DONUT, 12, + 2, Ned, Flanders, + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku,1", "NUCLEAR-ROD", + "orderLine.quantity,1", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + /******************************************************************************* ** *******************************************************************************/ From e9fc5f81d236e39298a8af58ccbf900cb4de4e24 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 12:58:12 -0600 Subject: [PATCH 074/209] CE-1955 Add some room for a PVS search to return duplicates... room to improve here though. --- .../bulk/insert/mapping/BulkLoadValueMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index f6b01561..641e11bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -262,12 +262,13 @@ public class BulkLoadValueMapper searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound)); - searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); // todo - a little sus... leaves no room for any dupes, which, can they happen? + searchPossibleValueSourceInput.setLimit(valuesNotFound.size() * 10); // todo - a little sus... leaves some room for dupes, which, can they happen? LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next())); searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { + // todo - deal with multiple values found - and maybe... if some end up not-found, but some dupes happened, should we try another search, in case we hit the limit? valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue); valuesNotFound.remove(possibleValue.getLabel()); } From 6a5f8fadade8a9cd0235c0224ba3f1a64596137c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 16:44:18 -0600 Subject: [PATCH 075/209] Add support for a list of "queries for new connections", to be ran when new connections are opened --- .../jdbc/BaseC3P0ConnectionCustomizer.java | 110 ++++++++++++++++++ .../jdbc/C3P0PooledConnectionProvider.java | 12 ++ .../rdbms/jdbc/SimpleConnectionProvider.java | 25 +++- .../model/metadata/RDBMSBackendMetaData.java | 33 ++++++ 4 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java new file mode 100644 index 00000000..2e0549d3 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.jdbc; + + +import java.sql.Connection; +import java.sql.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.mchange.v2.c3p0.ConnectionCustomizer; + + +/******************************************************************************* + ** Basic version of a C3P0 Connection Customizer used by QQQ - that does things + ** expected for an RDBMS backend - specifically: + ** - runs queriesForNewConnections, if they are set. + *******************************************************************************/ +public class BaseC3P0ConnectionCustomizer implements ConnectionCustomizer +{ + private static Map> queriesForNewConnections = new HashMap<>(); + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onAcquire(Connection connection, String dataSourceIdentityToken) throws Exception + { + List queries = queriesForNewConnections.get(dataSourceIdentityToken); + if(CollectionUtils.nullSafeHasContents(queries)) + { + for(String sql : queries) + { + Statement statement = connection.createStatement(); + statement.execute(sql); + } + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onDestroy(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onCheckOut(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void onCheckIn(Connection connection, String dataSourceIdentityToken) throws Exception + { + ////////// + // noop // + ////////// + } + + + /*************************************************************************** + * + ***************************************************************************/ + public static void setQueriesForNewConnections(String backendName, List queriesForNewConnections) + { + BaseC3P0ConnectionCustomizer.queriesForNewConnections.put(backendName, queriesForNewConnections); + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java index 1a4740e4..e5ea2e2f 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProvider.java @@ -26,6 +26,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.util.LinkedHashMap; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.ConnectionPoolSettings; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.mchange.v2.c3p0.ComboPooledDataSource; @@ -110,6 +111,17 @@ public class C3P0PooledConnectionProvider implements ConnectionProviderInterface } } + pool.setIdentityToken(backend.getName()); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the backend specifies queries to run for new connections, then set up a connection customizer to run them // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(backend.getQueriesForNewConnections())) + { + BaseC3P0ConnectionCustomizer.setQueriesForNewConnections(backend.getName(), backend.getQueriesForNewConnections()); + pool.setConnectionCustomizerClassName(BaseC3P0ConnectionCustomizer.class.getName()); + } + customizePool(pool); this.connectionPool = pool; diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java index 264f5528..870da925 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleConnectionProvider.java @@ -25,6 +25,8 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import static com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager.getJdbcUrl; @@ -57,7 +59,28 @@ public class SimpleConnectionProvider implements ConnectionProviderInterface public Connection getConnection() throws SQLException { String jdbcURL = getJdbcUrl(backend); - return DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + Connection connection = DriverManager.getConnection(jdbcURL, backend.getUsername(), backend.getPassword()); + + if(CollectionUtils.nullSafeHasContents(backend.getQueriesForNewConnections())) + { + runQueriesForNewConnections(connection); + } + + return (connection); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void runQueriesForNewConnections(Connection connection) throws SQLException + { + for(String sql : backend.getQueriesForNewConnections()) + { + Statement statement = connection.createStatement(); + statement.execute(sql); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 110db12d..b5b623b6 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.module.rdbms.model.metadata; +import java.util.List; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -49,6 +50,8 @@ public class RDBMSBackendMetaData extends QBackendMetaData private RDBMSBackendMetaData readOnlyBackendMetaData; + private List queriesForNewConnections = null; + /////////////////////////////////////////////////////////// // define well-known (and fully supported) vendor values // /////////////////////////////////////////////////////////// @@ -453,4 +456,34 @@ public class RDBMSBackendMetaData extends QBackendMetaData return (this); } + /******************************************************************************* + ** Getter for queriesForNewConnections + *******************************************************************************/ + public List getQueriesForNewConnections() + { + return (this.queriesForNewConnections); + } + + + + /******************************************************************************* + ** Setter for queriesForNewConnections + *******************************************************************************/ + public void setQueriesForNewConnections(List queriesForNewConnections) + { + this.queriesForNewConnections = queriesForNewConnections; + } + + + + /******************************************************************************* + ** Fluent setter for queriesForNewConnections + *******************************************************************************/ + public RDBMSBackendMetaData withQueriesForNewConnections(List queriesForNewConnections) + { + this.queriesForNewConnections = queriesForNewConnections; + return (this); + } + + } From b64efd02463f5da04d1f8cfa6bbe160f40541171 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 16:45:18 -0600 Subject: [PATCH 076/209] Add method buildConnectionString to RDBMSBackendMetaData --- .../module/rdbms/jdbc/ConnectionManager.java | 13 ++++++++++++- .../rdbms/model/metadata/RDBMSBackendMetaData.java | 10 ++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java index 5ec62e34..0aa7d18d 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionManager.java @@ -178,7 +178,18 @@ public class ConnectionManager case RDBMSBackendMetaData.VENDOR_AURORA_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull&useSSL=false"; case RDBMSBackendMetaData.VENDOR_MYSQL -> "jdbc:mysql://" + backend.getHostName() + ":" + backend.getPort() + "/" + backend.getDatabaseName() + "?rewriteBatchedStatements=true&zeroDateTimeBehavior=convertToNull"; case RDBMSBackendMetaData.VENDOR_H2 -> "jdbc:h2:" + backend.getHostName() + ":" + backend.getDatabaseName() + ";MODE=MySQL;DB_CLOSE_DELAY=-1"; - default -> throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor()); + default -> + { + String connectionString = backend.buildConnectionString(); + if(connectionString == null) + { + throw new IllegalArgumentException("Unsupported rdbms backend vendor: " + backend.getVendor() + " (and null returned by backendMetaData.buildConnectionString())"); + } + else + { + yield (connectionString); + } + } }; } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index b5b623b6..28d7dc7a 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -456,6 +456,16 @@ public class RDBMSBackendMetaData extends QBackendMetaData return (this); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public String buildConnectionString() + { + return null; + } + /******************************************************************************* ** Getter for queriesForNewConnections *******************************************************************************/ From aba5b9c5ecf44adeeac090df4b01adc91ff3aa3b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 16:45:50 -0600 Subject: [PATCH 077/209] Move backendType/name into constant --- .../qqq/backend/module/rdbms/RDBMSBackendModule.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java index 5a84d73b..928720dc 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/RDBMSBackendModule.java @@ -56,6 +56,8 @@ public class RDBMSBackendModule implements QBackendModuleInterface { private static final QLogger LOG = QLogger.getLogger(RDBMSBackendModule.class); + public static final String NAME = "rdbms"; + static { QBackendModuleDispatcher.registerBackendModule(new RDBMSBackendModule()); @@ -66,7 +68,7 @@ public class RDBMSBackendModule implements QBackendModuleInterface *******************************************************************************/ public String getBackendType() { - return ("rdbms"); + return NAME; } From dc6d37aad3e8cd8b71ab69fdce4300784dd8eb9f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 16:49:09 -0600 Subject: [PATCH 078/209] Introduce the concept of RDBMSActionStrategyInterface - to use strategy pattern for refinement of how different RDBMS sub-backends may need to behave (e.g., to support SQLite, and FULLTEXT INDEX in MySQL). --- .../model/metadata/RDBMSBackendMetaData.java | 84 ++ .../model/metadata/RDBMSFieldMetaData.java | 166 ++++ .../strategy/BaseRDBMSActionStrategy.java | 873 ++++++++++++++++++ .../MySQLFullTextIndexFieldStrategy.java | 62 ++ .../RDBMSActionStrategyInterface.java | 103 +++ .../module/rdbms/actions/RDBMSActionTest.java | 35 +- 6 files changed, 1320 insertions(+), 3 deletions(-) create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java create mode 100644 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java index 28d7dc7a..277502ed 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSBackendMetaData.java @@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.module.rdbms.model.metadata; import java.util.List; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; /******************************************************************************* @@ -50,6 +54,9 @@ public class RDBMSBackendMetaData extends QBackendMetaData private RDBMSBackendMetaData readOnlyBackendMetaData; + private QCodeReference actionStrategyCodeReference; + private RDBMSActionStrategyInterface actionStrategy; + private List queriesForNewConnections = null; /////////////////////////////////////////////////////////// @@ -466,6 +473,83 @@ public class RDBMSBackendMetaData extends QBackendMetaData return null; } + + + /******************************************************************************* + ** Getter for actionStrategyCodeReference + *******************************************************************************/ + public QCodeReference getActionStrategyCodeReference() + { + return (this.actionStrategyCodeReference); + } + + + + /******************************************************************************* + ** Setter for actionStrategyCodeReference + *******************************************************************************/ + public void setActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + } + + + + /******************************************************************************* + ** Fluent setter for actionStrategyCodeReference + *******************************************************************************/ + public RDBMSBackendMetaData withActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public RDBMSActionStrategyInterface getActionStrategy() + { + if(actionStrategy == null) + { + if(actionStrategyCodeReference != null) + { + actionStrategy = QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, actionStrategyCodeReference); + } + else + { + actionStrategy = new BaseRDBMSActionStrategy(); + } + } + + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategyField() + { + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected void setActionStrategyField(RDBMSActionStrategyInterface actionStrategy) + { + this.actionStrategy = actionStrategy; + } + + /******************************************************************************* ** Getter for queriesForNewConnections *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java new file mode 100644 index 00000000..58e6fca4 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/model/metadata/RDBMSFieldMetaData.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.model.metadata; + + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QSupplementalFieldMetaData; +import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class RDBMSFieldMetaData extends QSupplementalFieldMetaData +{ + private QCodeReference actionStrategyCodeReference; + private RDBMSActionStrategyInterface actionStrategy; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public RDBMSFieldMetaData() + { + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static RDBMSFieldMetaData of(QFieldMetaData field) + { + return ((RDBMSFieldMetaData) field.getSupplementalMetaData(RDBMSBackendModule.NAME)); + } + + + + /******************************************************************************* + ** either get the object attached to a field - or create a new one and attach + ** it to the field, and return that. + *******************************************************************************/ + public static RDBMSFieldMetaData ofOrWithNew(QFieldMetaData field) + { + RDBMSFieldMetaData rdbmsFieldMetaData = of(field); + if(rdbmsFieldMetaData == null) + { + rdbmsFieldMetaData = new RDBMSFieldMetaData(); + field.withSupplementalMetaData(rdbmsFieldMetaData); + } + return (rdbmsFieldMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getType() + { + return (RDBMSBackendModule.NAME); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public RDBMSActionStrategyInterface getActionStrategy() + { + if(actionStrategy == null) + { + if(actionStrategyCodeReference != null) + { + actionStrategy = QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, actionStrategyCodeReference); + } + else + { + return (null); + } + } + + return (actionStrategy); + } + + + + /******************************************************************************* + ** Getter for actionStrategyCodeReference + *******************************************************************************/ + public QCodeReference getActionStrategyCodeReference() + { + return (this.actionStrategyCodeReference); + } + + + + /******************************************************************************* + ** Setter for actionStrategyCodeReference + *******************************************************************************/ + public void setActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + } + + + + /******************************************************************************* + ** Fluent setter for actionStrategyCodeReference + *******************************************************************************/ + public RDBMSFieldMetaData withActionStrategyCodeReference(QCodeReference actionStrategyCodeReference) + { + this.actionStrategyCodeReference = actionStrategyCodeReference; + return (this); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategyField() + { + return (actionStrategy); + } + + + + /*************************************************************************** + * note - protected - meant for sub-classes to use in their implementation of + * getActionStrategy, but not for public use. + ***************************************************************************/ + protected void setActionStrategyField(RDBMSActionStrategyInterface actionStrategy) + { + this.actionStrategy = actionStrategy; + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java new file mode 100644 index 00000000..08309cd3 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java @@ -0,0 +1,873 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseRDBMSActionStrategy implements RDBMSActionStrategyInterface +{ + private static final QLogger LOG = QLogger.getLogger(BaseRDBMSActionStrategy.class); + + private static final int MILLIS_PER_SECOND = 1000; + + public static final int DEFAULT_PAGE_SIZE = 2000; + public static int PAGE_SIZE = DEFAULT_PAGE_SIZE; + + private boolean collectStatistics = false; + private final Map statistics = Collections.synchronizedMap(new HashMap<>()); + + public static final String STAT_QUERIES_RAN = "queriesRan"; + public static final String STAT_BATCHES_RAN = "batchesRan"; + + + + /*************************************************************************** + * + ***************************************************************************/ + public Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field) + { + clause.append(column); + + switch(criterion.getOperator()) + { + case EQUALS -> + { + clause.append(" = ?"); + return (1); + } + case NOT_EQUALS -> + { + clause.append(" != ?"); + return (1); + } + case NOT_EQUALS_OR_IS_NULL -> + { + clause.append(" != ? OR ").append(column).append(" IS NULL "); + return (1); + } + case IN -> + { + if(values.isEmpty()) + { + /////////////////////////////////////////////////////// + // if there are no values, then we want a false here // + /////////////////////////////////////////////////////// + clause.delete(0, clause.length()); + clause.append(" 0 = 1 "); + return (0); + } + else + { + clause.append(" IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + } + case IS_NULL_OR_IN -> + { + clause.append(" IS NULL "); + + if(!values.isEmpty()) + { + clause.append(" OR ").append(column).append(" IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + else + { + return (0); + } + } + case NOT_IN -> + { + if(values.isEmpty()) + { + ////////////////////////////////////////////////////// + // if there are no values, then we want a true here // + ////////////////////////////////////////////////////// + clause.delete(0, clause.length()); + clause.append(" 1 = 1 "); + return (0); + } + else + { + clause.append(" NOT IN (").append(values.stream().map(x -> "?").collect(Collectors.joining(","))).append(")"); + return (values.size()); + } + } + case LIKE -> + { + clause.append(" LIKE ?"); + return (1); + } + case NOT_LIKE -> + { + clause.append(" NOT LIKE ?"); + return (1); + } + case STARTS_WITH -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> s + "%")); + return (1); + } + case ENDS_WITH -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s)); + return (1); + } + case CONTAINS -> + { + clause.append(" LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); + return (1); + } + case NOT_STARTS_WITH -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> s + "%")); + return (1); + } + case NOT_ENDS_WITH -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s)); + return (1); + } + case NOT_CONTAINS -> + { + clause.append(" NOT LIKE ?"); + ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); + return (1); + } + case LESS_THAN -> + { + clause.append(" < ?"); + return (1); + } + case LESS_THAN_OR_EQUALS -> + { + clause.append(" <= ?"); + return (1); + } + case GREATER_THAN -> + { + clause.append(" > ?"); + return (1); + } + case GREATER_THAN_OR_EQUALS -> + { + clause.append(" >= ?"); + return (1); + } + case IS_BLANK -> + { + clause.append(" IS NULL"); + if(field.getType().isStringLike()) + { + clause.append(" OR ").append(column).append(" = ''"); + } + return (0); + } + case IS_NOT_BLANK -> + { + clause.append(" IS NOT NULL"); + if(field.getType().isStringLike()) + { + clause.append(" AND ").append(column).append(" != ''"); + } + return (0); + } + case BETWEEN -> + { + clause.append(" BETWEEN ? AND ?"); + return (2); + } + case NOT_BETWEEN -> + { + clause.append(" NOT BETWEEN ? AND ?"); + return (2); + } + case TRUE -> + { + clause.delete(0, clause.length()); + clause.append(" 1 = 1 "); + return (0); + } + case FALSE -> + { + clause.delete(0, clause.length()); + clause.append(" 0 = 1 "); + return (0); + } + default -> throw new IllegalStateException("Unexpected operator: " + criterion.getOperator()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException + { + return switch(type) + { + case STRING, TEXT, HTML, PASSWORD -> (QueryManager.getString(resultSet, i)); + case INTEGER -> (QueryManager.getInteger(resultSet, i)); + case LONG -> (QueryManager.getLong(resultSet, i)); + case DECIMAL -> (QueryManager.getBigDecimal(resultSet, i)); + case DATE -> (QueryManager.getDate(resultSet, i));// todo - queryManager.getLocalDate? + case TIME -> (QueryManager.getLocalTime(resultSet, i)); + case DATE_TIME -> (QueryManager.getInstant(resultSet, i)); + case BOOLEAN -> (QueryManager.getBoolean(resultSet, i)); + case BLOB -> (QueryManager.getByteArray(resultSet, i)); + default -> throw new IllegalStateException("Unexpected field type: " + type); + }; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException + { + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, new List[] { params })) + { + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + return (statement); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException + { + for(List> page : CollectionUtils.getPages(values, PAGE_SIZE)) + { + PreparedStatement updatePS = connection.prepareStatement(updateSQL); + for(List row : page) + { + Object[] params = new Object[row.size()]; + for(int i = 0; i < row.size(); i++) + { + params[i] = row.get(i); + } + + bindParams(params, updatePS); + updatePS.addBatch(); + } + incrementStatistic(STAT_BATCHES_RAN); + updatePS.executeBatch(); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException + { + try(PreparedStatement statement = connection.prepareStatement(sql, new String[] { getColumnName(primaryKeyField) })) + { + bindParams(params.toArray(), statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.executeUpdate(); + + ResultSet generatedKeys = statement.getGeneratedKeys(); + List rs = new ArrayList<>(); + while(generatedKeys.next()) + { + rs.add(getFieldValueFromResultSet(primaryKeyField.getType(), generatedKeys, 1)); + } + return (rs); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer executeUpdateForRowCount(Connection connection, String sql, Object... params) throws SQLException + { + try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) + { + incrementStatistic(STAT_QUERIES_RAN); + int rowCount = statement.executeUpdate(); + return (rowCount); + } + catch(SQLException e) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + throw (e); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException + { + ResultSet resultSet = null; + + try + { + bindParams(params, statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.execute(); + resultSet = statement.getResultSet(); + + if(processor != null) + { + processor.processResultSet(resultSet); + } + } + catch(SQLException e) + { + if(sql != null) + { + LOG.warn("SQLException", e, logPair("sql", sql)); + } + throw (e); + } + finally + { + if(resultSet != null) + { + resultSet.close(); + } + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer getPageSize(AbstractActionInput actionInput) + { + return PAGE_SIZE; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected PreparedStatement prepareStatementAndBindParams(Connection connection, String sql, Object[] params) throws SQLException + { + PreparedStatement statement = connection.prepareStatement(sql); + bindParams(params, statement); + return statement; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void bindParams(Object[] params, PreparedStatement statement) throws SQLException + { + int paramIndex = 0; + if(params != null) + { + for(Object param : params) + { + int paramsBound = bindParamObject(statement, (paramIndex + 1), param); + paramIndex += paramsBound; + } + } + } + + + + /******************************************************************************* + * index is 1-based!! + *******************************************************************************/ + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value instanceof Integer i) + { + bindParam(statement, index, i); + return (1); + } + else if(value instanceof Short s) + { + bindParam(statement, index, s.intValue()); + return (1); + } + else if(value instanceof Long l) + { + bindParam(statement, index, l); + return (1); + } + else if(value instanceof Double d) + { + bindParam(statement, index, d); + return (1); + } + else if(value instanceof String s) + { + bindParam(statement, index, s); + return (1); + } + else if(value instanceof Boolean b) + { + bindParam(statement, index, b); + return (1); + } + else if(value instanceof Timestamp ts) + { + bindParam(statement, index, ts); + return (1); + } + else if(value instanceof Date) + { + bindParam(statement, index, (Date) value); + return (1); + } + else if(value instanceof Calendar c) + { + bindParam(statement, index, c); + return (1); + } + else if(value instanceof BigDecimal bd) + { + bindParam(statement, index, bd); + return (1); + } + else if(value == null) + { + statement.setNull(index, Types.CHAR); + return (1); + } + else if(value instanceof Collection c) + { + int paramsBound = 0; + for(Object o : c) + { + paramsBound += bindParamObject(statement, (index + paramsBound), o); + } + return (paramsBound); + } + else if(value instanceof byte[] ba) + { + statement.setBytes(index, ba); + return (1); + } + else if(value instanceof Instant i) + { + statement.setObject(index, i); + return (1); + } + else if(value instanceof LocalDate ld) + { + @SuppressWarnings("deprecation") + Date date = new Date(ld.getYear() - 1900, ld.getMonthValue() - 1, ld.getDayOfMonth()); + statement.setDate(index, date); + return (1); + } + else if(value instanceof LocalTime lt) + { + @SuppressWarnings("deprecation") + Time time = new Time(lt.getHour(), lt.getMinute(), lt.getSecond()); + statement.setTime(index, time); + return (1); + } + else if(value instanceof OffsetDateTime odt) + { + long epochMillis = odt.toEpochSecond() * MILLIS_PER_SECOND; + Timestamp timestamp = new Timestamp(epochMillis); + statement.setTimestamp(index, timestamp); + return (1); + } + else if(value instanceof LocalDateTime ldt) + { + ZoneOffset offset = OffsetDateTime.now().getOffset(); + long epochMillis = ldt.toEpochSecond(offset) * MILLIS_PER_SECOND; + Timestamp timestamp = new Timestamp(epochMillis); + statement.setTimestamp(index, timestamp); + return (1); + } + else if(value instanceof PossibleValueEnum pve) + { + return (bindParamObject(statement, index, pve.getPossibleValueId())); + } + else + { + throw (new SQLException("Unexpected value type [" + value.getClass().getSimpleName() + "] in bindParamObject.")); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Integer value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.INTEGER); + } + else + { + statement.setInt(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Long value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.INTEGER); + } + else + { + statement.setLong(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Double value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DOUBLE); + } + else + { + statement.setDouble(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, String value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.CHAR); + } + else + { + statement.setString(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Boolean value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.BOOLEAN); + } + else + { + statement.setBoolean(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Date value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + statement.setDate(index, new Date(value.getTime())); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Timestamp value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.TIMESTAMP); + } + else + { + statement.setTimestamp(index, value); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, Calendar value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + statement.setTimestamp(index, new Timestamp(value.getTimeInMillis())); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, LocalDate value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DATE); + } + else + { + LocalDateTime localDateTime = value.atTime(0, 0); + Timestamp timestamp = new Timestamp(localDateTime.atZone(ZoneId.systemDefault()).toEpochSecond() * MILLIS_PER_SECOND); // TimeStamp expects millis, not seconds, after epoch + statement.setTimestamp(index, timestamp); + } + } + + + + /******************************************************************************* + * + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, LocalDateTime value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.TIMESTAMP); + } + else + { + Timestamp timestamp = new Timestamp(value.atZone(ZoneId.systemDefault()).toEpochSecond() * MILLIS_PER_SECOND); // TimeStamp expects millis, not seconds, after epoch + statement.setTimestamp(index, timestamp); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, BigDecimal value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.DECIMAL); + } + else + { + statement.setBigDecimal(index, value); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void bindParam(PreparedStatement statement, int index, byte[] value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.ARRAY); + } + else + { + statement.setBytes(index, value); + } + } + + + + /******************************************************************************* + ** + ** + *******************************************************************************/ + protected void bindParamNull(PreparedStatement statement, int index) throws SQLException + { + statement.setNull(index, Types.NULL); + } + + + + /******************************************************************************* + ** Increment a statistic + ** + *******************************************************************************/ + protected void incrementStatistic(String statName) + { + if(collectStatistics) + { + statistics.putIfAbsent(statName, 0); + statistics.put(statName, statistics.get(statName) + 1); + } + } + + + + /******************************************************************************* + ** Setter for collectStatistics + ** + *******************************************************************************/ + public void setCollectStatistics(boolean collectStatistics) + { + this.collectStatistics = collectStatistics; + } + + + + /******************************************************************************* + ** clear the map of statistics + ** + *******************************************************************************/ + public void resetStatistics() + { + statistics.clear(); + } + + + + /******************************************************************************* + ** Getter for statistics + ** + *******************************************************************************/ + public Map getStatistics() + { + return statistics; + } + + + + /******************************************************************************* + ** Setter for pageSize + ** + *******************************************************************************/ + public void setPageSize(int pageSize) + { + BaseRDBMSActionStrategy.PAGE_SIZE = pageSize; + } + + /******************************************************************************* + ** Get the column name to use for a field in the RDBMS, from the fieldMetaData. + ** + ** That is, field.backendName if set -- else, field.name + *******************************************************************************/ + protected String getColumnName(QFieldMetaData field) + { + if(field.getBackendName() != null) + { + return (field.getBackendName()); + } + return (field.getName()); + } + +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java new file mode 100644 index 00000000..f49ccf04 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategy.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** RDBMS action strategy for a field with a FULLTEXT INDEX on it in a MySQL + ** database. Makes a LIKE or CONTAINS (or NOT those) query use the special + ** syntax that hits the FULLTEXT INDEX. + *******************************************************************************/ +public class MySQLFullTextIndexFieldStrategy extends BaseRDBMSActionStrategy +{ + /*************************************************************************** + * + ***************************************************************************/ + @Override + public Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field) + { + switch(criterion.getOperator()) + { + case LIKE, CONTAINS -> + { + clause.append(" MATCH (").append(column).append(") AGAINST (?) "); + return (1); + } + case NOT_LIKE, NOT_CONTAINS -> + { + clause.append(" NOT MATCH (").append(column).append(") AGAINST (?) "); + return (1); + } + default -> + { + return super.appendCriterionToWhereClause(criterion, clause, column, values, field); + } + } + } +} diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java new file mode 100644 index 00000000..892d0974 --- /dev/null +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/RDBMSActionStrategyInterface.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RDBMSActionStrategyInterface +{ + + /*************************************************************************** + * modifies the clause StringBuilder (appending to it) + * returning the number of expected number of params to bind + ***************************************************************************/ + Integer appendCriterionToWhereClause(QFilterCriteria criterion, StringBuilder clause, String column, List values, QFieldMetaData field); + + /*************************************************************************** + * + ***************************************************************************/ + Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + PreparedStatement executeUpdate(Connection connection, String sql, List params) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + Integer executeUpdateForRowCount(Connection connection, String sql, Object... params) throws SQLException; + + + /*************************************************************************** + * + ***************************************************************************/ + void executeStatement(PreparedStatement statement, CharSequence sql, ResultSetProcessor processor, Object... params) throws SQLException, QException; + + + /*************************************************************************** + * + ***************************************************************************/ + Integer getPageSize(AbstractActionInput actionInput); + + + /******************************************************************************* + ** + *******************************************************************************/ + @FunctionalInterface + interface ResultSetProcessor + { + /******************************************************************************* + ** + *******************************************************************************/ + void processResultSet(ResultSet rs) throws SQLException, QException; + } +} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java index 08ebe9a9..9e12c9b5 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSActionTest.java @@ -23,10 +23,13 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.sql.Connection; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.module.rdbms.BaseTest; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.AfterEach; @@ -42,9 +45,10 @@ public class RDBMSActionTest extends BaseTest @AfterEach void afterEachRDBMSActionTest() { - QueryManager.resetPageSize(); - QueryManager.resetStatistics(); - QueryManager.setCollectStatistics(false); + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setPageSize(BaseRDBMSActionStrategy.DEFAULT_PAGE_SIZE); + actionStrategy.resetStatistics(); + actionStrategy.setCollectStatistics(false); } @@ -59,6 +63,31 @@ public class RDBMSActionTest extends BaseTest + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategy() + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + BaseRDBMSActionStrategy actionStrategy = (BaseRDBMSActionStrategy) backend.getActionStrategy(); + return actionStrategy; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setCollectStatistics(true); + actionStrategy.resetStatistics(); + return actionStrategy; + } + + + /******************************************************************************* ** *******************************************************************************/ From db1269824ce9fbe6da190f518a366aae1d7fcba5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 16:49:50 -0600 Subject: [PATCH 079/209] Refactor to use RDBMSActionStrategy --- .../rdbms/actions/AbstractRDBMSAction.java | 262 ++++-------------- .../rdbms/actions/RDBMSAggregateAction.java | 4 +- .../rdbms/actions/RDBMSCountAction.java | 4 +- .../rdbms/actions/RDBMSDeleteAction.java | 9 +- .../rdbms/actions/RDBMSInsertAction.java | 13 +- .../rdbms/actions/RDBMSQueryAction.java | 4 +- .../rdbms/actions/RDBMSUpdateAction.java | 8 +- .../rdbms/actions/RDBMSDeleteActionTest.java | 37 ++- .../rdbms/actions/RDBMSInsertActionTest.java | 11 +- .../rdbms/actions/RDBMSUpdateActionTest.java | 23 +- 10 files changed, 108 insertions(+), 267 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java index 68ecce54..e5511ed8 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java @@ -39,8 +39,6 @@ import java.util.ListIterator; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; -import com.kingsrook.qqq.backend.core.actions.ActionHelper; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -63,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.AbstractFilterExpression; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -80,9 +79,10 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSFieldMetaData; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -101,6 +101,9 @@ public abstract class AbstractRDBMSAction private static Memoization doesSelectClauseRequireDistinctMemoization = new Memoization() .withTimeout(Duration.ofDays(365)); + private RDBMSBackendMetaData backendMetaData; + private RDBMSActionStrategyInterface actionStrategy; + /******************************************************************************* @@ -313,9 +316,9 @@ public abstract class AbstractRDBMSAction } joinClauseList.add(escapeIdentifier(baseTableOrAlias) - + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) - + " = " + escapeIdentifier(joinTableOrAlias) - + "." + escapeIdentifier(getColumnName((rightTable.getField(joinOn.getRightField()))))); + + "." + escapeIdentifier(getColumnName(leftTable.getField(joinOn.getLeftField()))) + + " = " + escapeIdentifier(joinTableOrAlias) + + "." + escapeIdentifier(getColumnName((rightTable.getField(joinOn.getRightField()))))); } if(CollectionUtils.nullSafeHasContents(queryJoin.getSecurityCriteria())) @@ -479,172 +482,25 @@ public abstract class AbstractRDBMSAction JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getFieldName()); - List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); - QFieldMetaData field = fieldAndTableNameOrAlias.field(); - String column = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(field)); - String clause = column; - Integer expectedNoOfParams = null; - switch(criterion.getOperator()) - { - case EQUALS -> - { - clause += " = ?"; - expectedNoOfParams = 1; - } - case NOT_EQUALS -> - { - clause += " != ?"; - expectedNoOfParams = 1; - } - case NOT_EQUALS_OR_IS_NULL -> - { - clause += " != ? OR " + column + " IS NULL "; - expectedNoOfParams = 1; - } - case IN -> - { - if(values.isEmpty()) - { - /////////////////////////////////////////////////////// - // if there are no values, then we want a false here // - /////////////////////////////////////////////////////// - clause = " 0 = 1 "; - } - else - { - clause += " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } - } - case IS_NULL_OR_IN -> - { - clause += " IS NULL "; + List values = criterion.getValues() == null ? new ArrayList<>() : new ArrayList<>(criterion.getValues()); + QFieldMetaData field = fieldAndTableNameOrAlias.field(); + String column = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(field)); + StringBuilder clause = new StringBuilder(); - if(!values.isEmpty()) - { - clause += " OR " + column + " IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } - } - case NOT_IN -> + RDBMSActionStrategyInterface actionStrategy = getActionStrategy(); + + RDBMSFieldMetaData rdbmsFieldMetaData = RDBMSFieldMetaData.of(field); + if(rdbmsFieldMetaData != null) + { + RDBMSActionStrategyInterface fieldActionStrategy = rdbmsFieldMetaData.getActionStrategy(); + if(fieldActionStrategy != null) { - if(values.isEmpty()) - { - ////////////////////////////////////////////////////// - // if there are no values, then we want a true here // - ////////////////////////////////////////////////////// - clause = " 1 = 1 "; - } - else - { - clause += " NOT IN (" + values.stream().map(x -> "?").collect(Collectors.joining(",")) + ")"; - } + actionStrategy = fieldActionStrategy; } - case LIKE -> - { - clause += " LIKE ?"; - expectedNoOfParams = 1; - } - case NOT_LIKE -> - { - clause += " NOT LIKE ?"; - expectedNoOfParams = 1; - } - case STARTS_WITH -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - } - case ENDS_WITH -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - } - case CONTAINS -> - { - clause += " LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - } - case NOT_STARTS_WITH -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> s + "%")); - expectedNoOfParams = 1; - } - case NOT_ENDS_WITH -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s)); - expectedNoOfParams = 1; - } - case NOT_CONTAINS -> - { - clause += " NOT LIKE ?"; - ActionHelper.editFirstValue(values, (s -> "%" + s + "%")); - expectedNoOfParams = 1; - } - case LESS_THAN -> - { - clause += " < ?"; - expectedNoOfParams = 1; - } - case LESS_THAN_OR_EQUALS -> - { - clause += " <= ?"; - expectedNoOfParams = 1; - } - case GREATER_THAN -> - { - clause += " > ?"; - expectedNoOfParams = 1; - } - case GREATER_THAN_OR_EQUALS -> - { - clause += " >= ?"; - expectedNoOfParams = 1; - } - case IS_BLANK -> - { - clause += " IS NULL"; - if(field.getType().isStringLike()) - { - clause += " OR " + column + " = ''"; - } - expectedNoOfParams = 0; - } - case IS_NOT_BLANK -> - { - clause += " IS NOT NULL"; - if(field.getType().isStringLike()) - { - clause += " AND " + column + " != ''"; - } - expectedNoOfParams = 0; - } - case BETWEEN -> - { - clause += " BETWEEN ? AND ?"; - expectedNoOfParams = 2; - } - case NOT_BETWEEN -> - { - clause += " NOT BETWEEN ? AND ?"; - expectedNoOfParams = 2; - } - case TRUE -> - { - clause = " 1 = 1 "; - expectedNoOfParams = 0; - } - case FALSE -> - { - clause = " 0 = 1 "; - expectedNoOfParams = 0; - } - default -> throw new IllegalStateException("Unexpected operator: " + criterion.getOperator()); } + Integer expectedNoOfParams = actionStrategy.appendCriterionToWhereClause(criterion, clause, column, values, field); + if(expectedNoOfParams != null) { if(expectedNoOfParams.equals(1) && StringUtils.hasContent(criterion.getOtherFieldName())) @@ -652,7 +508,7 @@ public abstract class AbstractRDBMSAction JoinsContext.FieldAndTableNameOrAlias otherFieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(criterion.getOtherFieldName()); String otherColumn = escapeIdentifier(otherFieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(getColumnName(otherFieldAndTableNameOrAlias.field())); - clause = clause.replace("?", otherColumn); + clause = new StringBuilder(clause.toString().replace("?", otherColumn)); ///////////////////////////////////////////////////////////////////// // make sure we don't add any values in this case, just in case... // @@ -797,53 +653,7 @@ public abstract class AbstractRDBMSAction *******************************************************************************/ protected Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException { - switch(type) - { - case STRING: - case TEXT: - case HTML: - case PASSWORD: - { - return (QueryManager.getString(resultSet, i)); - } - case INTEGER: - { - return (QueryManager.getInteger(resultSet, i)); - } - case LONG: - { - return (QueryManager.getLong(resultSet, i)); - } - case DECIMAL: - { - return (QueryManager.getBigDecimal(resultSet, i)); - } - case DATE: - { - // todo - queryManager.getLocalDate? - return (QueryManager.getDate(resultSet, i)); - } - case TIME: - { - return (QueryManager.getLocalTime(resultSet, i)); - } - case DATE_TIME: - { - return (QueryManager.getInstant(resultSet, i)); - } - case BOOLEAN: - { - return (QueryManager.getBoolean(resultSet, i)); - } - case BLOB: - { - return (QueryManager.getByteArray(resultSet, i)); - } - default: - { - throw new IllegalStateException("Unexpected field type: " + type); - } - } + return (actionStrategy.getFieldValueFromResultSet(type, resultSet, i)); } @@ -1151,4 +961,26 @@ public abstract class AbstractRDBMSAction return (filter.clone()); } } + + + + /******************************************************************************* + ** Setter for backendMetaData + ** + *******************************************************************************/ + protected void setBackendMetaData(QBackendMetaData backendMetaData) + { + this.backendMetaData = (RDBMSBackendMetaData) backendMetaData; + this.actionStrategy = this.backendMetaData.getActionStrategy(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected RDBMSActionStrategyInterface getActionStrategy() + { + return (this.actionStrategy); + } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java index 5c2f8710..e4d69585 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java @@ -48,7 +48,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -70,6 +69,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega try { QTableMetaData table = aggregateInput.getTable(); + setBackendMetaData(aggregateInput.getBackend()); QQueryFilter filter = clonedOrNewFilter(aggregateInput.getFilter()); JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), table.getName(), aggregateInput.getQueryJoins(), filter); @@ -126,7 +126,7 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega queryStat.setStartTimestamp(Instant.now()); } - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java index 121c2c9b..2aac8fb2 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java @@ -40,7 +40,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.JoinsContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import org.apache.commons.lang.BooleanUtils; @@ -63,6 +62,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf try { QTableMetaData table = countInput.getTable(); + setBackendMetaData(countInput.getBackend()); QQueryFilter filter = clonedOrNewFilter(countInput.getFilter()); JoinsContext joinsContext = new JoinsContext(QContext.getQInstance(), countInput.getTableName(), countInput.getQueryJoins(), filter); @@ -106,7 +106,7 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf queryStat.setStartTimestamp(Instant.now()); } - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java index 9869435d..b59e9588 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java @@ -39,7 +39,6 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -67,6 +66,8 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte *******************************************************************************/ public DeleteOutput execute(DeleteInput deleteInput) throws QException { + setBackendMetaData(deleteInput.getBackend()); + DeleteOutput deleteOutput = new DeleteOutput(); deleteOutput.setRecordsWithErrors(new ArrayList<>()); @@ -196,7 +197,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte try { - int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKey); + int rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, primaryKey); deleteOutput.addToDeletedRecordCount(rowCount); ///////////////////////////////////////////////////////////////////////////////////////////////////// @@ -249,7 +250,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte // todo sql customization - can edit sql and/or param list - Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys); + Integer rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, primaryKeys); deleteOutput.addToDeletedRecordCount(rowCount); } catch(Exception e) @@ -287,7 +288,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte try { - int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params); + int rowCount = getActionStrategy().executeUpdateForRowCount(connection, sql, params); deleteOutput.setDeletedRecordCount(rowCount); } catch(Exception e) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index 42ac344c..d6b89246 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -37,7 +37,6 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -56,6 +55,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte { InsertOutput rs = new InsertOutput(); QTableMetaData table = insertInput.getTable(); + setBackendMetaData(insertInput.getBackend()); Connection connection = null; boolean needToCloseConnection = false; @@ -90,7 +90,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte needToCloseConnection = true; } - for(List page : CollectionUtils.getPages(insertInput.getRecords(), QueryManager.PAGE_SIZE)) + for(List page : CollectionUtils.getPages(insertInput.getRecords(), getActionStrategy().getPageSize(insertInput))) { String tableName = escapeIdentifier(getTableName(table)); sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); @@ -146,7 +146,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte // todo sql customization - can edit sql and/or param list // todo - non-serial-id style tables // todo - other generated values, e.g., createDate... maybe need to re-select? - List idList = QueryManager.executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField()).getType()); + List idList = getActionStrategy().executeInsertForGeneratedIds(connection, sql.toString(), params, table.getField(table.getPrimaryKeyField())); int index = 0; for(QRecord record : page) { @@ -155,8 +155,11 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) { - Serializable id = idList.get(index++); - outputRecord.setValue(table.getPrimaryKeyField(), id); + if(idList.size() > index) + { + Serializable id = idList.get(index++); + outputRecord.setValue(table.getPrimaryKeyField(), id); + } } } diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java index 8ead329e..5c98b8c4 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java @@ -57,7 +57,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -94,6 +93,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf { QTableMetaData table = queryInput.getTable(); String tableName = queryInput.getTableName(); + setBackendMetaData(queryInput.getBackend()); List params = new ArrayList<>(); Selection selection = makeSelection(queryInput); @@ -140,7 +140,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf ////////////////////////////////////////////// QueryOutput queryOutput = new QueryOutput(queryInput); - QueryManager.executeStatement(statement, sql, ((ResultSet resultSet) -> + getActionStrategy().executeStatement(statement, sql, ((ResultSet resultSet) -> { ///////////////////////////////////////////////////////////////////////// // once we've started getting results, go ahead and cancel the timeout // diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java index 627b9c60..1894d911 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java @@ -41,7 +41,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; /******************************************************************************* @@ -66,6 +65,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte public UpdateOutput execute(UpdateInput updateInput) throws QException { QTableMetaData table = updateInput.getTable(); + setBackendMetaData(updateInput.getBackend()); UpdateActionRecordSplitHelper updateActionRecordSplitHelper = new UpdateActionRecordSplitHelper(); updateActionRecordSplitHelper.init(updateInput); @@ -181,7 +181,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte //////////////////////////////////////////////////////////////////////////////// try { - QueryManager.executeBatchUpdate(connection, sql, values); + getActionStrategy().executeBatchUpdate(connection, sql, values); incrementStatus(updateInput, recordList.size()); } finally @@ -214,7 +214,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte *******************************************************************************/ private void updateRecordsWithMatchingValuesAndFields(UpdateInput updateInput, Connection connection, QTableMetaData table, List recordList, List fieldsBeingUpdated) throws SQLException { - for(List page : CollectionUtils.getPages(recordList, QueryManager.PAGE_SIZE)) + for(List page : CollectionUtils.getPages(recordList, getActionStrategy().getPageSize(updateInput))) { ////////////////////////////// // skip records with errors // @@ -256,7 +256,7 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte ///////////////////////////////////// try { - QueryManager.executeUpdate(connection, sql, params); + getActionStrategy().executeUpdate(connection, sql, params); incrementStatus(updateInput, page.size()); } finally diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java index 61e72681..27b17dcd 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteActionTest.java @@ -34,8 +34,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -128,6 +128,11 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest deleteInput.setPrimaryKeys(List.of(1, -1)); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + + ///////////////////////////////////////////////////////////////////////////////////// + // note - that if we went to the top-level DeleteAction, then it would have pre- // + // checked that the ids existed, and it WOULD give us an error for the -1 row here // + ///////////////////////////////////////////////////////////////////////////////////// assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors (the one not found is just noop)"); } @@ -162,17 +167,15 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); //////////////////////////////////////////////////////////////////////////////////////// // assert that 6 queries ran - the initial delete (which failed), then 5 more deletes // //////////////////////////////////////////////////////////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(6, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(6, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); @@ -212,17 +215,15 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest //////////////////////////////////////////////////////////////////////// deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 4, 5)))); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); ////////////////////////////////// // assert that just 1 query ran // ////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(1, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(1, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); runTestSql("SELECT id FROM child_table", (rs -> @@ -259,9 +260,7 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2, 3, 4, 5)))); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); - + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -270,9 +269,9 @@ public class RDBMSDeleteActionTest extends RDBMSActionTest // todo - maybe we shouldn't do that 2nd "try to delete 'em all by id"... why would it ever work, // // but the original filter query didn't (other than malformed SQL)? // /////////////////////////////////////////////////////////////////////////////////////////////////////// - QueryManager.setCollectStatistics(false); - Map queryStats = QueryManager.getStatistics(); - assertEquals(8, queryStats.get(QueryManager.STAT_QUERIES_RAN), "Number of queries ran"); + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(8, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java index c7689286..0daa1381 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertActionTest.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.rdbms.actions; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -32,7 +33,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -53,6 +54,8 @@ public class RDBMSInsertActionTest extends RDBMSActionTest public void beforeEach() throws Exception { super.primeTestDatabase(); + + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); } @@ -113,7 +116,7 @@ public class RDBMSInsertActionTest extends RDBMSActionTest @Test public void testInsertMany() throws Exception { - QueryManager.setPageSize(2); + getBaseRDBMSActionStrategy().setPageSize(2); InsertInput insertInput = initInsertRequest(); QRecord record1 = new QRecord().withTableName("person") @@ -137,6 +140,10 @@ public class RDBMSInsertActionTest extends RDBMSActionTest assertEquals(6, insertOutput.getRecords().get(0).getValue("id"), "Should have next id in the row"); assertEquals(7, insertOutput.getRecords().get(1).getValue("id"), "Should have next id in the row"); assertEquals(8, insertOutput.getRecords().get(2).getValue("id"), "Should have next id in the row"); + + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + assertAnInsertedPersonRecord("Jean-Luc", "Picard", 6); assertAnInsertedPersonRecord("William", "Riker", 7); assertAnInsertedPersonRecord("Beverly", "Crusher", 8); diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java index 64f4c950..6eb9357f 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateActionTest.java @@ -36,7 +36,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.rdbms.TestUtils; -import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -61,8 +61,7 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest { super.primeTestDatabase(); - QueryManager.setCollectStatistics(true); - QueryManager.resetStatistics(); + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); } @@ -112,8 +111,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(List.of(record)); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(1, updateResult.getRecords().size(), "Should return 1 row"); assertEquals(2, updateResult.getRecords().get(0).getValue("id"), "Should have id=2 in the row"); @@ -169,9 +168,9 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest UpdateOutput updateResult = new UpdateAction().execute(updateInput); // this test runs one batch and one regular query - Map statistics = QueryManager.getStatistics(); - assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(3, updateResult.getRecords().size(), "Should return 3 rows"); assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); @@ -241,8 +240,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(List.of(record1, record2)); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(1, statistics.get(QueryManager.STAT_BATCHES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); @@ -296,8 +295,8 @@ public class RDBMSUpdateActionTest extends RDBMSActionTest updateInput.setRecords(records); UpdateOutput updateResult = new UpdateAction().execute(updateInput); - Map statistics = QueryManager.getStatistics(); - assertEquals(2, statistics.get(QueryManager.STAT_QUERIES_RAN)); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); assertEquals(5, updateResult.getRecords().size(), "Should return 5 rows"); // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); From 2260fbde84d609556a1711c00380eb84a0b26d78 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 19:36:11 -0600 Subject: [PATCH 080/209] Initial checkin of sqlite module --- pom.xml | 1 + qqq-backend-module-sqlite/pom.xml | 113 ++ .../module/sqlite/SQLiteBackendModule.java | 57 + .../model/metadata/SQLiteBackendMetaData.java | 119 ++ .../metadata/SQLiteTableBackendDetails.java | 45 + .../strategy/SQLiteRDBMSActionStrategy.java | 133 ++ .../qqq/backend/module/sqlite/BaseTest.java | 124 ++ .../qqq/backend/module/sqlite/TestUtils.java | 470 +++++++ .../sqlite/actions/SQLiteCountActionTest.java | 208 +++ .../actions/SQLiteDeleteActionTest.java | 322 +++++ .../actions/SQLiteInsertActionTest.java | 219 ++++ .../sqlite/actions/SQLiteQueryActionTest.java | 1136 +++++++++++++++++ .../actions/SQLiteUpdateActionTest.java | 443 +++++++ ...rime-test-database-parent-child-tables.sql | 49 + .../test/resources/prime-test-database.sql | 215 ++++ 15 files changed, 3654 insertions(+) create mode 100644 qqq-backend-module-sqlite/pom.xml create mode 100644 qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java create mode 100644 qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java create mode 100644 qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java create mode 100644 qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java create mode 100644 qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java create mode 100644 qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql create mode 100644 qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql diff --git a/pom.xml b/pom.xml index 68110a6b..c3f02707 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ qqq-backend-module-api qqq-backend-module-filesystem qqq-backend-module-rdbms + qqq-backend-module-sqlite qqq-backend-module-mongodb qqq-language-support-javascript qqq-openapi diff --git a/qqq-backend-module-sqlite/pom.xml b/qqq-backend-module-sqlite/pom.xml new file mode 100644 index 00000000..106dede0 --- /dev/null +++ b/qqq-backend-module-sqlite/pom.xml @@ -0,0 +1,113 @@ + + + + + 4.0.0 + + qqq-backend-module-sqlite + + + com.kingsrook.qqq + qqq-parent-project + ${revision} + + + + + + + + + + + com.kingsrook.qqq + qqq-backend-module-rdbms + ${revision} + + + + + org.xerial + sqlite-jdbc + 3.47.1.0 + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + *:* + + META-INF/* + + + + + + + ${plugin.shade.phase} + + shade + + + + + + + + diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java new file mode 100644 index 00000000..b733f796 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/SQLiteBackendModule.java @@ -0,0 +1,57 @@ +package com.kingsrook.qqq.backend.module.sqlite; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.module.rdbms.RDBMSBackendModule; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteBackendMetaData; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteTableBackendDetails; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteBackendModule extends RDBMSBackendModule +{ + private static final QLogger LOG = QLogger.getLogger(SQLiteBackendModule.class); + + private static final String NAME = "sqlite"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SQLiteBackendModule()); + } + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + public String getBackendType() + { + return NAME; + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SQLiteBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SQLiteTableBackendDetails.class); + } + +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java new file mode 100644 index 00000000..25f1af26 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteBackendMetaData.java @@ -0,0 +1,119 @@ +package com.kingsrook.qqq.backend.module.sqlite.model.metadata; + + +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.RDBMSActionStrategyInterface; +import com.kingsrook.qqq.backend.module.sqlite.SQLiteBackendModule; +import com.kingsrook.qqq.backend.module.sqlite.strategy.SQLiteRDBMSActionStrategy; +import org.sqlite.JDBC; + + +/******************************************************************************* + ** Meta-data to provide details of an SQLite backend (e.g., path to the database file) + *******************************************************************************/ +public class SQLiteBackendMetaData extends RDBMSBackendMetaData +{ + private String path; + + // todo - overrides to setters for unsupported fields? + // todo - or - change rdbms connection manager to not require an RDBMSBackendMetaData? + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SQLiteBackendMetaData() + { + super(); + setVendor("sqlite"); + setBackendType(SQLiteBackendModule.class); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String buildConnectionString() + { + return "jdbc:sqlite:" + this.path; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getJdbcDriverClassName() + { + return (JDBC.class.getName()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SQLiteBackendMetaData withName(String name) + { + setName(name); + return (this); + } + + + + /******************************************************************************* + ** Getter for path + *******************************************************************************/ + public String getPath() + { + return (this.path); + } + + + + /******************************************************************************* + ** Setter for path + *******************************************************************************/ + public void setPath(String path) + { + this.path = path; + } + + + + /******************************************************************************* + ** Fluent setter for path + *******************************************************************************/ + public SQLiteBackendMetaData withPath(String path) + { + this.path = path; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public RDBMSActionStrategyInterface getActionStrategy() + { + if(getActionStrategyField() == null) + { + if(getActionStrategyCodeReference() != null) + { + setActionStrategyField(QCodeLoader.getAdHoc(RDBMSActionStrategyInterface.class, getActionStrategyCodeReference())); + } + else + { + setActionStrategyField(new SQLiteRDBMSActionStrategy()); + } + } + + return (getActionStrategyField()); + } +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java new file mode 100644 index 00000000..98430d91 --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/model/metadata/SQLiteTableBackendDetails.java @@ -0,0 +1,45 @@ +package com.kingsrook.qqq.backend.module.sqlite.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteTableBackendDetails extends QTableBackendDetails +{ + private String tableName; + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public SQLiteTableBackendDetails withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + +} diff --git a/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java new file mode 100644 index 00000000..ee948e1d --- /dev/null +++ b/qqq-backend-module-sqlite/src/main/java/com/kingsrook/qqq/backend/module/sqlite/strategy/SQLiteRDBMSActionStrategy.java @@ -0,0 +1,133 @@ +package com.kingsrook.qqq.backend.module.sqlite.strategy; + + +import java.io.Serializable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; + + +/******************************************************************************* + ** SQLite specialization of the default RDBMS/JDBC action strategy + *******************************************************************************/ +public class SQLiteRDBMSActionStrategy extends BaseRDBMSActionStrategy +{ + + /*************************************************************************** + ** deal with sqlite not having temporal types... so temporal values + ** i guess are stored as strings, as that's how they come back to us - so + ** the JDBC methods fail trying to getDate or whatever from them - but + ** getting the values as strings, they parse nicely, so do that. + ***************************************************************************/ + @Override + public Serializable getFieldValueFromResultSet(QFieldType type, ResultSet resultSet, int i) throws SQLException + { + return switch(type) + { + case DATE -> + { + try + { + yield parseString(s -> LocalDate.parse(s), resultSet, i); + } + catch(Exception e) + { + ///////////////////////////////////////////////////////////////////////////////// + // handle the case of, the value we got back is actually a date-time -- so -- // + // let's parse it as such, and then map into a LocalDate in the session zoneId // + ///////////////////////////////////////////////////////////////////////////////// + Instant instant = (Instant) parseString(s -> Instant.parse(s), resultSet, i); + if(instant == null) + { + yield null; + } + ZoneId zoneId = ValueUtils.getSessionOrInstanceZoneId(); + yield instant.atZone(zoneId).toLocalDate(); + } + } + case TIME -> parseString(s -> LocalTime.parse(s), resultSet, i); + case DATE_TIME -> parseString(s -> Instant.parse(s), resultSet, i); + default -> super.getFieldValueFromResultSet(type, resultSet, i); + }; + } + + + + /*************************************************************************** + ** helper method for getFieldValueFromResultSet + ***************************************************************************/ + private Serializable parseString(Function parser, ResultSet resultSet, int i) throws SQLException + { + String valueString = QueryManager.getString(resultSet, i); + if(valueString == null) + { + return (null); + } + else + { + return parser.apply(valueString); + } + } + + + + /*************************************************************************** + * bind temporal types as strings (see above comment re: sqlite temporal types) + ***************************************************************************/ + @Override + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value instanceof Instant || value instanceof LocalTime || value instanceof LocalDate) + { + bindParam(statement, index, value.toString()); + return 1; + } + else + { + return super.bindParamObject(statement, index, value); + } + } + + + + /*************************************************************************** + ** per discussion (and rejected PR mentioned) on https://github.com/prrvchr/sqlite-jdbc + ** sqlite jdbc by default will only return the latest generated serial. but we can get + ** them all by appending this "RETURNING id" to the query, and then calling execute() + ** (instead of executeUpdate()) and getResultSet (instead of getGeneratedKeys()) + ***************************************************************************/ + @Override + public List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldMetaData primaryKeyField) throws SQLException + { + sql = sql + " RETURNING " + getColumnName(primaryKeyField); + + try(PreparedStatement statement = connection.prepareStatement(sql)) + { + bindParams(params.toArray(), statement); + incrementStatistic(STAT_QUERIES_RAN); + statement.execute(); + + ResultSet generatedKeys = statement.getResultSet(); + List rs = new ArrayList<>(); + while(generatedKeys.next()) + { + rs.add(getFieldValueFromResultSet(primaryKeyField.getType(), generatedKeys, 1)); + } + return (rs); + } + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java new file mode 100644 index 00000000..86228aea --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/BaseTest.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite; + + +import java.sql.Connection; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseTest.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void baseBeforeEach() throws Exception + { + QContext.init(TestUtils.defineInstance(), new QSession()); + TestUtils.primeTestDatabase("prime-test-database.sql"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void baseAfterEach() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setPageSize(BaseRDBMSActionStrategy.DEFAULT_PAGE_SIZE); + actionStrategy.resetStatistics(); + actionStrategy.setCollectStatistics(false); + + QContext.clear(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategy() + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) QContext.getQInstance().getBackend(TestUtils.DEFAULT_BACKEND_NAME); + BaseRDBMSActionStrategy actionStrategy = (BaseRDBMSActionStrategy) backend.getActionStrategy(); + return actionStrategy; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected static BaseRDBMSActionStrategy getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + { + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategy(); + actionStrategy.setCollectStatistics(true); + actionStrategy.resetStatistics(); + return actionStrategy; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected static void reInitInstanceInContext(QInstance qInstance) + { + if(qInstance.equals(QContext.getQInstance())) + { + LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit. You probably want a new QInstance object instance."); + } + QContext.init(qInstance, new QSession()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + protected void runTestSql(String sql, QueryManager.ResultSetProcessor resultSetProcessor) throws Exception + { + ConnectionManager connectionManager = new ConnectionManager(); + Connection connection = connectionManager.getConnection(TestUtils.defineBackend()); + QueryManager.executeStatement(connection, sql, resultSetProcessor); + connection.close(); + } +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java new file mode 100644 index 00000000..bdf26b24 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/TestUtils.java @@ -0,0 +1,470 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite; + + +import java.io.File; +import java.io.InputStream; +import java.sql.Connection; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.C3P0PooledConnectionProvider; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteBackendMetaData; +import org.apache.commons.io.IOUtils; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestUtils +{ + public static final String DEFAULT_BACKEND_NAME = "default"; + public static final String MEMORY_BACKEND_NAME = "memory"; + + public static final String TABLE_NAME_PERSON = "personTable"; + public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard"; + public static final String TABLE_NAME_STORE = "store"; + public static final String TABLE_NAME_ORDER = "order"; + public static final String TABLE_NAME_ORDER_INSTRUCTIONS = "orderInstructions"; + public static final String TABLE_NAME_ITEM = "item"; + public static final String TABLE_NAME_ORDER_LINE = "orderLine"; + public static final String TABLE_NAME_LINE_ITEM_EXTRINSIC = "orderLineExtrinsic"; + public static final String TABLE_NAME_WAREHOUSE = "warehouse"; + public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt"; + + public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static void primeTestDatabase(String sqlFileName) throws Exception + { + SQLiteBackendMetaData backend = TestUtils.defineBackend(); + + File file = new File(backend.getPath()); + + /* + if(file.exists()) + { + if(!file.delete()) + { + throw (new Exception("SQLite database at [" + file.getAbsolutePath() + "] exists, and could not be deleted before (re)priming the database.")); + } + } + */ + + file.getParentFile().mkdirs(); + + try(Connection connection = ConnectionManager.getConnection(backend)) + { + InputStream primeTestDatabaseSqlStream = SQLiteBackendModule.class.getResourceAsStream("/" + sqlFileName); + assertNotNull(primeTestDatabaseSqlStream); + List lines = (List) IOUtils.readLines(primeTestDatabaseSqlStream); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + if(sql.matches("(?s).*[a-zA-Z0-9_].*")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QInstance defineInstance() + { + QInstance qInstance = new QInstance(); + qInstance.addBackend(defineBackend()); + qInstance.addBackend(defineMemoryBackend()); + qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(definePvsPerson()); + qInstance.addTable(defineTablePersonalIdCard()); + qInstance.addJoin(defineJoinPersonAndPersonalIdCard()); + addOmsTablesAndJoins(qInstance); + qInstance.setAuthentication(defineAuthentication()); + return (qInstance); + } + + + + /******************************************************************************* + ** Define the in-memory backend used in standard tests + *******************************************************************************/ + public static QBackendMetaData defineMemoryBackend() + { + return new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class); + } + + + + /******************************************************************************* + ** Define the authentication used in standard tests - using 'mock' type. + ** + *******************************************************************************/ + public static QAuthenticationMetaData defineAuthentication() + { + return new QAuthenticationMetaData() + .withName("mock") + .withType(QAuthenticationType.MOCK); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static SQLiteBackendMetaData defineBackend() + { + SQLiteBackendMetaData sqLiteBackendMetaData = new SQLiteBackendMetaData() + .withName(DEFAULT_BACKEND_NAME) + .withPath("/tmp/sqlite/test.db"); + + sqLiteBackendMetaData.setQueriesForNewConnections(List.of( + "PRAGMA foreign_keys = ON" + )); + + sqLiteBackendMetaData.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class)); + + return sqLiteBackendMetaData; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePerson() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON) + .withLabel("Person") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("firstName", "lastName") + .withBackendName(DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name")) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name")) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + .withField(new QFieldMetaData("email", QFieldType.STRING).withBackendName("email")) + .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) + .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary")) + .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked")) + .withField(new QFieldMetaData("homeTown", QFieldType.STRING).withBackendName("home_town")) + .withField(new QFieldMetaData("startTime", QFieldType.TIME).withBackendName("start_time")) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("person")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QPossibleValueSource definePvsPerson() + { + return (new QPossibleValueSource() + .withName(TABLE_NAME_PERSON) + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_PERSON) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)); + } + + + + /******************************************************************************* + ** Define a 1:1 table with Person. + ** + *******************************************************************************/ + private static QTableMetaData defineTablePersonalIdCard() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSONAL_ID_CARD) + .withLabel("Personal Id Card") + .withBackendName(DEFAULT_BACKEND_NAME) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("personal_id_card")) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date")) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) + .withField(new QFieldMetaData("personId", QFieldType.INTEGER).withBackendName("person_id")) + .withField(new QFieldMetaData("idNumber", QFieldType.STRING).withBackendName("id_number")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QJoinMetaData defineJoinPersonAndPersonalIdCard() + { + return new QJoinMetaData() + .withLeftTable(TABLE_NAME_PERSON) + .withRightTable(TABLE_NAME_PERSONAL_ID_CARD) + .withInferredName() + .withType(JoinType.ONE_TO_ONE) + .withJoinOn(new JoinOn("id", "personId")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void addOmsTablesAndJoins(QInstance qInstance) + { + qInstance.addTable(defineBaseTable(TABLE_NAME_STORE, "store") + .withRecordLabelFormat("%s") + .withRecordLabelFields("name") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("id")) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER, "order") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withAssociation(new Association().withName("orderLine").withAssociatedTableName(TABLE_NAME_ORDER_LINE).withJoinName("orderJoinOrderLine")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ITEM).withJoinPath(List.of("orderJoinOrderLine", "orderLineJoinItem"))) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER_INSTRUCTIONS).withJoinPath(List.of("orderJoinCurrentOrderInstructions")).withLabel("Current Order Instructions")) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("billToPersonId", QFieldType.INTEGER).withBackendName("bill_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("shipToPersonId", QFieldType.INTEGER).withBackendName("ship_to_person_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("currentOrderInstructionsId", QFieldType.INTEGER).withBackendName("current_order_instructions_id").withPossibleValueSourceName(TABLE_NAME_PERSON)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_INSTRUCTIONS, "order_instructions") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderInstructionsJoinOrder"))) + .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) + .withField(new QFieldMetaData("instructions", QFieldType.STRING)) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderInstructionsJoinOrder"))) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ITEM, "item") + .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(TABLE_NAME_STORE).withFieldName("storeId")) + .withExposedJoin(new ExposedJoin().withJoinTable(TABLE_NAME_ORDER).withJoinPath(List.of("orderLineJoinItem", "orderJoinOrderLine"))) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("description", QFieldType.STRING)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_ORDER_LINE, "order_line") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderJoinOrderLine"))) + .withAssociation(new Association().withName("extrinsics").withAssociatedTableName(TABLE_NAME_LINE_ITEM_EXTRINSIC).withJoinName("orderLineJoinLineItemExtrinsic")) + .withField(new QFieldMetaData("orderId", QFieldType.INTEGER).withBackendName("order_id")) + .withField(new QFieldMetaData("sku", QFieldType.STRING)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id").withPossibleValueSourceName(TABLE_NAME_STORE)) + .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_LINE_ITEM_EXTRINSIC, "line_item_extrinsic") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName("order.storeId") + .withJoinNameChain(List.of("orderJoinOrderLine", "orderLineJoinLineItemExtrinsic"))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("orderLineId", QFieldType.INTEGER).withBackendName("order_line_id")) + .withField(new QFieldMetaData("key", QFieldType.STRING)) + .withField(new QFieldMetaData("value", QFieldType.STRING)) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int") + .withField(new QFieldMetaData("warehouseId", QFieldType.INTEGER).withBackendName("warehouse_id")) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id")) + ); + + qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse") + .withRecordSecurityLock(new RecordSecurityLock() + .withSecurityKeyType(TABLE_NAME_STORE) + .withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeId") + .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT))) + ) + .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withType(JoinType.ONE_TO_MANY) + .withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE) + .withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT) + .withInferredName() + .withJoinOn(new JoinOn("id", "warehouseId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinStore") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinBillToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("billToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinShipToPerson") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_PERSON) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("shipToPersonId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("itemJoinStore") + .withLeftTable(TABLE_NAME_ITEM) + .withRightTable(TABLE_NAME_STORE) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("storeId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinOrderLine") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_LINE) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinItem") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_ITEM) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("sku", "sku")) + .withJoinOn(new JoinOn("storeId", "storeId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderLineJoinLineItemExtrinsic") + .withLeftTable(TABLE_NAME_ORDER_LINE) + .withRightTable(TABLE_NAME_LINE_ITEM_EXTRINSIC) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "orderLineId")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderJoinCurrentOrderInstructions") + .withLeftTable(TABLE_NAME_ORDER) + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withType(JoinType.ONE_TO_ONE) + .withJoinOn(new JoinOn("currentOrderInstructionsId", "id")) + ); + + qInstance.addJoin(new QJoinMetaData() + .withName("orderInstructionsJoinOrder") + .withRightTable(TABLE_NAME_ORDER_INSTRUCTIONS) + .withLeftTable(TABLE_NAME_ORDER) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("id", "orderId")) + ); + + qInstance.addPossibleValueSource(new QPossibleValueSource() + .withName("store") + .withType(QPossibleValueSourceType.TABLE) + .withTableName(TABLE_NAME_STORE) + .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY) + ); + + qInstance.addSecurityKeyType(new QSecurityKeyType() + .withName(TABLE_NAME_STORE) + .withAllAccessKeyName(SECURITY_KEY_STORE_ALL_ACCESS) + .withPossibleValueSourceName(TABLE_NAME_STORE)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableMetaData defineBaseTable(String tableName, String backendTableName) + { + return new QTableMetaData() + .withName(tableName) + .withBackendName(DEFAULT_BACKEND_NAME) + .withBackendDetails(new RDBMSTableBackendDetails().withTableName(backendTableName)) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static List queryTable(String tableName) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + return (queryOutput.getRecords()); + } +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java new file mode 100644 index 00000000..c28c4c9c --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteCountActionTest.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteCountActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredCount() throws QException + { + CountInput countInput = initCountRequest(); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(5, countOutput.getCount(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQueryCount() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountInput countInput = initCountRequest(); + countInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(1, countOutput.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + CountInput countInput = initCountRequest(); + countInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(4, countOutput.getCount(), "Expected # of rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.defineTablePerson().getName()); + return countInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(3, countOutput.getCount(), "Join count should find 3 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneLeftJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.LEFT)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(5, countOutput.getCount(), "Left Join count should find 5 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneRightJoinWithoutWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withType(QueryJoin.Type.RIGHT)); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(6, countOutput.getCount(), "Right Join count should find 6 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneToOneInnerJoinWithWhere() throws QException + { + CountInput countInput = initCountRequest(); + countInput.withQueryJoin(new QueryJoin(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_PERSONAL_ID_CARD).withSelect(true)); + countInput.setFilter(new QQueryFilter(new QFilterCriteria(TestUtils.TABLE_NAME_PERSONAL_ID_CARD + ".idNumber", QCriteriaOperator.STARTS_WITH, "1980"))); + CountOutput countOutput = new CountAction().execute(countInput); + assertEquals(2, countOutput.getCount(), "Right Join count should find 2 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurity() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + QContext.setQSession(new QSession()); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(0); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(8); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2).withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(5); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException + { + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE); + + assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(4); + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java new file mode 100644 index 00000000..86d2cadf --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteDeleteActionTest.java @@ -0,0 +1,322 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSDeleteAction; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteDeleteActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteAll() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(5, deleteResult.getDeletedRecordCount(), "Unfiltered delete should return all rows"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person", (rs -> assertFalse(rs.next()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteOne() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person WHERE id = 1", (rs -> assertFalse(rs.next()))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteSome() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, 3, 5)); + DeleteOutput deleteResult = new DeleteAction().execute(deleteInput); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should delete one row"); + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors"); + runTestSql("SELECT id FROM person", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertTrue(rs.getInt(1) == 2 || rs.getInt(1) == 4); + } + assertEquals(2, rowsFound); + })); + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDeleteSomeIdsThatExistAndSomeThatDoNot() throws Exception + { + DeleteInput deleteInput = initStandardPersonDeleteRequest(); + deleteInput.setPrimaryKeys(List.of(1, -1)); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + assertEquals(1, deleteResult.getDeletedRecordCount(), "Should delete one row"); + + ///////////////////////////////////////////////////////////////////////////////////// + // note - that if we went to the top-level DeleteAction, then it would have pre- // + // checked that the ids existed, and it WOULD give us an error for the -1 row here // + ///////////////////////////////////////////////////////////////////////////////////// + assertEquals(0, deleteResult.getRecordsWithErrors().size(), "should have no errors (the one not found is just noop)"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private DeleteInput initStandardPersonDeleteRequest() + { + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(TestUtils.defineTablePerson().getName()); + return deleteInput; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteWhereForeignKeyBlocksSome() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try to delete all of the child records - 2 should fail, because they are referenced by parent_table.child_id // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteInput.setPrimaryKeys(List.of(1, 2, 3, 4, 5)); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + //////////////////////////////////////////////////////////////////////////////////////// + // assert that 6 queries ran - the initial delete (which failed), then 5 more deletes // + //////////////////////////////////////////////////////////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(6, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + + assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); + assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteByFilterThatJustWorks() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + //////////////////////////////////////////////////////////////////////// + // try to delete the records without a foreign key that'll block them // + //////////////////////////////////////////////////////////////////////// + deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(2, 4, 5)))); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + ////////////////////////////////// + // assert that just 1 query ran // + ////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(1, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDeleteByFilterWhereForeignKeyBlocksSome() throws Exception + { + ////////////////////////////////////////////////////////////////// + // load the parent-child tables, with foreign keys and instance // + ////////////////////////////////////////////////////////////////// + TestUtils.primeTestDatabase("prime-test-database-parent-child-tables.sql"); + DeleteInput deleteInput = initChildTableInstanceAndDeleteRequest(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // try to delete all of the child records - 2 should fail, because they are referenced by parent_table.child_id // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + deleteInput.setQueryFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2, 3, 4, 5)))); + + BaseRDBMSActionStrategy actionStrategy = getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + DeleteOutput deleteResult = new RDBMSDeleteAction().execute(deleteInput); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // assert that 8 queries ran - the initial delete (which failed), then 1 to look up the ids // + // from that query, another to try to delete all those ids (also fails), and finally 5 deletes by id // + // todo - maybe we shouldn't do that 2nd "try to delete 'em all by id"... why would it ever work, // + // but the original filter query didn't (other than malformed SQL)? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + actionStrategy.setCollectStatistics(false); + Map queryStats = actionStrategy.getStatistics(); + assertEquals(8, queryStats.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN), "Number of queries ran"); + + assertEquals(2, deleteResult.getRecordsWithErrors().size(), "Should get back the 2 records with errors"); + assertTrue(deleteResult.getRecordsWithErrors().stream().noneMatch(r -> r.getErrors().isEmpty()), "All we got back should have errors"); + assertEquals(3, deleteResult.getDeletedRecordCount(), "Should get back that 3 were deleted"); + + runTestSql("SELECT id FROM child_table", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + /////////////////////////////////////////// + // child_table rows 1 & 3 should survive // + /////////////////////////////////////////// + assertTrue(rs.getInt(1) == 1 || rs.getInt(1) == 3); + } + assertEquals(2, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private DeleteInput initChildTableInstanceAndDeleteRequest() + { + QInstance qInstance = TestUtils.defineInstance(); + + String childTableName = "childTable"; + qInstance.addTable(new QTableMetaData() + .withName(childTableName) + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("child_table"))); + + qInstance.addTable(new QTableMetaData() + .withName("parentTable") + .withBackendName(TestUtils.DEFAULT_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("name", QFieldType.STRING)) + .withField(new QFieldMetaData("childId", QFieldType.INTEGER).withBackendName("child_id")) + .withBackendDetails(new RDBMSTableBackendDetails() + .withTableName("parent_table"))); + + reInitInstanceInContext(qInstance); + + DeleteInput deleteInput = new DeleteInput(); + deleteInput.setTableName(childTableName); + + return deleteInput; + } +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java new file mode 100644 index 00000000..4c59d024 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteInsertActionTest.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteInsertActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertNullList() throws QException + { + InsertInput insertInput = initInsertRequest(); + insertInput.setRecords(null); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(0, insertOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertEmptyList() throws QException + { + InsertInput insertInput = initInsertRequest(); + insertInput.setRecords(Collections.emptyList()); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(0, insertOutput.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertOne() throws Exception + { + InsertInput insertInput = initInsertRequest(); + QRecord record = new QRecord().withTableName("person") + .withValue("firstName", "James") + .withValue("lastName", "Kirk") + .withValue("email", "jamestk@starfleet.net") + .withValue("birthDate", "2210-05-20"); + insertInput.setRecords(List.of(record)); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(1, insertOutput.getRecords().size(), "Should return 1 row"); + assertNotNull(insertOutput.getRecords().get(0).getValue("id"), "Should have an id in the row"); + // todo - add errors to QRecord? assertTrue(insertResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + assertAnInsertedPersonRecord("James", "Kirk", 6); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInsertMany() throws Exception + { + getBaseRDBMSActionStrategyAndActivateCollectingStatistics() + .setPageSize(2); + + InsertInput insertInput = initInsertRequest(); + QRecord record1 = new QRecord().withTableName("person") + .withValue("firstName", "Jean-Luc") + .withValue("lastName", "Picard") + .withValue("email", "jl@starfleet.net") + .withValue("birthDate", "2310-05-20"); + QRecord record2 = new QRecord().withTableName("person") + .withValue("firstName", "William") + .withValue("lastName", "Riker") + .withValue("email", "notthomas@starfleet.net") + .withValue("birthDate", "2320-05-20"); + QRecord record3 = new QRecord().withTableName("person") + .withValue("firstName", "Beverly") + .withValue("lastName", "Crusher") + .withValue("email", "doctor@starfleet.net") + .withValue("birthDate", "2320-06-26"); + insertInput.setRecords(List.of(record1, record2, record3)); + InsertOutput insertOutput = new InsertAction().execute(insertInput); + assertEquals(3, insertOutput.getRecords().size(), "Should return right # of rows"); + assertEquals(6, insertOutput.getRecords().get(0).getValue("id"), "Should have next id in the row"); + assertEquals(7, insertOutput.getRecords().get(1).getValue("id"), "Should have next id in the row"); + assertEquals(8, insertOutput.getRecords().get(2).getValue("id"), "Should have next id in the row"); + + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertAnInsertedPersonRecord("Jean-Luc", "Picard", 6); + assertAnInsertedPersonRecord("William", "Riker", 7); + assertAnInsertedPersonRecord("Beverly", "Crusher", 8); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testInsertAssociations() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1); + + int originalNoOfOrderLineExtrinsics = TestUtils.queryTable(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC).size(); + int originalNoOfOrderLines = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER_LINE).size(); + int originalNoOfOrders = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER).size(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_ORDER); + insertInput.setRecords(List.of( + new QRecord().withValue("storeId", 1).withValue("billToPersonId", 100).withValue("shipToPersonId", 200) + + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC1").withValue("quantity", 1) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-1.1").withValue("value", "LINE-VAL-1"))) + + .withAssociatedRecord("orderLine", new QRecord().withValue("storeId", 1).withValue("sku", "BASIC2").withValue("quantity", 2) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.1").withValue("value", "LINE-VAL-2")) + .withAssociatedRecord("extrinsics", new QRecord().withValue("key", "LINE-EXT-2.2").withValue("value", "LINE-VAL-3"))) + )); + new InsertAction().execute(insertInput); + + List orders = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER); + assertEquals(originalNoOfOrders + 1, orders.size()); + assertTrue(orders.stream().anyMatch(r -> Objects.equals(r.getValue("billToPersonId"), 100) && Objects.equals(r.getValue("shipToPersonId"), 200))); + + List orderLines = TestUtils.queryTable(TestUtils.TABLE_NAME_ORDER_LINE); + assertEquals(originalNoOfOrderLines + 2, orderLines.size()); + assertTrue(orderLines.stream().anyMatch(r -> Objects.equals(r.getValue("sku"), "BASIC1") && Objects.equals(r.getValue("quantity"), 1))); + assertTrue(orderLines.stream().anyMatch(r -> Objects.equals(r.getValue("sku"), "BASIC2") && Objects.equals(r.getValue("quantity"), 2))); + + List lineItemExtrinsics = TestUtils.queryTable(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC); + assertEquals(originalNoOfOrderLineExtrinsics + 3, lineItemExtrinsics.size()); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-1.1") && Objects.equals(r.getValue("value"), "LINE-VAL-1"))); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-2.1") && Objects.equals(r.getValue("value"), "LINE-VAL-2"))); + assertTrue(lineItemExtrinsics.stream().anyMatch(r -> Objects.equals(r.getValue("key"), "LINE-EXT-2.2") && Objects.equals(r.getValue("value"), "LINE-VAL-3"))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertAnInsertedPersonRecord(String firstName, String lastName, Integer id) throws Exception + { + runTestSql("SELECT * FROM person WHERE last_name = '" + lastName + "'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(id, rs.getInt("id")); + assertEquals(firstName, rs.getString("first_name")); + assertNotNull(rs.getString("create_date")); + assertNotNull(rs.getString("modify_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private InsertInput initInsertRequest() + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return insertInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java new file mode 100644 index 00000000..da415b84 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteQueryActionTest.java @@ -0,0 +1,1136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Predicate; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.Now; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.NowWithOffset; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.expressions.ThisOrLastPeriod; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; +import com.kingsrook.qqq.backend.module.rdbms.actions.AbstractRDBMSAction; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSQueryAction; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteQueryActionTest extends BaseTest +{ + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + AbstractRDBMSAction.setLogSQL(false); + QContext.getQSession().removeValue(QSession.VALUE_KEY_USER_TIMEZONE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUnfilteredQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTrueQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.TRUE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "'TRUE' query should find all rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFalseQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.FALSE))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "'FALSE' query should find no rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertEquals(email, queryOutput.getRecords().get(0).getValueString("email"), "Should find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsQuery() throws QException + { + String email = "darin.kelkhoff@gmail.com"; + + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(email))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").equals(email)), "Should NOT find expected email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEqualsOrIsNullQuery() throws QException + { + ///////////////////////////////////////////////////////////////////////////// + // 5 rows, 1 has a null salary, 1 has 1,000,000. // + // first confirm that query for != returns 3 (the null does NOT come back) // + // then, confirm that != or is null gives the (more humanly expected) 4. // + ///////////////////////////////////////////////////////////////////////////// + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS) + .withValues(List.of(1_000_000)))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + + queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("annualSalary") + .withOperator(QCriteriaOperator.NOT_EQUALS_OR_IS_NULL) + .withValues(List.of(1_000_000)))); + queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> Objects.equals(1_000_000, r.getValueInteger("annualSalary"))), "Should NOT find expected salary"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.IN) + .withValues(List.of(2, 4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotInQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.NOT_IN) + .withValues(List.of(2, 3, 4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotLike() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_LIKE) + .withValues(List.of("%kelk%"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotStartsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_STARTS_WITH) + .withValues(List.of("darin"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches("darin.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotContains() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_CONTAINS) + .withValues(List.of("kelkhoff"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*kelkhoff.*")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotEndsWith() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("email") + .withOperator(QCriteriaOperator.NOT_ENDS_WITH) + .withValues(List.of("gmail.com"))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(4, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().noneMatch(r -> r.getValueString("email").matches(".*gmail.com")), "Should find matching email address"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.LESS_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLessThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.LESS_THAN_OR_EQUALS) + .withValues(List.of(2))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(2)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.GREATER_THAN) + .withValues(List.of(3))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testGreaterThanOrEqualsQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.GREATER_THAN_OR_EQUALS) + .withValues(List.of(4))) + ); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(4) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("birthDate") + .withOperator(QCriteriaOperator.IS_BLANK) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("birthDate") == null), "Should find expected row"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testIsNotBlankQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("firstName") + .withOperator(QCriteriaOperator.IS_NOT_BLANK) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValue("firstName") != null), "Should find expected rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(3, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(2) || r.getValueInteger("id").equals(3) || r.getValueInteger("id").equals(4)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testNotBetweenQuery() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria() + .withFieldName("id") + .withOperator(QCriteriaOperator.NOT_BETWEEN) + .withValues(List.of(2, 4)) + )); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().allMatch(r -> r.getValueInteger("id").equals(1) || r.getValueInteger("id").equals(5)), "Should find expected ids"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFilterExpressions() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + insertInput.setRecords(List.of( + new QRecord().withValue("email", "-").withValue("firstName", "past").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().minus(3, ChronoUnit.DAYS)), + new QRecord().withValue("email", "-").withValue("firstName", "future").withValue("lastName", "ExpressionTest").withValue("birthDate", Instant.now().plus(3, ChronoUnit.DAYS)) + )); + new InsertAction().execute(insertInput); + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(new Now())))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.LESS_THAN).withValues(List.of(NowWithOffset.plus(2, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + } + + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria().withFieldName("lastName").withOperator(QCriteriaOperator.EQUALS).withValues(List.of("ExpressionTest"))) + .withCriteria(new QFilterCriteria().withFieldName("birthDate").withOperator(QCriteriaOperator.GREATER_THAN).withValues(List.of(NowWithOffset.minus(5, ChronoUnit.DAYS))))); + QueryOutput queryOutput = new RDBMSQueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("past")), "Should find expected row"); + assertTrue(queryOutput.getRecords().stream().anyMatch(r -> r.getValue("firstName").equals("future")), "Should find expected row"); + } + } + + + + /******************************************************************************* + ** Adding additional test conditions, specifically for DATE-precision + *******************************************************************************/ + @ParameterizedTest() + @ValueSource(strings = { "UTC", "US/Eastern", "UTC+12" }) + void testMoreFilterExpressions(String userTimezone) throws QException + { + QContext.getQSession().setValue(QSession.VALUE_KEY_USER_TIMEZONE, userTimezone); + + LocalDate today = Instant.now().atZone(ZoneId.of(userTimezone)).toLocalDate(); + LocalDate yesterday = today.minusDays(1); + LocalDate tomorrow = today.plusDays(1); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON).withRecords(List.of( + new QRecord().withValue("email", "-").withValue("firstName", "yesterday").withValue("lastName", "ExpressionTest").withValue("birthDate", yesterday), + new QRecord().withValue("email", "-").withValue("firstName", "today").withValue("lastName", "ExpressionTest").withValue("birthDate", today), + new QRecord().withValue("email", "-").withValue("firstName", "tomorrow").withValue("lastName", "ExpressionTest").withValue("birthDate", tomorrow)) + )); + + UnsafeFunction, List, QException> testFunction = (filterConsumer) -> + { + QQueryFilter filter = new QQueryFilter().withCriteria("lastName", QCriteriaOperator.EQUALS, "ExpressionTest"); + filter.withOrderBy(new QFilterOrderBy("birthDate")); + filterConsumer.accept(filter); + + return QueryAction.execute(TestUtils.TABLE_NAME_PERSON, filter); + }; + + assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, new Now())))); + assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, new Now())))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, new Now())))); + assertTwoRecordsWithFirstNames("yesterday", "today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, new Now())))); + assertTwoRecordsWithFirstNames("today", "tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, new Now())))); + + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.minus(1, ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN_OR_EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.minus(1, ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("tomorrow", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, NowWithOffset.plus(1, ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.WEEKS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.MONTHS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.YEARS))))); + + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, NowWithOffset.plus(1, ChronoUnit.HOURS))))) + .hasRootCauseMessage("Unsupported unit: Hours"); + + assertOneRecordWithFirstName("today", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.this_(ChronoUnit.DAYS))))); + assertOneRecordWithFirstName("yesterday", testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.EQUALS, ThisOrLastPeriod.last(ChronoUnit.DAYS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS))))); + assertNoOfRecords(3, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.GREATER_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.WEEKS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MONTHS))))); + assertNoOfRecords(0, testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.YEARS))))); + + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.this_(ChronoUnit.HOURS))))) + .hasRootCauseMessage("Unsupported unit: Hours"); + assertThatThrownBy(() -> testFunction.apply(filter -> filter.withCriteria(new QFilterCriteria("birthDate", QCriteriaOperator.LESS_THAN, ThisOrLastPeriod.last(ChronoUnit.MINUTES))))) + .hasRootCauseMessage("Unsupported unit: Minutes"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertNoOfRecords(Integer expectedSize, List actualRecords) + { + assertEquals(expectedSize, actualRecords.size()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertOneRecordWithFirstName(String expectedFirstName, List actualRecords) + { + assertEquals(1, actualRecords.size()); + assertEquals(expectedFirstName, actualRecords.get(0).getValueString("firstName")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertTwoRecordsWithFirstNames(String expectedFirstName0, String expectedFirstName1, List actualRecords) + { + assertEquals(2, actualRecords.size()); + assertEquals(expectedFirstName0, actualRecords.get(0).getValueString("firstName")); + assertEquals(expectedFirstName1, actualRecords.get(1).getValueString("firstName")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return queryInput; + } + + + + /******************************************************************************* + ** This doesn't really test any RDBMS code, but is a checkpoint that the core + ** module is populating displayValues when it performs the system-level query action + ** (if so requested by input field). + *******************************************************************************/ + @Test + public void testThatDisplayValuesGetSetGoingThroughQueryAction() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setShouldGenerateDisplayValues(true); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Unfiltered query should find all rows"); + + for(QRecord record : queryOutput.getRecords()) + { + assertThat(record.getValues()).isNotEmpty(); + assertThat(record.getDisplayValues()).isNotEmpty(); + assertThat(record.getErrors()).isEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLookInsideTransaction() throws QException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON); + + InsertAction insertAction = new InsertAction(); + QBackendTransaction transaction = QBackendTransaction.openFor(insertInput); + + insertInput.setTransaction(transaction); + insertInput.setRecords(List.of( + new QRecord().withValue("firstName", "George").withValue("lastName", "Washington").withValue("email", "gw@kingsrook.com") + )); + + insertAction.execute(insertInput); + + QueryInput queryInput = initQueryRequest(); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "Query without the transaction should not see the new row."); + + queryInput = initQueryRequest(); + queryInput.setTransaction(transaction); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(6, queryOutput.getRecords().size(), "Query with the transaction should see the new row."); + + transaction.rollback(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testEmptyInList() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.IN, List.of()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "IN empty list should find nothing."); + + queryInput.setFilter(new QQueryFilter().withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.NOT_IN, List.of()))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(5, queryOutput.getRecords().size(), "NOT_IN empty list should find everything."); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "OR should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndOrOr() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Maes"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Darin"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(2, queryOutput.getRecords().size(), "Complex query should find 2 rows"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("James") && r.getValueString("lastName").equals("Maes")); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Darin") && r.getValueString("lastName").equals("Kelkhoff")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterOrAndAnd() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testNestedFilterAndTopLevelFilter() throws QException + { + QueryInput queryInput = initQueryRequest(); + queryInput.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 3)) + .withBooleanOperator(QQueryFilter.BooleanOperator.AND) + .withSubFilters(List.of( + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("James"))) + .withCriteria(new QFilterCriteria("firstName", QCriteriaOperator.EQUALS, List.of("Tim"))), + new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Kelkhoff"))) + .withCriteria(new QFilterCriteria("lastName", QCriteriaOperator.EQUALS, List.of("Chamberlain"))) + )) + ); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + assertEquals(1, queryOutput.getRecords().size(), "Complex query should find 1 row"); + assertThat(queryOutput.getRecords()).anyMatch(r -> r.getValueInteger("id").equals(3) && r.getValueString("firstName").equals("Tim") && r.getValueString("lastName").equals("Chamberlain")); + + queryInput.getFilter().setCriteria(List.of(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, 3))); + queryOutput = new QueryAction().execute(queryInput); + assertEquals(0, queryOutput.getRecords().size(), "Next complex query should find 0 rows"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithOrQueries() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, List.of(1)), + new QFilterCriteria("shipToPersonId", QCriteriaOperator.EQUALS, List.of(5)) + ).withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(5) + .allMatch(r -> Objects.equals(r.getValueInteger("billToPersonId"), 1) || Objects.equals(r.getValueInteger("shipToPersonId"), 5)); + + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, List.of(1)), + new QFilterCriteria("shipToPersonId", QCriteriaOperator.EQUALS, List.of(5)) + ).withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(2)) + .allMatch(r -> Objects.equals(r.getValueInteger("billToPersonId"), 1) || Objects.equals(r.getValueInteger("shipToPersonId"), 5)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityWithSubFilters() throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + + queryInput.setFilter(new QQueryFilter() + .withBooleanOperator(QQueryFilter.BooleanOperator.OR) + .withSubFilters(List.of( + new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 2), new QFilterCriteria("billToPersonId", QCriteriaOperator.EQUALS, 1)), + new QQueryFilter(new QFilterCriteria("billToPersonId", QCriteriaOperator.IS_BLANK), new QFilterCriteria("shipToPersonId", QCriteriaOperator.IS_BLANK)).withBooleanOperator(QQueryFilter.BooleanOperator.OR) + ))); + Predicate p = r -> r.getValueInteger("billToPersonId") == null || r.getValueInteger("shipToPersonId") == null || (r.getValueInteger("id") >= 2 && r.getValueInteger("billToPersonId") == 1); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(4) + .allMatch(p); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(1) + .allMatch(r -> r.getValueInteger("storeId").equals(1)) + .allMatch(p); + + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 3)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(3) + .allMatch(r -> r.getValueInteger("storeId").equals(3)) + .allMatch(p); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRecordSecurityNullValues() throws Exception + { + runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (9, NULL, 1, 6)", null); + runTestSql("INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (10, NULL, 6, 5)", null); + + QInstance qInstance = TestUtils.defineInstance(); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_ORDER); + reInitInstanceInContext(qInstance); + + Predicate hasNullStoreId = r -> r.getValueInteger("storeId") == null; + + //////////////////////////////////////////// + // all-access user should get all 10 rows // + //////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // no-values user should get 0 rows (given that default null-behavior on this key type is DENY) // + ////////////////////////////////////////////////////////////////////////////////////////////////// + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // user with list of all ids shouldn't see the nulls (given that default null-behavior on this key type is DENY) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(8) + .noneMatch(hasNullStoreId); + } + + ////////////////////////////////////////////////////////////////////////// + // specifically set the null behavior to deny - repeat the last 2 tests // + ////////////////////////////////////////////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.DENY); + + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()).isEmpty(); + + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(8) + .noneMatch(hasNullStoreId); + } + + /////////////////////////////////// + // change null behavior to ALLOW // + /////////////////////////////////// + qInstance.getTable(TestUtils.TABLE_NAME_ORDER).getRecordSecurityLocks().get(0).setNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW); + + ///////////////////////////////////////////// + // all-access user should still get all 10 // + ///////////////////////////////////////////// + QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true)); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + + ///////////////////////////////////////////////////// + // no-values user should only get the rows w/ null // + ///////////////////////////////////////////////////// + QContext.setQSession(new QSession()); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(2) + .allMatch(hasNullStoreId); + + //////////////////////////////////////////////////// + // user with list of all ids should see the nulls // + //////////////////////////////////////////////////// + { + QSession qSession = new QSession(); + for(Integer i : List.of(1, 2, 3, 4, 5)) + { + qSession.withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, i); + } + QContext.setQSession(qSession); + assertThat(new QueryAction().execute(queryInput).getRecords()) + .hasSize(10) + .anyMatch(hasNullStoreId); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + @SuppressWarnings("unchecked") + void testHeavyFields() throws QException + { + ////////////////////////////////////////////////////////// + // set homeTown field as heavy - so it won't be fetched // + ////////////////////////////////////////////////////////// + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON) + .getField("homeTown") + .withIsHeavy(true); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(TestUtils.TABLE_NAME_PERSON); + List records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("No records should have the heavy homeTown field set").noneMatch(r -> r.getValue("homeTown") != null); + assertThat(records).describedAs("Some records should have a homeTown length backend detail set").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") != null); + assertThat(records).describedAs("Some records should have a null homeTown length backend").anyMatch(r -> ((Map) r.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).get("homeTown") == null); + + ////////////////////////////////////////////// + // re-do the query, requesting heavy fields // + ////////////////////////////////////////////// + queryInput.setShouldFetchHeavyFields(true); + records = new QueryAction().execute(queryInput).getRecords(); + assertThat(records).describedAs("Some records should have the heavy homeTown field set when heavies are requested").anyMatch(r -> r.getValue("homeTown") != null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNamesToInclude() throws QException + { + QQueryFilter filter = new QQueryFilter().withCriteria("id", QCriteriaOperator.EQUALS, 1); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_PERSON).withFilter(filter); + + QRecord record = new QueryAction().execute(queryInput.withFieldNamesToInclude(null)).getRecords().get(0); + assertTrue(record.getValues().containsKey("id")); + assertTrue(record.getValues().containsKey("firstName")); + assertTrue(record.getValues().containsKey("createDate")); + assertEquals(QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON).getFields().size(), record.getValues().size()); + + record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("id", "firstName"))).getRecords().get(0); + assertTrue(record.getValues().containsKey("id")); + assertTrue(record.getValues().containsKey("firstName")); + assertFalse(record.getValues().containsKey("createDate")); + assertEquals(2, record.getValues().size()); + + record = new QueryAction().execute(queryInput.withFieldNamesToInclude(Set.of("homeTown"))).getRecords().get(0); + assertFalse(record.getValues().containsKey("id")); + assertFalse(record.getValues().containsKey("firstName")); + assertFalse(record.getValues().containsKey("createDate")); + assertEquals(1, record.getValues().size()); + } + +} diff --git a/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java new file mode 100644 index 00000000..29d6540c --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/java/com/kingsrook/qqq/backend/module/sqlite/actions/SQLiteUpdateActionTest.java @@ -0,0 +1,443 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.sqlite.actions; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; +import com.kingsrook.qqq.backend.module.sqlite.BaseTest; +import com.kingsrook.qqq.backend.module.sqlite.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SQLiteUpdateActionTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + public void beforeEach() throws Exception + { + getBaseRDBMSActionStrategyAndActivateCollectingStatistics(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateNullList() throws QException + { + UpdateInput updateInput = initUpdateRequest(); + updateInput.setRecords(null); + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + assertEquals(0, updateResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateEmptyList() throws QException + { + UpdateInput updateInput = initUpdateRequest(); + updateInput.setRecords(Collections.emptyList()); + new UpdateAction().execute(updateInput); + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + assertEquals(0, updateResult.getRecords().size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateOne() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record = new QRecord() + .withValue("id", 2) + .withValue("firstName", "James") + .withValue("lastName", "Kirk") + .withValue("email", "jamestk@starfleet.net") + .withValue("birthDate", "2210-05-20"); + updateInput.setRecords(List.of(record)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(1, updateResult.getRecords().size(), "Should return 1 row"); + assertEquals(2, updateResult.getRecords().get(0).getValue("id"), "Should have id=2 in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'Kirk'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(2, rs.getInt("id")); + assertEquals("James", rs.getString("first_name")); + assertEquals("2210-05-20", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Maes'", (rs -> + { + if(rs.next()) + { + fail("Should not have found Maes any more."); + } + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithDifferentColumnsAndValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record1 = new QRecord() + .withValue("id", 1) + .withValue("firstName", "Darren") + .withValue("lastName", "From Bewitched") + .withValue("birthDate", "1900-01-01"); + + QRecord record2 = new QRecord() + .withValue("id", 3) + .withValue("firstName", "Wilt") + .withValue("birthDate", null); + + QRecord record3 = new QRecord() + .withValue("id", 5) + .withValue("firstName", "Richard") + .withValue("birthDate", null); + + updateInput.setRecords(List.of(record1, record2, record3)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + + // this test runs one batch and one regular query + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(3, updateResult.getRecords().size(), "Should return 3 rows"); + assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); + assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + assertEquals(5, updateResult.getRecords().get(2).getValue("id"), "Should have expected ids in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(1, rs.getInt("id")); + assertEquals("Darren", rs.getString("first_name")); + assertEquals("From Bewitched", rs.getString("last_name")); + assertEquals("1900-01-01", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Chamberlain'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(3, rs.getInt("id")); + assertEquals("Wilt", rs.getString("first_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Richardson'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(5, rs.getInt("id")); + assertEquals("Richard", rs.getString("first_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsDifferentValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + QRecord record1 = new QRecord() + .withValue("id", 1) + .withValue("firstName", "Darren") + .withValue("lastName", "From Bewitched") + .withValue("birthDate", "1900-01-01"); + + QRecord record2 = new QRecord() + .withValue("id", 3) + .withValue("firstName", "Wilt") + .withValue("lastName", "Tim's Uncle") + .withValue("birthDate", null); + + updateInput.setRecords(List.of(record1, record2)); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(1, statistics.get(BaseRDBMSActionStrategy.STAT_BATCHES_RAN)); + + assertEquals(2, updateResult.getRecords().size(), "Should return 2 rows"); + assertEquals(1, updateResult.getRecords().get(0).getValue("id"), "Should have expected ids in the row"); + assertEquals(3, updateResult.getRecords().get(1).getValue("id"), "Should have expected ids in the row"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE last_name = 'From Bewitched'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(1, rs.getInt("id")); + assertEquals("Darren", rs.getString("first_name")); + assertEquals("From Bewitched", rs.getString("last_name")); + assertEquals("1900-01-01", rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + runTestSql("SELECT * FROM person WHERE last_name = 'Tim''s Uncle'", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals(3, rs.getInt("id")); + assertEquals("Wilt", rs.getString("first_name")); + assertEquals("Tim's Uncle", rs.getString("last_name")); + assertNull(rs.getString("birth_date")); + } + assertEquals(1, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testUpdateManyWithSameColumnsSameValues() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + for(int i = 1; i <= 5; i++) + { + records.add(new QRecord() + .withValue("id", i) + .withValue("birthDate", "1999-09-09")); + } + + updateInput.setRecords(records); + + UpdateOutput updateResult = new UpdateAction().execute(updateInput); + Map statistics = getBaseRDBMSActionStrategy().getStatistics(); + assertEquals(2, statistics.get(BaseRDBMSActionStrategy.STAT_QUERIES_RAN)); + + assertEquals(5, updateResult.getRecords().size(), "Should return 5 rows"); + // todo - add errors to QRecord? assertTrue(updateResult.getRecords().stream().noneMatch(qrs -> CollectionUtils.nullSafeHasContents(qrs.getErrors())), "There should be no errors"); + runTestSql("SELECT * FROM person WHERE id <= 5", (rs -> + { + int rowsFound = 0; + while(rs.next()) + { + rowsFound++; + assertEquals("1999-09-09", rs.getString("birth_date")); + } + assertEquals(5, rowsFound); + })); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testModifyDateGetsUpdated() throws Exception + { + String originalModifyDate = selectModifyDate(1); + + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", 1) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + new UpdateAction().execute(updateInput); + + String updatedModifyDate = selectModifyDate(1); + + assertTrue(StringUtils.hasContent(originalModifyDate)); + assertTrue(StringUtils.hasContent(updatedModifyDate)); + assertNotEquals(originalModifyDate, updatedModifyDate); + } + + + + /******************************************************************************* + ** This situation - fails in a real mysql, but not in h2... anyway, because mysql + ** didn't want to convert the date-time string format to a date-time. + *******************************************************************************/ + @Test + void testDateTimesCanBeModifiedFromIsoStrings() throws Exception + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", 1) + .withValue("createDate", "2022-10-03T10:29:35Z") + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + new UpdateAction().execute(updateInput); + } + + + + /******************************************************************************* + ** Make sure that records without a primary key come back with error. + *******************************************************************************/ + @Test + void testWithoutPrimaryKeyErrors() throws Exception + { + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + } + + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", null) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + } + + { + UpdateInput updateInput = initUpdateRequest(); + List records = new ArrayList<>(); + records.add(new QRecord() + .withValue("id", null) + .withValue("firstName", "Johnny Not Updated")); + records.add(new QRecord() + .withValue("id", 2) + .withValue("firstName", "Johnny Updated")); + updateInput.setRecords(records); + UpdateOutput updateOutput = new UpdateAction().execute(updateInput); + + assertFalse(updateOutput.getRecords().get(0).getErrors().isEmpty()); + assertEquals("Missing value in primary key field", updateOutput.getRecords().get(0).getErrors().get(0).getMessage()); + + assertTrue(updateOutput.getRecords().get(1).getErrors().isEmpty()); + + GetInput getInput = new GetInput(); + getInput.setTableName(TestUtils.TABLE_NAME_PERSON); + getInput.setPrimaryKey(2); + GetOutput getOutput = new GetAction().execute(getInput); + assertEquals("Johnny Updated", getOutput.getRecord().getValueString("firstName")); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String selectModifyDate(Integer id) throws Exception + { + StringBuilder modifyDate = new StringBuilder(); + runTestSql("SELECT modify_date FROM person WHERE id = " + id, (rs -> + { + if(rs.next()) + { + modifyDate.append(rs.getString("modify_date")); + } + })); + return (modifyDate.toString()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private UpdateInput initUpdateRequest() + { + UpdateInput updateInput = new UpdateInput(); + updateInput.setTableName(TestUtils.TABLE_NAME_PERSON); + return updateInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql b/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql new file mode 100644 index 00000000..e512a5d4 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/resources/prime-test-database-parent-child-tables.sql @@ -0,0 +1,49 @@ +-- +-- QQQ - Low-code Application Framework for Engineers. +-- Copyright (C) 2021-2025. Kingsrook, LLC +-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +-- contact@kingsrook.com +-- https://github.com/Kingsrook/ +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . +-- + +DROP TABLE IF EXISTS parent_table; +DROP TABLE IF EXISTS child_table; + +CREATE TABLE child_table +( + id INT AUTO_INCREMENT primary key, + name VARCHAR(80) NOT NULL +); + +INSERT INTO child_table (id, name) VALUES (1, 'Timmy'); +INSERT INTO child_table (id, name) VALUES (2, 'Jimmy'); +INSERT INTO child_table (id, name) VALUES (3, 'Johnny'); +INSERT INTO child_table (id, name) VALUES (4, 'Gracie'); +INSERT INTO child_table (id, name) VALUES (5, 'Suzie'); + +CREATE TABLE parent_table +( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(80) NOT NULL, + child_id INT, + foreign key (child_id) references child_table(id) +); + +INSERT INTO parent_table (id, name, child_id) VALUES (1, 'Tim''s Dad', 1); +INSERT INTO parent_table (id, name, child_id) VALUES (2, 'Tim''s Mom', 1); +INSERT INTO parent_table (id, name, child_id) VALUES (3, 'Childless Man', null); +INSERT INTO parent_table (id, name, child_id) VALUES (4, 'Childless Woman', null); +INSERT INTO parent_table (id, name, child_id) VALUES (5, 'Johny''s Single Dad', 3); diff --git a/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql b/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql new file mode 100644 index 00000000..874f7583 --- /dev/null +++ b/qqq-backend-module-sqlite/src/test/resources/prime-test-database.sql @@ -0,0 +1,215 @@ +-- +-- QQQ - Low-code Application Framework for Engineers. +-- Copyright (C) 2021-2025. Kingsrook, LLC +-- 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States +-- contact@kingsrook.com +-- https://github.com/Kingsrook/ +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU Affero General Public License for more details. +-- +-- You should have received a copy of the GNU Affero General Public License +-- along with this program. If not, see . +-- + +DROP TABLE IF EXISTS person; +CREATE TABLE person +( + id INTEGER PRIMARY KEY, + create_date TIMESTAMP, -- DEFAULT datetime('now'), -- can't get this to work! + modify_date TIMESTAMP, -- DEFAULT datetime('now'), + + first_name VARCHAR(80) NOT NULL, + last_name VARCHAR(80) NOT NULL, + birth_date DATE, + email VARCHAR(250) NOT NULL, + is_employed BOOLEAN, + annual_salary DECIMAL(12,2), + days_worked INTEGER, + home_town VARCHAR(80), + start_time TIME +); + +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (1, 'Darin', 'Kelkhoff', '1980-05-31', 'darin.kelkhoff@gmail.com', 1, 25000, 27, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (2, 'James', 'Maes', '1980-05-15', 'jmaes@mmltholdings.com', 1, 26000, 124, 'Chester'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (3, 'Tim', 'Chamberlain', '1976-05-28', 'tchamberlain@mmltholdings.com', 0, null, 0, 'Decatur'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 30000, 99, 'Texas'); +INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked, home_town) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 1, 1000000, 232, null); + +DROP TABLE IF EXISTS personal_id_card; +CREATE TABLE personal_id_card +( + id INTEGER PRIMARY KEY, + create_date TIMESTAMP, -- DEFAULT date(), + modify_date TIMESTAMP, -- DEFAULT date(), + person_id INTEGER, + id_number VARCHAR(250) +); + +INSERT INTO personal_id_card (person_id, id_number) VALUES (1, '19800531'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (2, '19800515'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (3, '19760528'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (6, '123123123'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (null, '987987987'); +INSERT INTO personal_id_card (person_id, id_number) VALUES (null, '456456456'); + +DROP TABLE IF EXISTS carrier; +CREATE TABLE carrier +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) NOT NULL, + company_code VARCHAR(80) NOT NULL, + service_level VARCHAR(80) NOT NULL +); + +INSERT INTO carrier (id, name, company_code, service_level) VALUES (1, 'UPS Ground', 'UPS', 'G'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (2, 'UPS 2Day', 'UPS', '2'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (3, 'UPS International', 'UPS', 'I'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (4, 'Fedex Ground', 'FEDEX', 'G'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (5, 'Fedex Next Day', 'UPS', '1'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (6, 'Will Call', 'WILL_CALL', 'W'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (7, 'USPS Priority', 'USPS', '1'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (8, 'USPS Super Slow', 'USPS', '4'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (9, 'USPS Super Fast', 'USPS', '0'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (10, 'DHL International', 'DHL', 'I'); +INSERT INTO carrier (id, name, company_code, service_level) VALUES (11, 'GSO', 'GSO', 'G'); + +DROP TABLE IF EXISTS line_item_extrinsic; +DROP TABLE IF EXISTS order_line; +DROP TABLE IF EXISTS item; +DROP TABLE IF EXISTS `order`; +DROP TABLE IF EXISTS order_instructions; +DROP TABLE IF EXISTS warehouse_store_int; +DROP TABLE IF EXISTS store; +DROP TABLE IF EXISTS warehouse; + +CREATE TABLE store +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) NOT NULL +); + +-- define 3 stores +INSERT INTO store (id, name) VALUES (1, 'Q-Mart'); +INSERT INTO store (id, name) VALUES (2, 'QQQ ''R'' Us'); +INSERT INTO store (id, name) VALUES (3, 'QDepot'); + +CREATE TABLE item +( + id INTEGER PRIMARY KEY, + sku VARCHAR(80) NOT NULL, + description VARCHAR(80), + store_id INT NOT NULL REFERENCES store +); + +-- three items for each store +INSERT INTO item (id, sku, description, store_id) VALUES (1, 'QM-1', 'Q-Mart Item 1', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (2, 'QM-2', 'Q-Mart Item 2', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (3, 'QM-3', 'Q-Mart Item 3', 1); +INSERT INTO item (id, sku, description, store_id) VALUES (4, 'QRU-1', 'QQQ R Us Item 4', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (5, 'QRU-2', 'QQQ R Us Item 5', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (6, 'QRU-3', 'QQQ R Us Item 6', 2); +INSERT INTO item (id, sku, description, store_id) VALUES (7, 'QD-1', 'QDepot Item 7', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (8, 'QD-2', 'QDepot Item 8', 3); +INSERT INTO item (id, sku, description, store_id) VALUES (9, 'QD-3', 'QDepot Item 9', 3); + +CREATE TABLE `order` +( + id INTEGER PRIMARY KEY, + store_id INT REFERENCES store, + bill_to_person_id INT, + ship_to_person_id INT, + current_order_instructions_id INT -- f-key to order_instructions, which also has an f-key back here! +); + +-- variable orders +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (1, 1, 1, 1); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (2, 1, 1, 2); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (3, 1, 2, 3); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (4, 2, 4, 5); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (5, 2, 5, 4); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (6, 3, 5, null); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (7, 3, null, 5); +INSERT INTO `order` (id, store_id, bill_to_person_id, ship_to_person_id) VALUES (8, 3, null, 5); + +CREATE TABLE order_instructions +( + id INTEGER PRIMARY KEY, + order_id INT, + instructions VARCHAR(250) +); + +-- give orders 1 & 2 multiple versions of the instruction record +INSERT INTO order_instructions (id, order_id, instructions) VALUES (1, 1, 'order 1 v1'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (2, 1, 'order 1 v2'); +UPDATE `order` SET current_order_instructions_id = 2 WHERE id=1; + +INSERT INTO order_instructions (id, order_id, instructions) VALUES (3, 2, 'order 2 v1'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (4, 2, 'order 2 v2'); +INSERT INTO order_instructions (id, order_id, instructions) VALUES (5, 2, 'order 2 v3'); +UPDATE `order` SET current_order_instructions_id = 5 WHERE id=2; + +-- give all other orders just 1 instruction +INSERT INTO order_instructions (order_id, instructions) SELECT id, concat('order ', id, ' v1') FROM `order` WHERE current_order_instructions_id IS NULL; +UPDATE `order` SET current_order_instructions_id = (SELECT MIN(id) FROM order_instructions WHERE order_id = `order`.id) WHERE current_order_instructions_id is null; + +CREATE TABLE order_line +( + id INTEGER PRIMARY KEY, + order_id INT REFERENCES `order`, + sku VARCHAR(80), + store_id INT REFERENCES store, + quantity INT +); + +-- various lines +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-1', 1, 10); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-2', 1, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (1, 'QM-3', 1, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (2, 'QRU-1', 2, 1); -- this line has an item from a different store than its order. +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (3, 'QM-1', 1, 20); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (4, 'QRU-1', 2, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (4, 'QRU-2', 2, 2); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (5, 'QRU-1', 2, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (6, 'QD-1', 3, 1); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (7, 'QD-1', 3, 2); +INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (8, 'QD-1', 3, 3); + + +CREATE TABLE warehouse +( + id INTEGER PRIMARY KEY, + name VARCHAR(80) +); + +INSERT INTO warehouse (name) VALUES ('Patterson'); +INSERT INTO warehouse (name) VALUES ('Edison'); +INSERT INTO warehouse (name) VALUES ('Stockton'); +INSERT INTO warehouse (name) VALUES ('Somewhere in Texas'); + +CREATE TABLE warehouse_store_int +( + id INTEGER PRIMARY KEY, + warehouse_id INT REFERENCES `warehouse`, + store_id INT REFERENCES `store` +); + +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 1); +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 2); +INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 3); + +CREATE TABLE line_item_extrinsic +( + id INTEGER PRIMARY KEY, + order_line_id INT REFERENCES order_line, + `key` VARCHAR(80), + `value` VARCHAR(80) +); + From 8e65255248d4ab6a427ccff168c8ac36ba7c7bf7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 19:58:12 -0600 Subject: [PATCH 081/209] Delete and cleanup in QueryManger; test coverage improvements --- .../jdbc/BaseC3P0ConnectionCustomizer.java | 22 +- .../module/rdbms/jdbc/QueryManager.java | 593 +----------------- .../strategy/BaseRDBMSActionStrategy.java | 2 +- .../C3P0PooledConnectionProviderTest.java | 31 + .../module/rdbms/jdbc/QueryManagerTest.java | 23 - .../MySQLFullTextIndexFieldStrategyTest.java | 76 +++ 6 files changed, 127 insertions(+), 620 deletions(-) create mode 100644 qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java index 2e0549d3..87d26536 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/BaseC3P0ConnectionCustomizer.java @@ -27,6 +27,7 @@ import java.sql.Statement; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.mchange.v2.c3p0.ConnectionCustomizer; @@ -38,6 +39,8 @@ import com.mchange.v2.c3p0.ConnectionCustomizer; *******************************************************************************/ public class BaseC3P0ConnectionCustomizer implements ConnectionCustomizer { + private static final QLogger LOG = QLogger.getLogger(BaseC3P0ConnectionCustomizer.class); + private static Map> queriesForNewConnections = new HashMap<>(); @@ -48,15 +51,23 @@ public class BaseC3P0ConnectionCustomizer implements ConnectionCustomizer @Override public void onAcquire(Connection connection, String dataSourceIdentityToken) throws Exception { - List queries = queriesForNewConnections.get(dataSourceIdentityToken); - if(CollectionUtils.nullSafeHasContents(queries)) + try { - for(String sql : queries) + List queries = queriesForNewConnections.get(dataSourceIdentityToken); + if(CollectionUtils.nullSafeHasContents(queries)) { - Statement statement = connection.createStatement(); - statement.execute(sql); + for(String sql : queries) + { + Statement statement = connection.createStatement(); + statement.execute(sql); + } } } + catch(Exception e) + { + LOG.warn("Exception on a query-for-new-connection", e); + throw (e); + } } @@ -99,6 +110,7 @@ public class BaseC3P0ConnectionCustomizer implements ConnectionCustomizer } + /*************************************************************************** * ***************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java index ceed311a..72069193 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManager.java @@ -22,7 +22,6 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; -import java.io.Serializable; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.Connection; @@ -31,7 +30,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; @@ -43,7 +41,6 @@ import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; @@ -53,16 +50,14 @@ import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.NotImplementedException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** + ** Note that much of this class is/was ported (well, copied) to BaseRDBMSActionStrategy + ** around 2025-01, during the addition of SQLite backend module. *******************************************************************************/ public class QueryManager { @@ -169,49 +164,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeStatementForeachResult(Connection connection, String sql, ResultSetProcessor processor, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - PreparedStatement statement = null; - ResultSet resultSet = null; - - try - { - if(params.length == 1 && params[0] instanceof Collection) - { - params = ((Collection) params[0]).toArray(); - } - - statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - resultSet = statement.getResultSet(); - - while(resultSet.next()) - { - processor.processResultSet(resultSet); - } - } - finally - { - if(statement != null) - { - statement.close(); - } - - if(resultSet != null) - { - resultSet.close(); - } - } - */ - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -269,57 +221,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static Map executeStatementForSingleRow(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - if(resultSet.next()) - { - Map rs = new LinkedHashMap<>(); - - ResultSetMetaData metaData = resultSet.getMetaData(); - for(int i = 1; i <= metaData.getColumnCount(); i++) - { - rs.put(metaData.getColumnName(i), getObject(resultSet, i)); - } - - return (rs); - } - else - { - return (null); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static SimpleEntity executeStatementForSimpleEntity(Connection connection, String sql, Object... params) throws SQLException - { - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - if(resultSet.next()) - { - return (buildSimpleEntity(resultSet)); - } - else - { - return (null); - } - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -347,48 +248,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static List executeStatementForSimpleEntityList(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - List rs = new ArrayList<>(); - - PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params); - statement.execute(); - ResultSet resultSet = statement.getResultSet(); - while(resultSet.next()) - { - SimpleEntity row = buildSimpleEntity(resultSet); - - rs.add(row); - } - - return (rs); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static SimpleEntity buildSimpleEntity(ResultSet resultSet) throws SQLException - { - SimpleEntity row = new SimpleEntity(); - - ResultSetMetaData metaData = resultSet.getMetaData(); - for(int i = 1; i <= metaData.getColumnCount(); i++) - { - row.put(metaData.getColumnName(i), getObject(resultSet, i)); - } - return row; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -429,38 +288,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeUpdateVoid(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeUpdateVoid(Connection connection, String sql, List params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - } - */ - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -481,189 +308,6 @@ public class QueryManager - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeUpdateForRowCount(Connection connection, String sql, List params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = prepareStatementAndBindParams(connection, sql, params)) - { - statement.executeUpdate(); - return (statement.getUpdateCount()); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeInsertForGeneratedId(Connection connection, String sql, Object... params) throws SQLException - { - throw (new NotImplementedException()); - /* - try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) - { - bindParams(params, statement); - statement.executeUpdate(); - ResultSet generatedKeys = statement.getGeneratedKeys(); - if(generatedKeys.next()) - { - return (getInteger(generatedKeys, 1)); - } - else - { - return (null); - } - } - */ - } - - - - /******************************************************************************* - ** todo - needs (specific) unit test - *******************************************************************************/ - public static List executeInsertForGeneratedIds(Connection connection, String sql, List params, QFieldType idType) throws SQLException - { - try(PreparedStatement statement = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS)) - { - bindParams(params.toArray(), statement); - incrementStatistic(STAT_QUERIES_RAN); - statement.executeUpdate(); - - ///////////////////////////////////////////////////////////// - // We default to idType of INTEGER if it was not passed in // - ///////////////////////////////////////////////////////////// - if(idType == null) - { - idType = QFieldType.INTEGER; - } - - ResultSet generatedKeys = statement.getGeneratedKeys(); - List rs = new ArrayList<>(); - while(generatedKeys.next()) - { - switch(idType) - { - case INTEGER: - { - rs.add(getInteger(generatedKeys, 1)); - break; - } - case LONG: - { - rs.add(getLong(generatedKeys, 1)); - break; - } - default: - { - LOG.warn("Unknown id data type, attempting to getInteger.", logPair("sql", sql)); - rs.add(getInteger(generatedKeys, 1)); - break; - } - } - } - return (rs); - } - catch(SQLException e) - { - LOG.warn("SQLException", e, logPair("sql", sql)); - throw (e); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeInsertForList(Connection connection, List entityList) throws SQLException - { - throw (new NotImplementedException()); - /* - List> pages = CollectionUtils.getPages(entityList, PAGE_SIZE); - for(List page : pages) - { - ArrayList columns = new ArrayList<>(page.get(0).keySet()); - String sql = "INSERT INTO " + page.get(0).getTableName() + "(" + StringUtils.join(",", columns) + ") VALUES (" + columns.stream().map(s -> "?").collect(Collectors.joining(",")) + ")"; - - PreparedStatement insertPS = connection.prepareStatement(sql); - for(SimpleEntity entity : page) - { - Object[] params = new Object[columns.size()]; - for(int i = 0; i < columns.size(); i++) - { - params[i] = entity.get(columns.get(i)); - } - - bindParams(insertPS, params); - insertPS.addBatch(); - } - insertPS.executeBatch(); - } - - for(List page : pages) - { - page.clear(); - } - pages.clear(); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer executeInsert(Connection connection, SimpleEntity entity) throws SQLException - { - throw (new NotImplementedException()); - /* - ArrayList columns = new ArrayList<>(entity.keySet()); - String sql = "INSERT INTO " + entity.getTableName() + "(" + StringUtils.join(",", columns) + ") VALUES (" + columns.stream().map(s -> "?").collect(Collectors.joining(",")) + ")"; - - Object[] params = new Object[columns.size()]; - for(int i = 0; i < columns.size(); i++) - { - params[i] = entity.get(columns.get(i)); - } - - return (executeInsertForGeneratedId(connection, sql, params)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static void executeBatchUpdate(Connection connection, String updateSQL, List> values) throws SQLException - { - for(List> page : CollectionUtils.getPages(values, PAGE_SIZE)) - { - PreparedStatement updatePS = connection.prepareStatement(updateSQL); - for(List row : page) - { - Object[] params = new Object[row.size()]; - for(int i = 0; i < row.size(); i++) - { - params[i] = row.get(i); - } - - bindParams(updatePS, params); - updatePS.addBatch(); - } - incrementStatistic(STAT_BATCHES_RAN); - updatePS.executeBatch(); - } - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -734,15 +378,8 @@ public class QueryManager /******************************************************************************* * index is 1-based!! *******************************************************************************/ - @SuppressWarnings("unchecked") public static int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException { - /* if(value instanceof TypeValuePair) - { - bindParamTypeValuePair(statement, index, (TypeValuePair) value); - return (1); - } - else*/ if(value instanceof Integer i) { bindParam(statement, index, i); @@ -856,68 +493,6 @@ public class QueryManager } } - /******************************************************************************* - ** - *******************************************************************************/ - /* - public static TypeValuePair param(Class c, T v) - { - return (new TypeValuePair<>(c, v)); - } - */ - - /******************************************************************************* - ** - *******************************************************************************/ - /* - private static void bindParamTypeValuePair(PreparedStatement statement, int index, TypeValuePair value) throws SQLException - { - Object v = value.getValue(); - Class t = value.getType(); - - if(t.equals(Integer.class)) - { - bindParam(statement, index, (Integer) v); - } - else if(t.equals(String.class)) - { - bindParam(statement, index, (String) v); - } - else if(t.equals(Boolean.class)) - { - bindParam(statement, index, (Boolean) v); - } - else if(t.equals(Timestamp.class)) - { - bindParam(statement, index, (Timestamp) v); - } - else if(t.equals(Date.class)) - { - bindParam(statement, index, (Date) v); - } - else if(t.equals(Calendar.class)) - { - bindParam(statement, index, (Calendar) v); - } - else if(t.equals(LocalDate.class)) - { - bindParam(statement, index, (LocalDate) v); - } - else if(t.equals(LocalDateTime.class)) - { - bindParam(statement, index, (LocalDateTime) v); - } - else if(t.equals(BigDecimal.class)) - { - bindParam(statement, index, (BigDecimal) v); - } - else - { - throw (new SQLException("Unexpected value type [" + t.getSimpleName() + "] in bindParamTypeValuePair.")); - } - } - */ - /******************************************************************************* @@ -1675,170 +1250,6 @@ public class QueryManager - /******************************************************************************* - ** Find an id from a "large" table that was created X days ago (assumes the date - ** field in the table isn't indexed, but id is - so do a binary search on id, - ** selecting the date of the min & max & mid id, then sub-dividing until the goal - ** days-ago is found). - ** - *******************************************************************************/ - public static Integer findIdForDaysAgo(Connection connection, String tableName, String dateFieldName, int goalDaysAgo) throws SQLException - { - throw (new NotImplementedException()); - /* - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalDaysAgo, ChronoUnit.DAYS)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer findIdForTimestamp(Connection connection, String tableName, String dateFieldName, LocalDateTime timestamp) throws SQLException - { - throw (new NotImplementedException()); - /* - long between = ChronoUnit.SECONDS.between(timestamp, LocalDateTime.now()); - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, (int) between, ChronoUnit.SECONDS)); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - Integer maxId = executeStatementForSingleValue(connection, Integer.class, "SELECT MAX(id) FROM " + tableName); - Integer minId = executeStatementForSingleValue(connection, Integer.class, "SELECT MIN(id) FROM " + tableName); - - if(maxId == null || minId == null) - { - // Logger.logDebug("For [" + tableName + "], returning null id for X time-units ago, because either a min or max wasn't found."); - return (null); - } - - Integer idForGoal = findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, minId, maxId, unit); - long foundUnitsAgo = getTimeUnitAgo(connection, tableName, dateFieldName, idForGoal, unit); - - // Logger.logDebug("For [" + tableName + "], using min id [" + idForGoal + "], which is from [" + foundUnitsAgo + "] Units[" + unit + "] ago."); - - return (idForGoal); - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static Integer findIdForTimeUnitAgo(Connection connection, String tableName, String dateFieldName, int goalUnitsAgo, Integer minId, Integer maxId, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - Integer midId = minId + ((maxId - minId) / 2); - if(midId.equals(minId) || midId.equals(maxId)) - { - return (midId); - } - - long foundUnitsAgo = getTimeUnitAgo(connection, tableName, dateFieldName, midId, unit); - if(foundUnitsAgo == goalUnitsAgo) - { - return (midId); - } - else if(foundUnitsAgo > goalUnitsAgo) - { - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, midId, maxId, unit)); - } - else - { - return (findIdForTimeUnitAgo(connection, tableName, dateFieldName, goalUnitsAgo, minId, midId, unit)); - } - */ - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - private static long getTimeUnitAgo(Connection connection, String tableName, String dateFieldName, Integer id, ChronoUnit unit) throws SQLException - { - throw (new NotImplementedException()); - /* - LocalDateTime now = LocalDateTime.now(); - - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // note, we used to just do where id=? here - but if that row is ever missing, we have a bad time - so - do id >= ? order by id, and just the first row. // - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - LocalDateTime date = executeStatementForSingleValue(connection, LocalDateTime.class, "SELECT " + dateFieldName + " FROM " + tableName + " WHERE id >= ? ORDER BY id LIMIT 1", id); - // System.out.println(date); - - // if(date == null) - { - // return now. - } - // else - { - long diff = unit.between(date, now); - // System.out.println("Unit[" + unit + "]'s ago: " + diff); - return diff; - } - */ - } - - /******************************************************************************* - ** - *******************************************************************************/ - // public static class TypeValuePair - // { - // private Class type; - // private T value; - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // @SuppressWarnings("unchecked") - // public TypeValuePair(T value) - // { - // this.value = value; - // this.type = (Class) value.getClass(); - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public TypeValuePair(Class type, T value) - // { - // this.type = type; - // this.value = value; - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public T getValue() - // { - // return (value); - // } - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Class getType() - // { - // return (type); - // } - - // } - - - /******************************************************************************* ** Setter for collectStatistics ** diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java index 08309cd3..5bde3252 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategy.java @@ -61,7 +61,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* - ** + ** Note that much of this class came from the old (old) QueryManager class. *******************************************************************************/ public class BaseRDBMSActionStrategy implements RDBMSActionStrategyInterface { diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java index c588a06a..8657eb68 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/C3P0PooledConnectionProviderTest.java @@ -24,12 +24,17 @@ package com.kingsrook.qqq.backend.module.rdbms.jdbc; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.session.QSession; @@ -217,6 +222,32 @@ class C3P0PooledConnectionProviderTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testQueriesForNewConnections() throws Exception + { + String uuid = UUID.randomUUID().toString(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // change the default database backend to use the class under test here - the C3PL connection pool provider // + // and add a new-connection query to it to insert a record with a UUID, that we can then query for. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + QInstance qInstance = TestUtils.defineInstance(); + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) qInstance.getBackend(TestUtils.DEFAULT_BACKEND_NAME); + backend.setQueriesForNewConnections(List.of("insert into person (first_name, last_name, email) values ('D', 'K', '" + uuid + "')")); + backend.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class)); + QContext.init(qInstance, new QSession()); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // we actually get multiple, because c3p0 default config opens multiple connections at once // + ////////////////////////////////////////////////////////////////////////////////////////////// + List records = QueryAction.execute(TestUtils.TABLE_NAME_PERSON, new QQueryFilter(new QFilterCriteria("email", QCriteriaOperator.EQUALS, uuid))); + assertThat(records.size()).isGreaterThanOrEqualTo(1); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java index 396a8c77..222734fa 100644 --- a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/QueryManagerTest.java @@ -422,29 +422,6 @@ class QueryManagerTest extends BaseTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryForSimpleEntity() throws SQLException - { - try(Connection connection = getConnection()) - { - QueryManager.executeUpdate(connection, """ - INSERT INTO test_table - ( int_col, datetime_col, char_col, date_col, time_col ) - VALUES - ( 47, '2022-08-10 19:22:08', 'Q', '2022-08-10', '19:22:08') - """); - SimpleEntity simpleEntity = QueryManager.executeStatementForSimpleEntity(connection, "SELECT * FROM test_table"); - assertNotNull(simpleEntity); - assertEquals(47, simpleEntity.get("INT_COL")); - assertEquals("Q", simpleEntity.get("CHAR_COL")); - } - } - - - /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java new file mode 100644 index 00000000..25c16621 --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/MySQLFullTextIndexFieldStrategyTest.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for MySQLFullTextIndexFieldStrategy + *******************************************************************************/ +class MySQLFullTextIndexFieldStrategyTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + /////////////////////////////////////////////////// + // test an operator that uses the fulltext index // + /////////////////////////////////////////////////// + QFieldMetaData field = new QFieldMetaData("myText", QFieldType.TEXT); + QFilterCriteria criterion = new QFilterCriteria(field.getName(), QCriteriaOperator.LIKE, "hello"); + StringBuilder clause = new StringBuilder(); + Integer expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(1, expectedNoOfParams); + assertEquals(" MATCH (my_text) AGAINST (?) ", clause.toString()); + + //////////////////////////////////////////// + // test a negated fulltext index operator // + //////////////////////////////////////////// + criterion.setOperator(QCriteriaOperator.NOT_CONTAINS); + clause.delete(0, clause.length()); + expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(1, expectedNoOfParams); + assertEquals(" NOT MATCH (my_text) AGAINST (?) ", clause.toString()); + + //////////////////////////////////////////// + // an operator that should defer to super // + //////////////////////////////////////////// + criterion.setOperator(QCriteriaOperator.IS_BLANK); + clause.delete(0, clause.length()); + expectedNoOfParams = new MySQLFullTextIndexFieldStrategy().appendCriterionToWhereClause(criterion, clause, "my_text", List.of("hello"), field); + assertEquals(0, expectedNoOfParams); + assertEquals("my_text IS NULL OR my_text = ''", clause.toString()); + } + +} \ No newline at end of file From 009e14436161566cbc321c7a546bf30810f2a232 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 20:42:15 -0600 Subject: [PATCH 082/209] test coverage for rdbms --- .../module/rdbms/jdbc/SimpleEntity.java | 228 ------------------ .../jdbc/ConnectionProviderInterfaceTest.java | 63 +++++ 2 files changed, 63 insertions(+), 228 deletions(-) delete mode 100755 qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java create mode 100644 qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java deleted file mode 100755 index 8541d794..00000000 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/SimpleEntity.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.module.rdbms.jdbc; - - -import java.util.HashMap; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class SimpleEntity extends HashMap -{ - // private String tableName; - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity() - // { - // super(); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity with(String key, Object value) - // { - // put(key, value); - // return (this); - // } - - - - // /******************************************************************************* - // ** Return the current value of tableName - // ** - // ** @return tableName - // *******************************************************************************/ - // public String getTableName() - // { - // return (tableName); - // } - - - - // /******************************************************************************* - // ** Set the current value of tableName - // ** - // ** @param tableName - // *******************************************************************************/ - // public void setTableName(String tableName) - // { - // this.tableName = tableName; - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public SimpleEntity withTableName(String tableName) - // { - // setTableName(tableName); - // return (this); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Boolean getBoolean(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - - // if(o instanceof Boolean) - // { - // return ((Boolean) o); - // } - // else if(o instanceof Number) - // { - // int i = ((Number) o).intValue(); - // return (i != 0); - // } - // else if(o instanceof String) - // { - // String s = (String) o; - // return (s.equalsIgnoreCase("1") || s.equalsIgnoreCase("true") || s.equalsIgnoreCase("t")); - // } - // else - // { - // throw new IllegalArgumentException("Could not get value of object of type [" + o.getClass() + "] as Boolean."); - // } - - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public String getString(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - // if(o instanceof String) - // { - // return ((String) o); - // } - // else if(o instanceof byte[]) - // { - // return (new String((byte[]) o)); - // } - - // return String.valueOf(o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Integer getInteger(String columnName) - // { - // Object o = get(columnName); - // if(o instanceof Long) - // { - // return ((Long) o).intValue(); - // } - // else if(o instanceof Short) - // { - // return ((Short) o).intValue(); - // } - // else if(o instanceof String) - // { - // return (Integer.parseInt((String) o)); - // } - - // return ((Integer) o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public BigDecimal getBigDecimal(String columnName) - // { - // Object o = get(columnName); - // if(o == null) - // { - // return (null); - // } - - // if(o instanceof BigDecimal) - // { - // return ((BigDecimal) o); - // } - // else - // { - // return new BigDecimal(String.valueOf(o)); - // } - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public Long getLong(String columnName) - // { - // Object o = get(columnName); - // if(o instanceof Integer) - // { - // return ((Integer) o).longValue(); - // } - - // return ((Long) o); - // } - - - - // /******************************************************************************* - // ** - // *******************************************************************************/ - // public void trimStrings() - // { - // for(String key : keySet()) - // { - // Object value = get(key); - // if(value != null && value instanceof String) - // { - // put(key, ((String) value).trim()); - // } - // } - // } -} diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java new file mode 100644 index 00000000..f71fbcad --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/jdbc/ConnectionProviderInterfaceTest.java @@ -0,0 +1,63 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.jdbc; + + +import java.sql.Connection; +import java.sql.SQLException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for ConnectionProviderInterface + *******************************************************************************/ +class ConnectionProviderInterfaceTest extends BaseTest +{ + + + /*************************************************************************** + * jacoco doesn't like that our interface isn't covered, so... cover it + ***************************************************************************/ + @Test + void dumpDebug() throws SQLException + { + new ConnectionProviderInterface() + { + @Override + public void init(RDBMSBackendMetaData backend) throws QException + { + + } + + + + @Override + public Connection getConnection() throws SQLException + { + return null; + } + }.dumpDebug(); + } +} \ No newline at end of file From 62bf361e360dae6c6a4ee83f23f0bdac14895409 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 6 Jan 2025 10:35:28 -0600 Subject: [PATCH 083/209] Initial checkin --- .../strategy/BaseRDBMSActionStrategyTest.java | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java diff --git a/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java new file mode 100644 index 00000000..95c7784e --- /dev/null +++ b/qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/strategy/BaseRDBMSActionStrategyTest.java @@ -0,0 +1,161 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.rdbms.strategy; + + +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.GregorianCalendar; +import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus; +import com.kingsrook.qqq.backend.module.rdbms.BaseTest; +import com.kingsrook.qqq.backend.module.rdbms.TestUtils; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** Unit test for BaseRDBMSActionStrategy + *******************************************************************************/ +class BaseRDBMSActionStrategyTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() throws SQLException + { + try(Connection connection = getConnection()) + { + QueryManager.executeUpdate(connection, """ + CREATE TABLE test_table + ( + int_col INTEGER, + datetime_col DATETIME, + char_col CHAR(1), + date_col DATE, + time_col TIME, + long_col LONG + ) + """); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() throws SQLException + { + try(Connection connection = getConnection()) + { + QueryManager.executeUpdate(connection, "DROP TABLE test_table"); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private Connection getConnection() throws SQLException + { + return new ConnectionManager().getConnection(TestUtils.defineBackend()); + } + + + + /******************************************************************************* + ** Test the various overloads that bind params. + ** Note, we're just confirming that these methods don't throw... + *******************************************************************************/ + @Test + void testBindParams() throws SQLException + { + try(Connection connection = getConnection()) + { + long ctMillis = System.currentTimeMillis(); + PreparedStatement ps = connection.prepareStatement("UPDATE test_table SET int_col = ? WHERE int_col > 0"); + + BaseRDBMSActionStrategy strategy = new BaseRDBMSActionStrategy(); + + /////////////////////////////////////////////////////////////////////////////// + // these calls - we just want to assert that they don't throw any exceptions // + /////////////////////////////////////////////////////////////////////////////// + strategy.bindParamObject(ps, 1, (short) 1); + strategy.bindParamObject(ps, 1, (long) 1); + strategy.bindParamObject(ps, 1, true); + strategy.bindParamObject(ps, 1, BigDecimal.ONE); + strategy.bindParamObject(ps, 1, "hello".getBytes(StandardCharsets.UTF_8)); + strategy.bindParamObject(ps, 1, new Timestamp(ctMillis)); + strategy.bindParamObject(ps, 1, new Date(ctMillis)); + strategy.bindParamObject(ps, 1, new GregorianCalendar()); + strategy.bindParamObject(ps, 1, LocalDate.now()); + strategy.bindParamObject(ps, 1, OffsetDateTime.now()); + strategy.bindParamObject(ps, 1, LocalDateTime.now()); + strategy.bindParamObject(ps, 1, AutomationStatus.PENDING_INSERT_AUTOMATIONS); + + assertThrows(SQLException.class, () -> strategy.bindParamObject(ps, 1, new Object())); + + strategy.bindParam(ps, 1, (Integer) null); + strategy.bindParam(ps, 1, (Boolean) null); + strategy.bindParam(ps, 1, (BigDecimal) null); + strategy.bindParam(ps, 1, (byte[]) null); + strategy.bindParam(ps, 1, (Timestamp) null); + strategy.bindParam(ps, 1, (String) null); + strategy.bindParam(ps, 1, (Date) null); + strategy.bindParam(ps, 1, (GregorianCalendar) null); + strategy.bindParam(ps, 1, (LocalDate) null); + strategy.bindParam(ps, 1, (LocalDateTime) null); + + strategy.bindParam(ps, 1, 1); + strategy.bindParam(ps, 1, true); + strategy.bindParam(ps, 1, BigDecimal.ONE); + strategy.bindParam(ps, 1, "hello".getBytes(StandardCharsets.UTF_8)); + strategy.bindParam(ps, 1, new Timestamp(ctMillis)); + strategy.bindParam(ps, 1, "hello"); + strategy.bindParam(ps, 1, new Date(ctMillis)); + strategy.bindParam(ps, 1, new GregorianCalendar()); + strategy.bindParam(ps, 1, LocalDate.now()); + strategy.bindParam(ps, 1, LocalDateTime.now()); + + //////////////////////////////////////////////////////////////////////////////////////////////// + // originally longs were being downgraded to int when binding, so, verify that doesn't happen // + //////////////////////////////////////////////////////////////////////////////////////////////// + } + } + +} \ No newline at end of file From 32a8d65a841b2c7f89d998a47fae5f496fd21eb2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 6 Jan 2025 11:01:23 -0600 Subject: [PATCH 084/209] Copyrights and checkstyle --- qqq-backend-module-sqlite/pom.xml | 2 +- .../module/sqlite/SQLiteBackendModule.java | 21 +++++++++++++++++++ .../model/metadata/SQLiteBackendMetaData.java | 21 +++++++++++++++++++ .../metadata/SQLiteTableBackendDetails.java | 21 +++++++++++++++++++ .../strategy/SQLiteRDBMSActionStrategy.java | 21 +++++++++++++++++++ .../qqq/backend/module/sqlite/BaseTest.java | 2 +- .../qqq/backend/module/sqlite/TestUtils.java | 2 +- .../sqlite/actions/SQLiteCountActionTest.java | 2 +- .../actions/SQLiteDeleteActionTest.java | 2 +- .../actions/SQLiteInsertActionTest.java | 2 +- .../sqlite/actions/SQLiteQueryActionTest.java | 2 +- .../actions/SQLiteUpdateActionTest.java | 7 ++++--- 12 files changed, 95 insertions(+), 10 deletions(-) diff --git a/qqq-backend-module-sqlite/pom.xml b/qqq-backend-module-sqlite/pom.xml index 106dede0..51c255c6 100644 --- a/qqq-backend-module-sqlite/pom.xml +++ b/qqq-backend-module-sqlite/pom.xml @@ -1,7 +1,7 @@ org.apache.maven.plugins diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java new file mode 100644 index 00000000..17c447dd --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/SFTPBackendModule.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; +import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.AbstractSFTPAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPCountAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPDeleteAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPInsertAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPQueryAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPStorageAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.actions.SFTPUpdateAction; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; + + +/******************************************************************************* + ** QQQ Backend module for working with SFTP filesystems (as a client) + *******************************************************************************/ +public class SFTPBackendModule implements QBackendModuleInterface, FilesystemBackendModuleInterface +{ + public static final String BACKEND_TYPE = "sftp"; + + static + { + QBackendModuleDispatcher.registerBackendModule(new SFTPBackendModule()); + } + + /******************************************************************************* + ** For filesystem backends, get the module-specific action base-class, that helps + ** with functions like listing and deleting files. + *******************************************************************************/ + @Override + public AbstractBaseFilesystemAction getActionBase() + { + return (new AbstractSFTPAction()); + } + + + + /******************************************************************************* + ** Method where a backend module must be able to provide its type (name). + *******************************************************************************/ + @Override + public String getBackendType() + { + return (BACKEND_TYPE); + } + + + + /******************************************************************************* + ** Method to identify the class used for backend meta data for this module. + *******************************************************************************/ + @Override + public Class getBackendMetaDataClass() + { + return (SFTPBackendMetaData.class); + } + + + + /******************************************************************************* + ** Method to identify the class used for table-backend details for this module. + *******************************************************************************/ + @Override + public Class getTableBackendDetailsClass() + { + return (SFTPTableBackendDetails.class); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QueryInterface getQueryInterface() + { + return new SFTPQueryAction(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public CountInterface getCountInterface() + { + return new SFTPCountAction(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public InsertInterface getInsertInterface() + { + return (new SFTPInsertAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public UpdateInterface getUpdateInterface() + { + return (new SFTPUpdateAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public DeleteInterface getDeleteInterface() + { + return (new SFTPDeleteAction()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QStorageInterface getStorageInterface() + { + return new SFTPStorageAction(); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java new file mode 100644 index 00000000..80db887e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -0,0 +1,298 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; +import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import org.apache.sshd.client.SshClient; +import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.sftp.client.SftpClient; +import org.apache.sshd.sftp.client.SftpClientFactory; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Base class for all SFTP filesystem actions + *******************************************************************************/ +public class AbstractSFTPAction extends AbstractBaseFilesystemAction +{ + private static final QLogger LOG = QLogger.getLogger(AbstractSFTPAction.class); + + private SshClient sshClient; + private ClientSession clientSession; + private SftpClient sftpClient; + + + + /******************************************************************************* + ** Set up the sftp utils object to be used for this action. + *******************************************************************************/ + @Override + public void preAction(QBackendMetaData backendMetaData) throws QException + { + super.preAction(backendMetaData); + + if(sftpClient != null) + { + LOG.debug("sftpClient object is already set - not re-setting it."); + return; + } + + try + { + SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData); + + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + String username = sftpBackendMetaData.getUsername(); + String password = sftpBackendMetaData.getPassword(); + String hostName = sftpBackendMetaData.getHostName(); + Integer port = sftpBackendMetaData.getPort(); + + if(backendMetaData.getUsesVariants()) + { + QRecord variantRecord = getVariantRecord(backendMetaData); + LOG.debug("Getting SFTP connection credentials from variant record", + logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()), + logPair("id", variantRecord.getValue("id")), + logPair("name", variantRecord.getRecordLabel())); + Map fieldNameMap = backendMetaData.getBackendVariantsConfig().getBackendSettingSourceFieldNameMap(); + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.USERNAME)) + { + username = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.USERNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PASSWORD)) + { + password = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.PASSWORD)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.HOSTNAME)) + { + hostName = variantRecord.getValueString(fieldNameMap.get(SFTPBackendVariantSetting.HOSTNAME)); + } + + if(fieldNameMap.containsKey(SFTPBackendVariantSetting.PORT)) + { + port = variantRecord.getValueInteger(fieldNameMap.get(SFTPBackendVariantSetting.PORT)); + } + } + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + } + catch(IOException e) + { + throw (new QException("Error setting up SFTP connection", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Long getFileSize(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getSize(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileCreateDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getCreateTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Instant getFileModifyDate(SFTPDirEntryWithPath sftpDirEntryWithPath) + { + try + { + return sftpDirEntryWithPath.dirEntry().getAttributes().getModifyTime().toInstant(); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + { + try + { + String fullPath = getFullBasePath(table, backendBase); + List rs = new ArrayList<>(); + + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) + { + if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) + { + continue; + } + + if(dirEntry.getAttributes().isDirectory()) + { + // todo - recursive?? + continue; + } + + // todo filter/glob + // todo skip + // todo limit + // todo order by + rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry)); + } + + return (rs); + } + catch(Exception e) + { + throw (new QException("Error listing files", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public InputStream readFile(SFTPDirEntryWithPath dirEntry) throws IOException + { + return (sftpClient.read(getFullPathForFile(dirEntry))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void writeFile(QBackendMetaData backend, String path, byte[] contents) throws IOException + { + sftpClient.put(new ByteArrayInputStream(contents), path); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getFullPathForFile(SFTPDirEntryWithPath dirEntry) + { + return (dirEntry.path() + "/" + dirEntry.dirEntry().getFilename()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void moveFile(QInstance instance, QTableMetaData table, String source, String destination) throws FilesystemException + { + throw (new QRuntimeException("Not yet implemented")); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected SftpClient getSftpClient(QBackendMetaData backend) throws QException + { + if(sftpClient == null) + { + preAction(backend); + } + + return (sftpClient); + } +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java new file mode 100644 index 00000000..f882662c --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountAction extends AbstractSFTPAction implements CountInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public CountOutput execute(CountInput countInput) throws QException + { + return (executeCount(countInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java new file mode 100644 index 00000000..365e3ec7 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteAction.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteAction implements DeleteInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public DeleteOutput execute(DeleteInput deleteInput) throws QException + { + throw new NotImplementedException("SFTP delete not implemented"); + /* + try + { + DeleteResult rs = new DeleteResult(); + QTableMetaData table = deleteRequest.getTable(); + + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java new file mode 100644 index 00000000..aa1c81dc --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertAction extends AbstractSFTPAction implements InsertInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public InsertOutput execute(InsertInput insertInput) throws QException + { + return (super.executeInsert(insertInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java new file mode 100644 index 00000000..5759c7ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryAction.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPQueryAction extends AbstractSFTPAction implements QueryInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public QueryOutput execute(QueryInput queryInput) throws QException + { + return (super.executeQuery(queryInput)); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java new file mode 100644 index 00000000..ddcb40f1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageAction.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.utils.SFTPOutputStream; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** (mass, streamed) storage action for sftp module + *******************************************************************************/ +public class SFTPStorageAction extends AbstractSFTPAction implements QStorageInterface +{ + + /******************************************************************************* + ** create an output stream in the storage backend - that can be written to, + ** for the purpose of inserting or writing a file into storage. + *******************************************************************************/ + @Override + public OutputStream createOutputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + + SFTPOutputStream sftpOutputStream = new SFTPOutputStream(sftpClient, getFullPath(storageInput)); + return (sftpOutputStream); + } + catch(Exception e) + { + throw (new QException("Exception creating sftp output stream for file", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullPath(StorageInput storageInput) throws QException + { + QTableMetaData table = storageInput.getTable(); + QBackendMetaData backend = storageInput.getBackend(); + String fullPath = stripDuplicatedSlashes(getFullBasePath(table, backend) + File.separator + storageInput.getReference()); + + ///////////////////////////////////////////////////////////// + // s3 seems to do better w/o leading slashes, so, strip... // + ///////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = fullPath.substring(1); + } + + return fullPath; + } + + + + /******************************************************************************* + ** create an input stream in the storage backend - that can be read from, + ** for the purpose of getting or reading a file from storage. + *******************************************************************************/ + @Override + public InputStream getInputStream(StorageInput storageInput) throws QException + { + try + { + SFTPBackendMetaData backend = (SFTPBackendMetaData) storageInput.getBackend(); + preAction(backend); + + SftpClient sftpClient = getSftpClient(backend); + InputStream inputStream = sftpClient.read(getFullPath(storageInput)); + + return (inputStream); + } + catch(Exception e) + { + throw (new QException("Exception getting sftp input stream for file.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getDownloadURL(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + //S3BackendMetaData backend = (S3BackendMetaData) storageInput.getBackend(); + //preAction(backend); + // + //AmazonS3 amazonS3 = getS3Utils().getAmazonS3(); + //String fullPath = getFullPath(storageInput); + //return (amazonS3.getUrl(backend.getBucketName(), fullPath).toString()); + } + catch(Exception e) + { + throw (new QException("Exception getting the sftp download URL.", e)); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void makePublic(StorageInput storageInput) throws QException + { + try + { + throw new QRuntimeException("Not implemented"); + } + catch(Exception e) + { + throw (new QException("Exception making sftp file publicly available.", e)); + } + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java new file mode 100644 index 00000000..93b31a69 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateAction.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.actions.interfaces.UpdateInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import org.apache.commons.lang.NotImplementedException; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateAction implements UpdateInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + public UpdateOutput execute(UpdateInput updateInput) throws QException + { + throw new NotImplementedException("SFTP update not implemented"); + /* + try + { + UpdateResult rs = new UpdateResult(); + QTableMetaData table = updateRequest.getTable(); + + List records = new ArrayList<>(); + rs.setRecords(records); + + // return rs; + } + catch(Exception e) + { + throw new QException("Error executing update: " + e.getMessage(), e); + } + */ + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java new file mode 100644 index 00000000..06cae267 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/SFTPDirEntryWithPath.java @@ -0,0 +1,33 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model; + + +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** + *******************************************************************************/ +public record SFTPDirEntryWithPath(String path, SftpClient.DirEntry dirEntry) +{ +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java new file mode 100644 index 00000000..c423f99e --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -0,0 +1,198 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP backend meta data. + *******************************************************************************/ +public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData +{ + private String username; + private String password; + private String hostName; + private Integer port; + + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPBackendMetaData() + { + super(); + setBackendType(SFTPBackendModule.class); + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPBackendMetaData withBasePath(String basePath) + { + setBasePath(basePath); + return (this); + } + + + + /******************************************************************************* + ** Fluent setter for name + ** + *******************************************************************************/ + public SFTPBackendMetaData withName(String name) + { + setName(name); + return this; + } + + + + /******************************************************************************* + ** Getter for username + *******************************************************************************/ + public String getUsername() + { + return (this.username); + } + + + + /******************************************************************************* + ** Setter for username + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + *******************************************************************************/ + public SFTPBackendMetaData withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + *******************************************************************************/ + public String getPassword() + { + return (this.password); + } + + + + /******************************************************************************* + ** Setter for password + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + *******************************************************************************/ + public SFTPBackendMetaData withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + *******************************************************************************/ + public String getHostName() + { + return (this.hostName); + } + + + + /******************************************************************************* + ** Setter for hostName + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + *******************************************************************************/ + public SFTPBackendMetaData withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + *******************************************************************************/ + public Integer getPort() + { + return (this.port); + } + + + + /******************************************************************************* + ** Setter for port + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + *******************************************************************************/ + public SFTPBackendMetaData withPort(Integer port) + { + this.port = port; + return (this); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java new file mode 100644 index 00000000..b5205cb9 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -0,0 +1,38 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; + + +/******************************************************************************* + ** + *******************************************************************************/ +public enum SFTPBackendVariantSetting implements BackendVariantSetting +{ + USERNAME, + PASSWORD, + HOSTNAME, + PORT, + BASE_PATH +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java new file mode 100644 index 00000000..3a1605e0 --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPTableBackendDetails.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata; + + +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.SFTPBackendModule; + + +/******************************************************************************* + ** SFTP specific Extension of QTableBackendDetails + *******************************************************************************/ +public class SFTPTableBackendDetails extends AbstractFilesystemTableBackendDetails +{ + + + /******************************************************************************* + ** Default Constructor. + *******************************************************************************/ + public SFTPTableBackendDetails() + { + super(); + setBackendType(SFTPBackendModule.class); + } + +} diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java new file mode 100644 index 00000000..19887c2a --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/utils/SFTPOutputStream.java @@ -0,0 +1,179 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.utils; + + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.apache.sshd.sftp.client.SftpClient; +import org.jetbrains.annotations.NotNull; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPOutputStream extends PipedOutputStream +{ + private static final QLogger LOG = QLogger.getLogger(SFTPOutputStream.class); + + private final SftpClient sftpClient; + + private final PipedInputStream pipedInputStream; + private final Future putFuture; + + private AtomicBoolean started = new AtomicBoolean(false); + private AtomicReference putException = new AtomicReference<>(null); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPOutputStream(SftpClient sftpClient, String path) throws IOException + { + pipedInputStream = new PipedInputStream(this, 32 * 1024); + + this.sftpClient = sftpClient; + + putFuture = Executors.newSingleThreadExecutor().submit(() -> + { + try + { + started.set(true); + sftpClient.put(pipedInputStream, path); + } + catch(Exception e) + { + putException.set(e); + LOG.error("Error putting SFTP output stream", e); + + try + { + pipedInputStream.close(); + } + catch(IOException ex) + { + LOG.error("Secondary error closing pipedInputStream after sftp put error", e); + } + + throw new RuntimeException(e); + } + }); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void write(@NotNull byte[] b) throws IOException + { + try + { + super.write(b); + } + catch(IOException e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error writing to SFTP output stream", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws IOException + { + try + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // don't try to close anything until we know that the sftpClient.put call's thread // + // has tried to start (otherwise, race condition could cause us to close things too early) // + ///////////////////////////////////////////////////////////////////////////////////////////// + int sleepLoops = 0; + while(!started.get() && sleepLoops++ <= 30) + { + SleepUtils.sleep(1, TimeUnit.SECONDS); + } + + /////////////////////////////////////////////////////////////////////////////////////// + // closing the pipedOutputStream (super) causes things to flush and complete the put // + /////////////////////////////////////////////////////////////////////////////////////// + super.close(); + + //////////////////////////////// + // wait for the put to finish // + //////////////////////////////// + putFuture.get(60 - sleepLoops, TimeUnit.SECONDS); + + /////////////////////////////////////////////////////////////////////////////// + // in case the put-future never did start, throw explicitly mentioning that. // + /////////////////////////////////////////////////////////////////////////////// + if(sleepLoops >= 30) + { + throw (new Exception("future to can sftpClient.put() was never started.")); + } + } + catch(ExecutionException ee) + { + throw new IOException("Error performing SFTP put", ee); + } + catch(Exception e) + { + if(putException.get() != null) + { + throw new IOException("Error performing SFTP put", putException.get()); + } + + throw new IOException("Error closing SFTP output stream", e); + } + finally + { + try + { + sftpClient.close(); + } + catch(IOException e) + { + LOG.error("Error closing SFTP client", e); + } + } + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index acad9100..233cad0d 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem; import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -37,12 +38,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsConfig; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockBackendModule; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamed.StreamedETLProcess; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality; +import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.FilesystemTableMetaDataBuilder; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; @@ -52,6 +55,10 @@ import com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fil import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; +import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails; import org.apache.commons.io.FileUtils; @@ -60,20 +67,26 @@ import org.apache.commons.io.FileUtils; *******************************************************************************/ public class TestUtils { - public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; - public static final String BACKEND_NAME_S3 = "s3"; - public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; - public static final String BACKEND_NAME_MOCK = "mock"; - public static final String BACKEND_NAME_MEMORY = "memory"; + public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem"; + public static final String BACKEND_NAME_S3 = "s3"; + public static final String BACKEND_NAME_S3_SANS_PREFIX = "s3sansPrefix"; + public static final String BACKEND_NAME_SFTP = "sftp"; + public static final String BACKEND_NAME_SFTP_WITH_VARIANTS = "sftpWithVariants"; + public static final String BACKEND_NAME_MOCK = "mock"; + public static final String BACKEND_NAME_MEMORY = "memory"; public static final String TABLE_NAME_PERSON_LOCAL_FS_JSON = "person-local-json"; public static final String TABLE_NAME_PERSON_LOCAL_FS_CSV = "person-local-csv"; public static final String TABLE_NAME_BLOB_LOCAL_FS = "local-blob"; public static final String TABLE_NAME_ARCHIVE_LOCAL_FS = "local-archive"; public static final String TABLE_NAME_PERSON_S3 = "person-s3"; + public static final String TABLE_NAME_PERSON_SFTP = "person-sftp"; public static final String TABLE_NAME_BLOB_S3 = "s3-blob"; public static final String TABLE_NAME_PERSON_MOCK = "person-mock"; public static final String TABLE_NAME_BLOB_S3_SANS_PREFIX = "s3-blob-sans-prefix"; + public static final String TABLE_NAME_SFTP_FILE = "sftp-file"; + public static final String TABLE_NAME_SFTP_FILE_VARIANTS = "sftp-file-with-variants"; + public static final String TABLE_NAME_VARIANT_OPTIONS = "variant-options-table"; public static final String PROCESS_NAME_STREAMED_ETL = "etl.streamed"; public static final String LOCAL_PERSON_CSV_FILE_IMPORTER_PROCESS_NAME = "localPersonCsvFileImporter"; @@ -148,6 +161,7 @@ public class TestUtils qInstance.addBackend(defineS3Backend()); qInstance.addBackend(defineS3BackendSansPrefix()); qInstance.addTable(defineS3CSVPersonTable()); + qInstance.addTable(defineSFTPCSVPersonTable()); qInstance.addTable(defineS3BlobTable()); qInstance.addTable(defineS3BlobSansPrefixTable()); qInstance.addBackend(defineMockBackend()); @@ -155,6 +169,15 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); + QBackendMetaData sftpBackend = defineSFTPBackend(); + qInstance.addBackend(sftpBackend); + qInstance.addTable(defineSFTPFileTable(sftpBackend)); + + QBackendMetaData sftpBackendWithVariants = defineSFTPBackendWithVariants(); + qInstance.addBackend(sftpBackendWithVariants); + qInstance.addTable(defineSFTPFileTableWithVariants(sftpBackendWithVariants)); + qInstance.addTable(defineVariantOptionsTable()); + definePersonCsvImporter(qInstance); return (qInstance); @@ -162,6 +185,21 @@ public class TestUtils + /*************************************************************************** + ** + ***************************************************************************/ + private static QTableMetaData defineVariantOptionsTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_VARIANT_OPTIONS) + .withBackendName(defineMemoryBackend().getName()) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("basePath", QFieldType.STRING)); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -379,6 +417,25 @@ public class TestUtils + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineSFTPCSVPersonTable() + { + return new QTableMetaData() + .withName(TABLE_NAME_PERSON_SFTP) + .withLabel("Person SFTP Table") + .withBackendName(BACKEND_NAME_SFTP) + .withPrimaryKeyField("id") + .withFields(defineCommonPersonTableFields()) + .withBackendDetails(new SFTPTableBackendDetails() + .withRecordFormat(RecordFormat.CSV) + .withCardinality(Cardinality.MANY) + ); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -463,4 +520,77 @@ public class TestUtils MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule(); return (mockAuthenticationModule.createSession(defineInstance(), null)); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTable(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackend() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER) + .withName(BACKEND_NAME_SFTP)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static QTableMetaData defineSFTPFileTableWithVariants(QBackendMetaData sftpBackend) + { + return new FilesystemTableMetaDataBuilder() + .withBasePath(BaseSFTPTest.TABLE_FOLDER) + .withBackend(sftpBackend) + .withName(TABLE_NAME_SFTP_FILE_VARIANTS) + .buildStandardCardinalityOneTable() + .withLabel("SFTP Files"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData defineSFTPBackendWithVariants() + { + return (new SFTPBackendMetaData() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withHostName(BaseSFTPTest.HOST_NAME) + .withPort(BaseSFTPTest.getCurrentPort()) + + //////////////////////////////////// + // only get basePath from variant // + //////////////////////////////////// + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TABLE_NAME_VARIANT_OPTIONS) + .withVariantTypeKey(TABLE_NAME_VARIANT_OPTIONS) + .withBackendSettingSourceFieldNameMap(Map.of( + SFTPBackendVariantSetting.BASE_PATH, "basePath" + )) + ) + .withName(BACKEND_NAME_SFTP_WITH_VARIANTS)); + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java new file mode 100644 index 00000000..69aa1e9e --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -0,0 +1,139 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp; + + +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.MountableFile; + + +/******************************************************************************* + ** Base class for tests that want to be able to work with sftp testcontainer + *******************************************************************************/ +public class BaseSFTPTest extends BaseTest +{ + public static final int PORT = 22; + public static final String USERNAME = "testuser"; + public static final String PASSWORD = "testpass"; + public static final String HOST_NAME = "localhost"; + + public static final String BACKEND_FOLDER = "upload"; + public static final String TABLE_FOLDER = "files"; + public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; + + private static GenericContainer sftpContainer; + private static Integer currentPort; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @BeforeAll + static void setUp() throws Exception + { + sftpContainer = new GenericContainer<>("atmoz/sftp:latest") + .withExposedPorts(PORT) + .withCommand(USERNAME + ":" + PASSWORD + ":1001"); + + sftpContainer.start(); + + for(int i = 0; i < 5; i++) + { + sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt"); + } + + grantUploadFilesDirWritePermission(); + + currentPort = sftpContainer.getMappedPort(22); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @AfterAll + static void tearDown() + { + if(sftpContainer != null) + { + sftpContainer.stop(); + } + } + + + + /******************************************************************************* + ** Getter for currentPort + ** + *******************************************************************************/ + public static Integer getCurrentPort() + { + return currentPort; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void revokeUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("444"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static void grantUploadFilesDirWritePermission() throws Exception + { + setUploadFilesDirPermission("777"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void setUploadFilesDirPermission(String mode) throws Exception + { + sftpContainer.execInContainer("chmod", mode, "/home/testuser/upload/files"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception + { + Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path); + System.out.println(mkdir.getExitCode()); + } +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java new file mode 100644 index 00000000..643fe2ba --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPCountActionTest.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPCountActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCount1() throws QException + { + CountInput countInput = initCountRequest(); + SFTPCountAction countAction = new SFTPCountAction(); + CountOutput countOutput = countAction.execute(countInput); + Assertions.assertEquals(5, countOutput.getCount(), "Expected # of rows from unfiltered count"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private CountInput initCountRequest() throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + return countInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java new file mode 100644 index 00000000..3a992fbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPDeleteActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java new file mode 100644 index 00000000..c648021c --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPInsertActionTest.java @@ -0,0 +1,120 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.IOException; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPInsertActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOne() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + assertThat(insertOutput.getRecords()) + .allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(BaseSFTPTest.BACKEND_FOLDER)); + + QRecord record = insertOutput.getRecords().get(0); + String fullPath = record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH); + assertThat(record.getErrors()).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityOnePermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_SFTP_FILE); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + InsertOutput insertOutput = insertAction.execute(insertInput); + + QRecord record = insertOutput.getRecords().get(0); + assertThat(record.getErrors()).isNotEmpty(); + assertThat(record.getErrors().get(0).getMessage()).contains("Error writing file: Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testCardinalityMany() throws QException, IOException + { + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_SFTP); + insertInput.setRecords(List.of( + new QRecord().withValue("id", "1").withValue("firstName", "Bob") + )); + + SFTPInsertAction insertAction = new SFTPInsertAction(); + + assertThatThrownBy(() -> insertAction.execute(insertInput)) + .hasRootCauseInstanceOf(NotImplementedException.class); + } +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java new file mode 100644 index 00000000..fdce8969 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Unit test for SFTPQueryAction + *******************************************************************************/ +class SFTPQueryActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQuery1() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testQueryVariantsTable() throws Exception + { + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_VARIANT_OPTIONS).withRecords(List.of( + new QRecord().withValue("id", 1).withValue("basePath", BaseSFTPTest.BACKEND_FOLDER), + new QRecord().withValue("id", 2).withValue("basePath", "empty-folder"), + new QRecord().withValue("id", 3).withValue("basePath", "non-existing-path") + ))); + + mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); + + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .hasMessageContaining("Could not find Backend Variant information for Backend"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 2)); + queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(0, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + + QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 3)); + assertThatThrownBy(() -> new QueryAction().execute(queryInput)) + .rootCause() + .hasMessageContaining("No such file"); + + // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QueryInput initQueryRequest() throws QException + { + QueryInput queryInput = new QueryInput(); + return queryInput; + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java new file mode 100644 index 00000000..ed49cc2d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPStorageActionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FilesystemStorageAction + *******************************************************************************/ +public class SFTPStorageActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSmall() throws Exception + { + String data = "Hellooo, Storage."; + runTest(data); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testPermissionError() throws Exception + { + try + { + revokeUploadFilesDirWritePermission(); + String data = "oops!"; + assertThatThrownBy(() -> runTest(data)) + .hasRootCauseInstanceOf(IOException.class) + .rootCause() + .hasMessageContaining("Permission denied"); + } + finally + { + grantUploadFilesDirWritePermission(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testLarge() throws Exception + { + String data = StringUtils.join("!", Collections.nCopies(5_000_000, "Hello")); + runTest(data); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void runTest(String data) throws QException, IOException + { + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_SFTP_FILE).withReference("fromStorageAction.txt"); + + StorageAction storageAction = new StorageAction(); + OutputStream outputStream = storageAction.createOutputStream(storageInput); + outputStream.write(data.getBytes(StandardCharsets.UTF_8)); + outputStream.close(); + + InputStream inputStream = storageAction.getInputStream(storageInput); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + inputStream.transferTo(byteArrayOutputStream); + + assertEquals(data.length(), byteArrayOutputStream.toString(StandardCharsets.UTF_8).length()); + assertEquals(data, byteArrayOutputStream.toString(StandardCharsets.UTF_8)); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java new file mode 100644 index 00000000..2f025320 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPUpdateActionTest.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SFTPUpdateActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void test() throws QException + { + assertThrows(NotImplementedException.class, () -> new SFTPUpdateAction().execute(new UpdateInput())); + } + +} \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt new file mode 100644 index 00000000..eab448b1 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/files/testfile.txt @@ -0,0 +1,3 @@ +This is a file. + +It is a test. From c341708d21327b28878f017f927820e7eed0ed69 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:30:13 -0600 Subject: [PATCH 142/209] Start (mostly done?) support for headless bulk-load --- .../BulkInsertReceiveFileMappingStep.java | 72 ++++++++++++++----- .../bulk/insert/BulkInsertStepUtils.java | 39 +++++++++- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index afe3c9f1..94032fe5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -32,16 +32,20 @@ import java.util.Set; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; @@ -63,12 +67,37 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { try { - BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + QRecord savedBulkLoadProfileRecord = BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); - /////////////////////////////////////////////////////////////////// - // read process values - construct a bulkLoadProfile out of them // - /////////////////////////////////////////////////////////////////// - BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + BulkLoadProfile bulkLoadProfile; + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) + { + ////////////////////////////////////////////////////////////////////////////// + // if running headless, build bulkLoadProfile from the saved profile record // + ////////////////////////////////////////////////////////////////////////////// + if(savedBulkLoadProfileRecord == null) + { + throw (new QUserFacingException("Did not receive a saved bulk load profile record as input - unable to perform headless bulk load")); + } + + SavedBulkLoadProfile savedBulkLoadProfile = new SavedBulkLoadProfile(savedBulkLoadProfileRecord); + + try + { + bulkLoadProfile = JsonUtils.toObject(savedBulkLoadProfile.getMappingJson(), BulkLoadProfile.class); + } + catch(Exception e) + { + throw (new QUserFacingException("Error processing saved bulk load profile record - unable to perform headless bulk load", e)); + } + } + else + { + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + } ///////////////////////////////////////////////////////////////////////// // put the list of bulk load profile into the process state - it's the // @@ -183,21 +212,32 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep ///////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); - if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + if(BulkInsertStepUtils.isHeadless(runBackendStepInput)) { - ////////////////////////////////////////////////////////////////////////////////// - // just go to the prepareValueMapping backend step - it'll figure out the rest. // - // it's also where the value-mapping loop of steps points. // - // and, this will actually be the default (e.g., the step after this one). // - ////////////////////////////////////////////////////////////////////////////////// - runBackendStepInput.addValue("valueMappingFieldIndex", -1); + //////////////////////////////////////////////////////////////////////// + // if running headless, always go straight to the preview screen next // + // todo actually, we could make this execute, right? // + //////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); } else { - ////////////////////////////////////////////////////////////////////////////////// - // else - if no values to map - continue with the standard streamed-ETL preview // - ////////////////////////////////////////////////////////////////////////////////// - BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } } } catch(Exception e) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index b0db3ad0..c52cd299 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; @@ -69,6 +70,18 @@ public class BulkInsertStepUtils + /*************************************************************************** + ** + ***************************************************************************/ + public static void setStorageInputForTheFile(RunProcessInput runProcessInput, StorageInput storageInput) + { + ArrayList storageInputs = new ArrayList<>(); + storageInputs.add(storageInput); + runProcessInput.addValue("theFile", storageInputs); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -144,13 +157,37 @@ public class BulkInsertStepUtils /*************************************************************************** ** ***************************************************************************/ - public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + public static QRecord handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); if(savedBulkLoadProfileId != null) { QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + return (savedBulkLoadProfileRecord); } + + return (null); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean isHeadless(RunBackendStepInput runBackendStepInput) + { + return (runBackendStepInput.getValuePrimitiveBoolean("isHeadless")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setHeadless(RunProcessInput runProcessInput) + { + runProcessInput.addValue("isHeadless", true); + } + } From 3f8c2957d1625723bbcfccccc833b0472157bde8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 14 Feb 2025 20:45:51 -0600 Subject: [PATCH 143/209] Swap setVariantOptionsTableTypeField for setVariantOptionsTableTypeValue re: which one sets the new config's setVariantTypeKey --- .../qqq/backend/core/model/metadata/QBackendMetaData.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java index 3d79e591..9bc7ad04 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QBackendMetaData.java @@ -452,8 +452,6 @@ public class QBackendMetaData implements TopLevelMetaDataInterface @Deprecated(since = "Replaced by fieldName in filter in backendVariantsConfig - but leaving as field to pair with ...TypeValue for building filter") public void setVariantOptionsTableTypeField(String variantOptionsTableTypeField) { - this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeField); - this.variantOptionsTableTypeField = variantOptionsTableTypeField; if(this.variantOptionsTableTypeValue != null) { @@ -481,6 +479,8 @@ public class QBackendMetaData implements TopLevelMetaDataInterface @Deprecated(since = "Replaced by variantTypeKey and value in filter in backendVariantsConfig - but leaving as field to pair with ...TypeField for building filter") public void setVariantOptionsTableTypeValue(String variantOptionsTableTypeValue) { + this.getOrWithNewBackendVariantsConfig().setVariantTypeKey(variantOptionsTableTypeValue); + this.variantOptionsTableTypeValue = variantOptionsTableTypeValue; if(this.variantOptionsTableTypeField != null) { From 5d2adb76e0610851283d36b7eea516dce676984c Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Wed, 19 Feb 2025 17:10:46 -0600 Subject: [PATCH 144/209] CE-2261: added grid widths to field metadata --- .../qqq/backend/core/model/data/QField.java | 5 +++ .../model/metadata/fields/QFieldMetaData.java | 33 +++++++++++++++++++ .../frontend/QFrontendFieldMetaData.java | 14 +++++++- 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java index 13ead81c..f8fbfbe8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java @@ -89,6 +89,11 @@ public @interface QField *******************************************************************************/ int maxLength() default Integer.MAX_VALUE; + /******************************************************************************* + ** + *******************************************************************************/ + int gridColumns() default -1; + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java index 5e5e61f0..a24bb992 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java @@ -81,6 +81,7 @@ public class QFieldMetaData implements Cloneable private QQueryFilter possibleValueSourceFilter; private QPossibleValueSource inlinePossibleValueSource; + private Integer gridColumns; private Integer maxLength; private Set> behaviors; @@ -198,6 +199,7 @@ public class QFieldMetaData implements Cloneable setIsRequired(fieldAnnotation.isRequired()); setIsEditable(fieldAnnotation.isEditable()); setIsHidden(fieldAnnotation.isHidden()); + setGridColumns(fieldAnnotation.gridColumns()); if(StringUtils.hasContent(fieldAnnotation.label())) { @@ -1062,6 +1064,7 @@ public class QFieldMetaData implements Cloneable } + /******************************************************************************* ** Getter for inlinePossibleValueSource *******************************************************************************/ @@ -1092,4 +1095,34 @@ public class QFieldMetaData implements Cloneable } + + /******************************************************************************* + ** Getter for gridColumns + *******************************************************************************/ + public Integer getGridColumns() + { + return (this.gridColumns); + } + + + + /******************************************************************************* + ** Setter for gridColumns + *******************************************************************************/ + public void setGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + } + + + + /******************************************************************************* + ** Fluent setter for gridColumns + *******************************************************************************/ + public QFieldMetaData withGridColumns(Integer gridColumns) + { + this.gridColumns = gridColumns; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java index 653c1dcc..c6c6014f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java @@ -51,6 +51,7 @@ public class QFrontendFieldMetaData implements Serializable private boolean isRequired; private boolean isEditable; private boolean isHeavy; + private Integer gridColumns; private String possibleValueSourceName; private String displayFormat; private Serializable defaultValue; @@ -66,7 +67,6 @@ public class QFrontendFieldMetaData implements Serializable ////////////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** Constructor *******************************************************************************/ @@ -78,6 +78,7 @@ public class QFrontendFieldMetaData implements Serializable this.isRequired = fieldMetaData.getIsRequired(); this.isEditable = fieldMetaData.getIsEditable(); this.isHeavy = fieldMetaData.getIsHeavy(); + this.gridColumns = fieldMetaData.getGridColumns(); this.possibleValueSourceName = fieldMetaData.getPossibleValueSourceName(); this.displayFormat = fieldMetaData.getDisplayFormat(); this.adornments = fieldMetaData.getAdornments(); @@ -166,6 +167,17 @@ public class QFrontendFieldMetaData implements Serializable + /******************************************************************************* + ** Getter for gridColumns + ** + *******************************************************************************/ + public Integer getGridColumns() + { + return gridColumns; + } + + + /******************************************************************************* ** Getter for displayFormat ** From be6d1b888ff87734f83b95580ab572bb4f6f69fb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:07:56 -0600 Subject: [PATCH 145/209] Add urlencoding to blob download urls --- .../backend/core/actions/values/QValueFormatter.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 39a6a520..6129b409 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -23,6 +23,8 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -32,6 +34,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -532,7 +535,7 @@ public class QValueFormatter //////////////////////////////////////////////////////////////////////////////////////////////// // if field type is blob OR if there's a supplemental process or code-ref that needs to run - // - // then update its value to be a callback-url that'll give access to the bytes to download // + // then update its value to be a callback-url that'll give access to the bytes to download. // // implied here is that a String value (w/o supplemental code/proc) has its value stay as a // // URL, which is where the file is directly downloaded from. And in the case of a String // // with code-to-run, then the code should run, followed by a redirect to the value URL. // @@ -541,7 +544,10 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName); + record.setValue(field.getName(), "/data/" + table.getName() + "/" + + URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/" + + field.getName() + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); } record.setDisplayValue(field.getName(), fileName); } From 8816177df8dd793f67cdaec858af86303988ff37 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:49:33 -0600 Subject: [PATCH 146/209] Add optional variantRecordLookupFunction to BackendVariantsConfig and validation of same; refactor up some shared backend code into BackendVariantsUtil --- .../core/instances/QInstanceValidator.java | 52 ++++++--- .../variants/BackendVariantsConfig.java | 40 ++++++- .../variants/BackendVariantsUtil.java | 106 ++++++++++++++++++ .../instances/QInstanceValidatorTest.java | 76 ++++++++++++- .../variants/BackendVariantsUtilTest.java | 100 +++++++++++++++++ 5 files changed, 358 insertions(+), 16 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index fa4d1a23..a214ee6b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.TimeZone; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler; import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; @@ -115,6 +117,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeLambda; import org.apache.commons.lang.BooleanUtils; import org.quartz.CronExpression; @@ -556,8 +559,8 @@ public class QInstanceValidator { assertCondition(StringUtils.hasContent(backendVariantsConfig.getVariantTypeKey()), "Missing variantTypeKey in backendVariantsConfig in [" + backendName + "]"); - String optionsTableName = backendVariantsConfig.getOptionsTableName(); - QTableMetaData optionsTable = qInstance.getTable(optionsTableName); + String optionsTableName = backendVariantsConfig.getOptionsTableName(); + QTableMetaData optionsTable = qInstance.getTable(optionsTableName); if(assertCondition(StringUtils.hasContent(optionsTableName), "Missing optionsTableName in backendVariantsConfig in [" + backendName + "]")) { if(assertCondition(optionsTable != null, "Unrecognized optionsTableName [" + optionsTableName + "] in backendVariantsConfig in [" + backendName + "]")) @@ -573,7 +576,11 @@ public class QInstanceValidator Map backendSettingSourceFieldNameMap = backendVariantsConfig.getBackendSettingSourceFieldNameMap(); if(assertCondition(CollectionUtils.nullSafeHasContents(backendSettingSourceFieldNameMap), "Missing or empty backendSettingSourceFieldNameMap in backendVariantsConfig in [" + backendName + "]")) { - if(optionsTable != null) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // only validate field names in the backendSettingSourceFieldNameMap if there is NOT a variantRecordSupplier // + // (the idea being, that the supplier might be building a record with fieldNames that aren't in the table... // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(optionsTable != null && backendVariantsConfig.getVariantRecordLookupFunction() == null) { for(Map.Entry entry : backendSettingSourceFieldNameMap.entrySet()) { @@ -581,6 +588,11 @@ public class QInstanceValidator } } } + + if(backendVariantsConfig.getVariantRecordLookupFunction() != null) + { + validateSimpleCodeReference("VariantRecordSupplier in backendVariantsConfig in backend [" + backendName + "]: ", backendVariantsConfig.getVariantRecordLookupFunction(), UnsafeFunction.class, Function.class); + } } } else @@ -1404,7 +1416,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(customizerInstance != null && tableCustomizer.getExpectedType() != null) { - assertObjectCanBeCasted(prefix, tableCustomizer.getExpectedType(), customizerInstance); + assertObjectCanBeCasted(prefix, customizerInstance, tableCustomizer.getExpectedType()); } } } @@ -1416,18 +1428,31 @@ public class QInstanceValidator /******************************************************************************* ** Make sure that a given object can be casted to an expected type. *******************************************************************************/ - private T assertObjectCanBeCasted(String errorPrefix, Class expectedType, Object object) + private void assertObjectCanBeCasted(String errorPrefix, Object object, Class... anyOfExpectedClasses) { - T castedObject = null; - try + for(Class expectedClass : anyOfExpectedClasses) { - castedObject = expectedType.cast(object); + try + { + expectedClass.cast(object); + return; + } + catch(ClassCastException e) + { + ///////////////////////////////////// + // try next type (if there is one) // + ///////////////////////////////////// + } } - catch(ClassCastException e) + + if(anyOfExpectedClasses.length == 1) { - errors.add(errorPrefix + "CodeReference is not of the expected type: " + expectedType); + errors.add(errorPrefix + "CodeReference is not of the expected type: " + anyOfExpectedClasses[0]); + } + else + { + errors.add(errorPrefix + "CodeReference is not any of the expected types: " + Arrays.stream(anyOfExpectedClasses).map(c -> c.getName()).collect(Collectors.joining(", "))); } - return castedObject; } @@ -2171,7 +2196,8 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class expectedClass) + @SafeVarargs + private void validateSimpleCodeReference(String prefix, QCodeReference codeReference, Class... anyOfExpectedClasses) { if(!preAssertionsForCodeReference(codeReference, prefix)) { @@ -2199,7 +2225,7 @@ public class QInstanceValidator //////////////////////////////////////////////////////////////////////// if(classInstance != null) { - assertObjectCanBeCasted(prefix, expectedClass, classInstance); + assertObjectCanBeCasted(prefix, classInstance, anyOfExpectedClasses); } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java index 20237e7a..290c099f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsConfig.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.model.metadata.variants; import java.util.HashMap; import java.util.Map; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; /******************************************************************************* @@ -37,13 +38,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; ** field names in that table that they come from. e.g., a backend may have a ** username attribute, whose value comes from a field named "theUser" in the ** variant options table. + ** - an optional code reference to a variantRecordLookupFunction - to customize + ** how the variant record is looked up (such as, adding joined or other custom + ** fields). *******************************************************************************/ public class BackendVariantsConfig { private String variantTypeKey; - private String optionsTableName; - private QQueryFilter optionsFilter; + private String optionsTableName; + private QQueryFilter optionsFilter; + private QCodeReference variantRecordLookupFunction; private Map backendSettingSourceFieldNameMap; @@ -186,4 +191,35 @@ public class BackendVariantsConfig return (this); } + + /******************************************************************************* + ** Getter for variantRecordLookupFunction + *******************************************************************************/ + public QCodeReference getVariantRecordLookupFunction() + { + return (this.variantRecordLookupFunction); + } + + + + /******************************************************************************* + ** Setter for variantRecordLookupFunction + *******************************************************************************/ + public void setVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + } + + + + /******************************************************************************* + ** Fluent setter for variantRecordLookupFunction + *******************************************************************************/ + public BackendVariantsConfig withVariantRecordLookupFunction(QCodeReference variantRecordLookupFunction) + { + this.variantRecordLookupFunction = variantRecordLookupFunction; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java new file mode 100644 index 00000000..a94fbc4e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtil.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.io.Serializable; +import java.util.function.Function; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; + + +/******************************************************************************* + ** Utility methods for backends working with Variants. + *******************************************************************************/ +public class BackendVariantsUtil +{ + + /******************************************************************************* + ** Get the variant id from the session for the backend. + *******************************************************************************/ + public static Serializable getVariantId(QBackendMetaData backendMetaData) throws QException + { + QSession session = QContext.getQSession(); + String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); + if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + { + throw (new QException("Could not find Backend Variant information in session under key '" + variantTypeKey + "' for Backend '" + backendMetaData.getName() + "'")); + } + Serializable variantId = session.getBackendVariants().get(variantTypeKey); + return variantId; + } + + + + /******************************************************************************* + ** For backends that use variants, look up the variant record (in theory, based + ** on an id in the session's backend variants map, then fetched from the backend's + ** variant options table. + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException + { + Serializable variantId = getVariantId(backendMetaData); + + QRecord record; + if(backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction() != null) + { + Object o = QCodeLoader.getAdHoc(Object.class, backendMetaData.getBackendVariantsConfig().getVariantRecordLookupFunction()); + if(o instanceof UnsafeFunction unsafeFunction) + { + record = ((UnsafeFunction) unsafeFunction).apply(variantId); + } + else if(o instanceof Function function) + { + record = ((Function) function).apply(variantId); + } + else + { + throw (new QException("Backend Variant's recordLookupFunction is not of any expected type (should have been caught by instance validation??)")); + } + } + else + { + GetInput getInput = new GetInput(); + getInput.setShouldMaskPasswords(false); + getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); + getInput.setPrimaryKey(variantId); + GetOutput getOutput = new GetAction().execute(getInput); + + record = getOutput.getRecord(); + } + + if(record == null) + { + throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); + } + return record; + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 9ce5d9f6..8419b05a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -103,6 +103,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaDeleteStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -246,6 +247,79 @@ public class QInstanceValidatorTest extends BaseTest .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) .withBackendSettingSourceFieldNameMap(Map.of(setting, "noSuchField")))), "Unrecognized fieldName [noSuchField] in backendSettingSourceFieldNameMap"); + + assertValidationFailureReasons((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(CustomizerThatIsNotOfTheRightBaseClass.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + )), + "VariantRecordSupplier in backendVariantsConfig in backend [variant]: CodeReference is not any of the expected types: com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeFunction, java.util.function.Function"); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "firstName")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + + assertValidationSuccess((qInstance) -> qInstance.addBackend(new QBackendMetaData() + .withName("variant") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withVariantTypeKey("myVariant") + .withOptionsTableName(TestUtils.TABLE_NAME_PERSON) + .withVariantRecordLookupFunction(new QCodeReference(VariantRecordUnsafeFunction.class)) + .withBackendSettingSourceFieldNameMap(Map.of(setting, "no-field-but-okay-custom-supplier")) + ))); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordFunction implements Function + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) + { + return null; + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class VariantRecordUnsafeFunction implements UnsafeFunction + { + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QRecord apply(Serializable serializable) throws QException + { + return null; + } } @@ -2437,7 +2511,7 @@ public class QInstanceValidatorTest extends BaseTest { int noOfReasons = actualReasons == null ? 0 : actualReasons.size(); assertEquals(expectedReasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", expectedReasons) - + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", actualReasons) : "--")); } for(String reason : expectedReasons) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java new file mode 100644 index 00000000..5957ee61 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/variants/BackendVariantsUtilTest.java @@ -0,0 +1,100 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.variants; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for BackendVariantsUtil + *******************************************************************************/ +class BackendVariantsUtilTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantId() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantId(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertEquals(1701, BackendVariantsUtil.getVariantId(myBackend)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QBackendMetaData getBackendMetaData() + { + QBackendMetaData myBackend = new QBackendMetaData() + .withName("TestBackend") + .withUsesVariants(true) + .withBackendVariantsConfig(new BackendVariantsConfig() + .withOptionsTableName(TestUtils.TABLE_NAME_SHAPE) + .withVariantTypeKey("yourSelectedShape")); + return myBackend; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetVariantRecord() throws QException + { + QBackendMetaData myBackend = getBackendMetaData(); + + TestUtils.insertDefaultShapes(QContext.getQInstance()); + + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant information in session under key 'yourSelectedShape' for Backend 'TestBackend'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1701)); + assertThatThrownBy(() -> BackendVariantsUtil.getVariantRecord(myBackend)) + .hasMessageContaining("Could not find Backend Variant in table shape with id '1701'"); + + QContext.getQSession().setBackendVariants(Map.of("yourSelectedShape", 1)); + QRecord variantRecord = BackendVariantsUtil.getVariantRecord(myBackend); + assertEquals(1, variantRecord.getValueInteger("id")); + assertNotNull(variantRecord.getValue("name")); + } + +} \ No newline at end of file From 143ed927fa845009cd1e9d3fe6a97c6fadfa1cf2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:50:27 -0600 Subject: [PATCH 147/209] add ability to set and trace processTracerKeyRecord in bulk load --- .../processes/RunBackendStepInput.java | 8 ++- .../bulk/insert/BulkInsertExtractStep.java | 2 + .../bulk/insert/BulkInsertStepUtils.java | 26 ++++++++ .../ProcessTracerKeyRecordMessage.java | 66 +++++++++++++++++++ 4 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 764f930c..84da02f7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -257,7 +257,11 @@ public class RunBackendStepInput extends AbstractActionInput public List getRecordsAsEntities(Class entityClass) throws QException { List rs = new ArrayList<>(); - for(QRecord record : processState.getRecords()) + + /////////////////////////////////////////////////////////////////////////////////// + // note - important to call getRecords here, which is overwritten in subclasses! // + /////////////////////////////////////////////////////////////////////////////////// + for(QRecord record : getRecords()) { rs.add(QRecordEntity.fromQRecord(entityClass, record)); } @@ -601,7 +605,7 @@ public class RunBackendStepInput extends AbstractActionInput ***************************************************************************/ public void traceMessage(ProcessTracerMessage message) { - if(processTracer != null) + if(processTracer != null && message != null) { try { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index 694ae332..dabfb3f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -55,6 +55,8 @@ public class BulkInsertExtractStep extends AbstractExtractStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + runBackendStepInput.traceMessage(BulkInsertStepUtils.getProcessTracerKeyRecordMessage(runBackendStepInput)); + int rowsAdded = 0; int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index c52cd299..e9c09cea 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.apache.commons.lang3.BooleanUtils; import org.json.JSONArray; @@ -190,4 +191,29 @@ public class BulkInsertStepUtils runProcessInput.addValue("isHeadless", true); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setProcessTracerKeyRecordMessage(RunProcessInput runProcessInput, ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + runProcessInput.addValue("processTracerKeyRecordMessage", processTracerKeyRecordMessage); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProcessTracerKeyRecordMessage getProcessTracerKeyRecordMessage(RunBackendStepInput runBackendStepInput) + { + Serializable value = runBackendStepInput.getValue("processTracerKeyRecordMessage"); + if(value instanceof ProcessTracerKeyRecordMessage processTracerKeyRecordMessage) + { + return (processTracerKeyRecordMessage); + } + + return (null); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java new file mode 100644 index 00000000..53938092 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerKeyRecordMessage.java @@ -0,0 +1,66 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.tracing; + + +/******************************************************************************* + ** Specialization of process tracer message, to indicate a 'key record' that was + ** used as an input or trigger to a process. + *******************************************************************************/ +public class ProcessTracerKeyRecordMessage extends ProcessTracerMessage +{ + private final String tableName; + private final Integer recordId; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProcessTracerKeyRecordMessage(String tableName, Integer recordId) + { + super("Process Key Record is " + tableName + " " + recordId); + this.tableName = tableName; + this.recordId = recordId; + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Getter for recordId + *******************************************************************************/ + public Integer getRecordId() + { + return (this.recordId); + } + +} From 0005c51ecdbdfd6e88e3a61baa9f9fca456586af Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:50:47 -0600 Subject: [PATCH 148/209] Add capturing and reporting first & last inserted primary keys --- .../bulk/insert/BulkInsertLoadStep.java | 82 ++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index c8b73ec2..b35bb24b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -22,16 +22,38 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; /******************************************************************************* ** *******************************************************************************/ -public class BulkInsertLoadStep extends LoadViaInsertStep +public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSummaryProviderInterface { + private static final QLogger LOG = QLogger.getLogger(BulkInsertLoadStep.class); + + private Serializable firstInsertedPrimaryKey = null; + private Serializable lastInsertedPrimaryKey = null; + + /******************************************************************************* ** @@ -42,4 +64,62 @@ public class BulkInsertLoadStep extends LoadViaInsertStep return (QInputSource.USER); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + super.runOnePage(runBackendStepInput, runBackendStepOutput); + + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getValueString("tableName")); + + List insertedRecords = runBackendStepOutput.getRecords(); + for(QRecord insertedRecord : insertedRecords) + { + if(CollectionUtils.nullSafeIsEmpty(insertedRecord.getErrors())) + { + if(firstInsertedPrimaryKey == null) + { + firstInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + + lastInsertedPrimaryKey = insertedRecord.getValue(table.getPrimaryKeyField()); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList processSummary = getTransformStep().getProcessSummary(runBackendStepOutput, isForResultScreen); + + try + { + if(firstInsertedPrimaryKey != null) + { + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepOutput.getValueString("tableName")); + QFieldMetaData field = table.getField(table.getPrimaryKeyField()); + if(field.getType().isNumeric()) + { + ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + idsLine.setCount(null); + processSummary.add(idsLine); + } + } + } + catch(Exception e) + { + LOG.warn("Error adding inserted-keys process summary line", e); + } + + return (processSummary); + } } From 2fd3ed256136fbde585c51825ff7c4c5176e5c58 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:51:05 -0600 Subject: [PATCH 149/209] add serializable --- .../backend/core/processes/tracing/ProcessTracerMessage.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java index df2dd558..bb27a0e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/tracing/ProcessTracerMessage.java @@ -22,13 +22,16 @@ package com.kingsrook.qqq.backend.core.processes.tracing; +import java.io.Serializable; + + /******************************************************************************* ** Basic class that can be passed in to ProcessTracerInterface.handleMessage. ** This class just provides for a string message. We anticipate subclasses ** that may have more specific data, that specific tracer implementations may ** be aware of. *******************************************************************************/ -public class ProcessTracerMessage +public class ProcessTracerMessage implements Serializable { private String message; From dc25f6b289aa69dd4b8b86de595c69da2d1a1f35 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:52:49 -0600 Subject: [PATCH 150/209] Explicit exception if table name is not given. --- .../qqq/backend/core/actions/tables/DeleteAction.java | 5 +++++ .../qqq/backend/core/actions/tables/InsertAction.java | 7 +++++++ .../qqq/backend/core/actions/tables/UpdateAction.java | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index 0ded3c93..a5028027 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -82,6 +82,11 @@ public class DeleteAction { ActionHelper.validateSession(deleteInput); + if(deleteInput.getTableName() == null) + { + throw (new QException("Table name was not specified in delete input")); + } + QTableMetaData table = deleteInput.getTable(); String primaryKeyFieldName = table.getPrimaryKeyField(); QFieldMetaData primaryKeyField = table.getField(primaryKeyFieldName); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 64a4ddc3..846ed476 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -67,6 +67,7 @@ import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -110,6 +111,12 @@ public class InsertAction extends AbstractQActionFunction Date: Wed, 19 Feb 2025 19:53:14 -0600 Subject: [PATCH 151/209] Move variant lookups to new BackendVariantsUtil --- .../module/api/actions/BaseAPIActionUtil.java | 50 ++----------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 3623db11..2d34e264 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -62,6 +61,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.metadata.variants.LegacyBackendVariantSetting; import com.kingsrook.qqq.backend.core.model.session.QSession; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; @@ -779,7 +779,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.API_KEY, APIBackendVariantSetting.API_KEY))); } @@ -807,7 +807,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (Pair.of( record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.USERNAME, APIBackendVariantSetting.USERNAME)), record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.PASSWORD, APIBackendVariantSetting.PASSWORD)) @@ -819,46 +819,6 @@ public class BaseAPIActionUtil - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord() throws QException - { - Serializable variantId = getVariantId(); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); - } - return record; - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId() throws QException - { - QSession session = QContext.getQSession(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(backendMetaData.getBackendVariantsConfig().getVariantTypeKey())) - { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); - } - Serializable variantId = session.getBackendVariants().get(backendMetaData.getBackendVariantsConfig().getVariantTypeKey()); - return variantId; - } - - - /******************************************************************************* ** *******************************************************************************/ @@ -871,7 +831,7 @@ public class BaseAPIActionUtil String accessTokenKey = "accessToken"; if(backendMetaData.getUsesVariants()) { - Serializable variantId = getVariantId(); + Serializable variantId = BackendVariantsUtil.getVariantId(backendMetaData); accessTokenKey = accessTokenKey + ":" + variantId; } @@ -961,7 +921,7 @@ public class BaseAPIActionUtil { if(backendMetaData.getUsesVariants()) { - QRecord record = getVariantRecord(); + QRecord record = BackendVariantsUtil.getVariantRecord(backendMetaData); return (Pair.of( record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_ID, APIBackendVariantSetting.CLIENT_ID)), record.getValueString(getVariantSettingSourceFieldName(backendMetaData, LegacyBackendVariantSetting.CLIENT_SECRET, APIBackendVariantSetting.CLIENT_SECRET)) From 154c5442af5e0149682b34936b3b00f8bf895901 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:37 -0600 Subject: [PATCH 152/209] Add postAction(); move variants stuff to new BackendVariantsUtil; add baseName to ONE records; remove path criteria when filtering (assuming the listFiles method did it) --- .../actions/AbstractBaseFilesystemAction.java | 210 +++++++++++------- 1 file changed, 130 insertions(+), 80 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index a896230c..0557e68a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -34,18 +34,16 @@ import java.util.List; import java.util.Map; import java.util.Optional; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; -import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; import com.kingsrook.qqq.backend.core.adapters.JsonToQRecordAdapter; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -56,10 +54,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; -import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; @@ -156,9 +156,10 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ public String stripBackendAndTableBasePathsFromFileName(String filePath, QBackendMetaData backend, QTableMetaData table) { - String tablePath = getFullBasePath(table, backend); - String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); - return (strippedPath); + String tablePath = getFullBasePath(table, backend); + String strippedPath = filePath.replaceFirst(".*" + tablePath, ""); + String withoutLeadingSlash = stripLeadingSlash(strippedPath); // todo - dangerous, do all backends really want this?? + return (withoutLeadingSlash); } @@ -211,6 +212,34 @@ public abstract class AbstractBaseFilesystemAction + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripLeadingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("^/+", "")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String stripTrailingSlash(String path) + { + if(path == null) + { + return (null); + } + return (path.replaceFirst("/+$", "")); + } + + + /******************************************************************************* ** Get the backend metaData, type-checked as the requested type. *******************************************************************************/ @@ -269,9 +298,31 @@ public abstract class AbstractBaseFilesystemAction LOG.warn("Error executing query", e); throw new QException("Error executing query", e); } + finally + { + postAction(); + } } + /*************************************************************************** + ** + ***************************************************************************/ + private void setRecordValueIfFieldNameHasContent(QRecord record, String fieldName, UnsafeSupplier valueSupplier) + { + if(StringUtils.hasContent(fieldName)) + { + try + { + record.setValue(fieldName, valueSupplier.get()); + } + catch(Exception e) + { + LOG.warn("Error setting record value for field", e, logPair("fieldName", fieldName)); + } + } + } + /*************************************************************************** ** @@ -287,24 +338,15 @@ public abstract class AbstractBaseFilesystemAction // for one-record tables, put the entire file's contents into a single record // //////////////////////////////////////////////////////////////////////////////// String filePathWithoutBase = stripBackendAndTableBasePathsFromFileName(getFullPathForFile(file), queryInput.getBackend(), table); - QRecord record = new QRecord().withValue(tableDetails.getFileNameFieldName(), filePathWithoutBase); + QRecord record = new QRecord(); - if(StringUtils.hasContent(tableDetails.getSizeFieldName())) - { - record.setValue(tableDetails.getSizeFieldName(), getFileSize(file)); - } + setRecordValueIfFieldNameHasContent(record, tableDetails.getFileNameFieldName(), () -> filePathWithoutBase); + setRecordValueIfFieldNameHasContent(record, tableDetails.getBaseNameFieldName(), () -> stripAllPaths(filePathWithoutBase)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getSizeFieldName(), () -> getFileSize(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getCreateDateFieldName(), () -> getFileCreateDate(file)); + setRecordValueIfFieldNameHasContent(record, tableDetails.getModifyDateFieldName(), () -> getFileModifyDate(file)); - if(StringUtils.hasContent(tableDetails.getCreateDateFieldName())) - { - record.setValue(tableDetails.getCreateDateFieldName(), getFileCreateDate(file)); - } - - if(StringUtils.hasContent(tableDetails.getModifyDateFieldName())) - { - record.setValue(tableDetails.getModifyDateFieldName(), getFileModifyDate(file)); - } - - if(shouldFileContentsBeRead(queryInput, table, tableDetails)) + if(shouldHeavyFileContentsBeRead(queryInput, table, tableDetails)) { try(InputStream inputStream = readFile(file)) { @@ -318,27 +360,37 @@ public abstract class AbstractBaseFilesystemAction } else { - if(StringUtils.hasContent(tableDetails.getSizeFieldName())) + Long size = record.getValueLong(tableDetails.getSizeFieldName()); + if(size != null) { - Long size = record.getValueLong(tableDetails.getSizeFieldName()); - if(size != null) + if(record.getBackendDetails() == null) { - if(record.getBackendDetails() == null) - { - record.setBackendDetails(new HashMap<>()); - } - - if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) - { - record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); - } - - ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); + record.setBackendDetails(new HashMap<>()); } + + if(record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS) == null) + { + record.addBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS, new HashMap<>()); + } + + ((Map) record.getBackendDetail(QRecord.BACKEND_DETAILS_TYPE_HEAVY_FIELD_LENGTHS)).put(tableDetails.getContentsFieldName(), size); } } - if(BackendQueryFilterUtils.doesRecordMatch(queryInput.getFilter(), null, record)) + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the listFiles method may have used a "path" criteria. // + // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filterForRecords = queryInput.getFilter(); + if(filterForRecords != null) + { + filterForRecords = filterForRecords.clone(); + + CollectionUtils.nonNullList(filterForRecords.getCriteria()) + .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + } + + if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) { records.add(record); } @@ -351,6 +403,31 @@ public abstract class AbstractBaseFilesystemAction + /*************************************************************************** + ** + ***************************************************************************/ + private Serializable stripAllPaths(String filePath) + { + if(filePath == null) + { + return null; + } + + return (filePath.replaceFirst(".*/", "")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected static boolean isPathEqualsCriteria(QFilterCriteria criteria) + { + return "path".equals(criteria.getFieldName()) && QCriteriaOperator.EQUALS.equals(criteria.getOperator()); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -409,7 +486,7 @@ public abstract class AbstractBaseFilesystemAction /*************************************************************************** ** ***************************************************************************/ - private static boolean shouldFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) + private static boolean shouldHeavyFileContentsBeRead(QueryInput queryInput, QTableMetaData table, AbstractFilesystemTableBackendDetails tableDetails) { boolean doReadContents = true; if(table.getField(tableDetails.getContentsFieldName()).getIsHeavy()) @@ -478,11 +555,21 @@ public abstract class AbstractBaseFilesystemAction { if(backendMetaData.getUsesVariants()) { - this.backendVariantRecord = getVariantRecord(backendMetaData); + this.backendVariantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData); } } + /*************************************************************************** + ** Method that subclasses can override to add post-action things (e.g., closing resources) + ***************************************************************************/ + public void postAction() + { + ////////////////// + // noop in base // + ////////////////// + } + /******************************************************************************* ** @@ -558,47 +645,10 @@ public abstract class AbstractBaseFilesystemAction { throw new QException("Error executing insert: " + e.getMessage(), e); } - } - - - - /******************************************************************************* - ** Get the variant id from the session for the backend. - *******************************************************************************/ - protected Serializable getVariantId(QBackendMetaData backendMetaData) throws QException - { - QSession session = QContext.getQSession(); - String variantTypeKey = backendMetaData.getBackendVariantsConfig().getVariantTypeKey(); - if(session.getBackendVariants() == null || !session.getBackendVariants().containsKey(variantTypeKey)) + finally { - throw (new QException("Could not find Backend Variant information for Backend '" + backendMetaData.getName() + "'")); + postAction(); } - Serializable variantId = session.getBackendVariants().get(variantTypeKey); - return variantId; - } - - - - /******************************************************************************* - ** For backends that use variants, look up the variant record (in theory, based - ** on an id in the session's backend variants map, then fetched from the backend's - ** variant options table. - *******************************************************************************/ - protected QRecord getVariantRecord(QBackendMetaData backendMetaData) throws QException - { - Serializable variantId = getVariantId(backendMetaData); - GetInput getInput = new GetInput(); - getInput.setShouldMaskPasswords(false); - getInput.setTableName(backendMetaData.getBackendVariantsConfig().getOptionsTableName()); - getInput.setPrimaryKey(variantId); - GetOutput getOutput = new GetAction().execute(getInput); - - QRecord record = getOutput.getRecord(); - if(record == null) - { - throw (new QException("Could not find Backend Variant in table " + backendMetaData.getBackendVariantsConfig().getOptionsTableName() + " with id '" + variantId + "'")); - } - return record; } } From 91aa8faca2d5e07451857920e00fcf440b27a84e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:47 -0600 Subject: [PATCH 153/209] Add baseNameFieldName --- ...AbstractFilesystemTableBackendDetails.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 6c9c6bc5..307849b6 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -41,6 +41,7 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private String contentsFieldName; private String fileNameFieldName; + private String baseNameFieldName; private String sizeFieldName; private String createDateFieldName; private String modifyDateFieldName; @@ -377,4 +378,35 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } + + /******************************************************************************* + ** Getter for baseNameFieldName + *******************************************************************************/ + public String getBaseNameFieldName() + { + return (this.baseNameFieldName); + } + + + + /******************************************************************************* + ** Setter for baseNameFieldName + *******************************************************************************/ + public void setBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + } + + + + /******************************************************************************* + ** Fluent setter for baseNameFieldName + *******************************************************************************/ + public AbstractFilesystemTableBackendDetails withBaseNameFieldName(String baseNameFieldName) + { + this.baseNameFieldName = baseNameFieldName; + return (this); + } + + } From 31a586f23ee853946530e029797ae2e72cff06dc Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 19:54:58 -0600 Subject: [PATCH 154/209] Move stripLeadingSlash up to base class --- .../filesystem/s3/actions/AbstractS3Action.java | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index bc0d96d5..4c5e7b8c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -213,20 +213,6 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction Date: Wed, 19 Feb 2025 20:01:30 -0600 Subject: [PATCH 155/209] Move makeConnection to its own method (for use by test process); add postAction to try to close the things; add looking for 'path' criteria and adding it to readDir call --- .../sftp/actions/AbstractSFTPAction.java | 84 +++++++++++++++---- .../module/filesystem/sftp/BaseSFTPTest.java | 23 ++++- .../sftp/actions/SFTPQueryActionTest.java | 83 +++++++++++++++--- 3 files changed, 158 insertions(+), 32 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 80db887e..c9e3c8b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -23,21 +23,26 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; @@ -81,9 +86,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction closer = closable -> + { + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } + }; + + closer.accept(sshClient); + closer.accept(clientSession); + closer.accept(sftpClient); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException + { + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + return (this.sftpClient); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -195,8 +239,22 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction rs = new ArrayList<>(); + String fullPath = getFullBasePath(table, backendBase); + + // todo - move somewhere shared + // todo - should all do this? + if(filter != null) + { + for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) + { + if(isPathEqualsCriteria(criteria)) + { + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); + } + } + } + + List rs = new ArrayList<>(); for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) { @@ -211,10 +269,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction patternExpectedCountMap = Map.of( + "%.txt", 3, + "sub%", 2, + "%1%", 1, + "%", 3, + "*", 0 + ); + + for(Map.Entry entry : patternExpectedCountMap.entrySet()) + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) + .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); + } + } + finally + { + rmrfInContainer(subfolderPath); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -71,7 +139,7 @@ class SFTPQueryActionTest extends BaseSFTPTest mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); assertThatThrownBy(() -> new QueryAction().execute(queryInput)) .hasMessageContaining("Could not find Backend Variant information for Backend"); @@ -91,15 +159,4 @@ class SFTPQueryActionTest extends BaseSFTPTest // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); } - - - /******************************************************************************* - ** - *******************************************************************************/ - private QueryInput initQueryRequest() throws QException - { - QueryInput queryInput = new QueryInput(); - return queryInput; - } - } \ No newline at end of file From bb1a43f11f0dfe016614deded7034d0fab4f2d99 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:02:05 -0600 Subject: [PATCH 156/209] Initial checkin --- .../actions/SFTPTestConnectionAction.java | 404 ++++++++++++++++++ .../actions/SFTPTestConnectionActionTest.java | 178 ++++++++ 2 files changed, 582 insertions(+) create mode 100644 qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java new file mode 100644 index 00000000..53b93ccf --- /dev/null +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -0,0 +1,404 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.apache.sshd.sftp.client.SftpClient; + + +/******************************************************************************* + ** action for testing credentials for an SFTP backend connection + *******************************************************************************/ +public class SFTPTestConnectionAction extends AbstractSFTPAction +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) + { + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword())) + { + SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); + + if(StringUtils.hasContent(input.basePath)) + { + try + { + Iterable dirEntries = sftpClient.readDir(input.basePath); + + ///////////////////////////////////////////////////////////////////////// + // it seems like only the .iterator call throws if bad directory here. // + ///////////////////////////////////////////////////////////////////////// + dirEntries.iterator(); + output.setIsListBasePathSuccess(true); + } + catch(Exception e) + { + output.setIsListBasePathSuccess(false); + output.setListBasePathErrorMessage(e.getMessage()); + } + } + + return output; + } + catch(Exception e) + { + return new SFTPTestConnectionTestOutput().withIsConnectionSuccess(false).withConnectionErrorMessage(e.getMessage()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestInput + { + private String username; + private String hostName; + private Integer port; + private String password; + private String basePath; + + + + /******************************************************************************* + ** Getter for username + ** + *******************************************************************************/ + public String getUsername() + { + return username; + } + + + + /******************************************************************************* + ** Setter for username + ** + *******************************************************************************/ + public void setUsername(String username) + { + this.username = username; + } + + + + /******************************************************************************* + ** Fluent setter for username + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withUsername(String username) + { + this.username = username; + return (this); + } + + + + /******************************************************************************* + ** Getter for hostName + ** + *******************************************************************************/ + public String getHostName() + { + return hostName; + } + + + + /******************************************************************************* + ** Setter for hostName + ** + *******************************************************************************/ + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + + + /******************************************************************************* + ** Fluent setter for hostName + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withHostName(String hostName) + { + this.hostName = hostName; + return (this); + } + + + + /******************************************************************************* + ** Getter for port + ** + *******************************************************************************/ + public Integer getPort() + { + return port; + } + + + + /******************************************************************************* + ** Setter for port + ** + *******************************************************************************/ + public void setPort(Integer port) + { + this.port = port; + } + + + + /******************************************************************************* + ** Fluent setter for port + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPort(Integer port) + { + this.port = port; + return (this); + } + + + + /******************************************************************************* + ** Getter for password + ** + *******************************************************************************/ + public String getPassword() + { + return password; + } + + + + /******************************************************************************* + ** Setter for password + ** + *******************************************************************************/ + public void setPassword(String password) + { + this.password = password; + } + + + + /******************************************************************************* + ** Fluent setter for password + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPassword(String password) + { + this.password = password; + return (this); + } + + + + /******************************************************************************* + ** Getter for basePath + ** + *******************************************************************************/ + public String getBasePath() + { + return basePath; + } + + + + /******************************************************************************* + ** Setter for basePath + ** + *******************************************************************************/ + public void setBasePath(String basePath) + { + this.basePath = basePath; + } + + + + /******************************************************************************* + ** Fluent setter for basePath + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withBasePath(String basePath) + { + this.basePath = basePath; + return (this); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class SFTPTestConnectionTestOutput + { + private Boolean isConnectionSuccess; + private String connectionErrorMessage; + + private Boolean isListBasePathSuccess; + private String listBasePathErrorMessage; + + + + /******************************************************************************* + ** Getter for isSuccess + ** + *******************************************************************************/ + public Boolean getIsConnectionSuccess() + { + return isConnectionSuccess; + } + + + + /******************************************************************************* + ** Setter for isSuccess + ** + *******************************************************************************/ + public void setIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsConnectionSuccess(Boolean isSuccess) + { + this.isConnectionSuccess = isSuccess; + return (this); + } + + + + /******************************************************************************* + ** Getter for connectionErrorMessage + ** + *******************************************************************************/ + public String getConnectionErrorMessage() + { + return connectionErrorMessage; + } + + + + /******************************************************************************* + ** Setter for connectionErrorMessage + ** + *******************************************************************************/ + public void setConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for connectionErrorMessage + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withConnectionErrorMessage(String connectionErrorMessage) + { + this.connectionErrorMessage = connectionErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for listBasePathErrorMessage + *******************************************************************************/ + public String getListBasePathErrorMessage() + { + return (this.listBasePathErrorMessage); + } + + + + /******************************************************************************* + ** Setter for listBasePathErrorMessage + *******************************************************************************/ + public void setListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + } + + + + /******************************************************************************* + ** Fluent setter for listBasePathErrorMessage + *******************************************************************************/ + public SFTPTestConnectionTestOutput withListBasePathErrorMessage(String listBasePathErrorMessage) + { + this.listBasePathErrorMessage = listBasePathErrorMessage; + return (this); + } + + + + /******************************************************************************* + ** Getter for isListBasePathSuccess + ** + *******************************************************************************/ + public Boolean getIsListBasePathSuccess() + { + return isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Setter for isListBasePathSuccess + ** + *******************************************************************************/ + public void setIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + } + + + + /******************************************************************************* + ** Fluent setter for isListBasePathSuccess + ** + *******************************************************************************/ + public SFTPTestConnectionTestOutput withIsListBasePathSuccess(Boolean isListBasePathSuccess) + { + this.isListBasePathSuccess = isListBasePathSuccess; + return (this); + } + + } + +} diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java new file mode 100644 index 00000000..b1439cbf --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -0,0 +1,178 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; + + +import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for SFTPTestConnectionAction + *******************************************************************************/ +class SFTPTestConnectionActionTest extends BaseSFTPTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithoutPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessWithPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath(BaseSFTPTest.BACKEND_FOLDER); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertTrue(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSuccessfulConnectFailedPath() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME) + .withBasePath("no-such-path"); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + assertFalse(output.getIsListBasePathSuccess()); + assertNotNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadUsername() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername("not-" + BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPassword() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword("not-" + BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadHostname() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName("not-" + BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBadPort() + { + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPassword(BaseSFTPTest.PASSWORD) + .withPort(10 * BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertFalse(output.getIsConnectionSuccess()); + assertNotNull(output.getConnectionErrorMessage()); + assertNull(output.getIsListBasePathSuccess()); + assertNull(output.getListBasePathErrorMessage()); + } + +} \ No newline at end of file From dcf7218abf1038bdaf1d88698187555d41aa28ec Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:16:58 -0600 Subject: [PATCH 157/209] add basename field --- .../model/metadata/FilesystemTableMetaDataBuilder.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 4157ffb6..3c33eb39 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -81,21 +81,25 @@ public class FilesystemTableMetaDataBuilder .withPrimaryKeyField("fileName") .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("baseName", QFieldType.STRING)) .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("contents", QFieldType.BLOB) .withIsHeavy(true) .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "File Contents"))) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName") + )) .withSection(SectionFactory.defaultT1("fileName")) - .withSection(SectionFactory.defaultT2("contents", "size")) + .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) .withBackendDetails(tableBackendDetails .withCardinality(Cardinality.ONE) .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") .withContentsFieldName("contents") .withSizeFieldName("size") .withCreateDateFieldName("createDate") From 2502d102d97ea9aca663a5edcb899e946a71210e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Feb 2025 20:17:01 -0600 Subject: [PATCH 158/209] Better version (i hope) of using ssh & sftp client objects --- .../sftp/actions/AbstractSFTPAction.java | 68 ++++++++++++++----- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index c9e3c8b5..18cb6c70 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -62,7 +62,43 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction closer = closable -> { - if(closable != null) - { - try - { - closable.close(); - } - catch(Exception e) - { - LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); - } - } + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } }; - closer.accept(sshClient); - closer.accept(clientSession); closer.accept(sftpClient); + closer.accept(clientSession); } @@ -164,10 +199,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction Date: Wed, 19 Feb 2025 20:17:56 -0600 Subject: [PATCH 159/209] Update expected error message --- .../module/filesystem/sftp/actions/SFTPQueryActionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 9cf0234a..9708f68e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -141,7 +141,7 @@ class SFTPQueryActionTest extends BaseSFTPTest QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); assertThatThrownBy(() -> new QueryAction().execute(queryInput)) - .hasMessageContaining("Could not find Backend Variant information for Backend"); + .hasMessageContaining("Could not find Backend Variant information in session under key 'variant-options-table' for Backend"); QContext.getQSession().setBackendVariants(MapBuilder.of(TestUtils.TABLE_NAME_VARIANT_OPTIONS, 1)); QueryOutput queryOutput = new QueryAction().execute(queryInput); From d25eb6ee485cd72616a523c34985d909a2bd2538 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 11:41:29 -0600 Subject: [PATCH 160/209] Simplify file listing by replacing filters with requested paths Refactor file listing mechanisms to replace the use of complex query filters with simpler, path-based requests. Updated module-specific implementations and removed unused filtering logic. Updated tests (zombie'ing some) --- .../actions/AbstractBaseFilesystemAction.java | 45 +++++++---- .../SharedFilesystemBackendModuleUtils.java | 6 +- .../actions/AbstractFilesystemAction.java | 13 +++- .../s3/actions/AbstractS3Action.java | 5 +- .../module/filesystem/s3/utils/S3Utils.java | 68 ++++------------- .../sftp/actions/AbstractSFTPAction.java | 19 +---- .../local/FilesystemBackendModuleTest.java | 76 +++++++++---------- .../filesystem/s3/S3BackendModuleTest.java | 74 +++++++++--------- .../sftp/actions/SFTPQueryActionTest.java | 67 ---------------- 9 files changed, 143 insertions(+), 230 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 0557e68a..76a28706 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -57,8 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; @@ -111,9 +111,10 @@ public abstract class AbstractBaseFilesystemAction public abstract Instant getFileModifyDate(FILE file); /******************************************************************************* - ** List the files for a table - WITH an input filter - to be implemented in module-specific subclasses. + ** List the files for a table - or optionally, just a single file name - + ** to be implemented in module-specific subclasses. *******************************************************************************/ - public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException; + public abstract List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedSingleFileName) throws QException; /******************************************************************************* ** Read the contents of a file - to be implemented in module-specific subclasses. @@ -278,11 +279,26 @@ public abstract class AbstractBaseFilesystemAction try { - QueryOutput queryOutput = new QueryOutput(queryInput); - QTableMetaData table = queryInput.getTable(); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); - List files = listFiles(table, queryInput.getBackend(), queryInput.getFilter()); + + QueryOutput queryOutput = new QueryOutput(queryInput); + + String requestedPath = null; + QQueryFilter filter = queryInput.getFilter(); + if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) + { + if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + { + QFilterCriteria criteria = filter.getCriteria().get(0); + if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) + { + requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0)); + } + } + } + + List files = listFiles(table, queryInput.getBackend(), requestedPath); switch(tableDetails.getCardinality()) { @@ -305,6 +321,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** ***************************************************************************/ @@ -324,6 +341,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** ***************************************************************************/ @@ -382,13 +400,12 @@ public abstract class AbstractBaseFilesystemAction // if so, remove that criteria here, so that its presence doesn't cause all records to be filtered away // ////////////////////////////////////////////////////////////////////////////////////////////////////////// QQueryFilter filterForRecords = queryInput.getFilter(); - if(filterForRecords != null) - { - filterForRecords = filterForRecords.clone(); - - CollectionUtils.nonNullList(filterForRecords.getCriteria()) - .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); - } + // if(filterForRecords != null) + // { + // filterForRecords = filterForRecords.clone(); + // CollectionUtils.nonNullList(filterForRecords.getCriteria()) + // .removeIf(AbstractBaseFilesystemAction::isPathEqualsCriteria); + // } if(BackendQueryFilterUtils.doesRecordMatch(filterForRecords, null, record)) { @@ -560,6 +577,7 @@ public abstract class AbstractBaseFilesystemAction } + /*************************************************************************** ** Method that subclasses can override to add post-action things (e.g., closing resources) ***************************************************************************/ @@ -571,6 +589,7 @@ public abstract class AbstractBaseFilesystemAction } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java index 026386b0..295aee06 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/utils/SharedFilesystemBackendModuleUtils.java @@ -130,7 +130,11 @@ public class SharedFilesystemBackendModuleUtils } else { - throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + /////////////////////////////////////////////////////////////////////////////////////////////// + // this happens in base class now, like, for query action, so, we think okay to just ignore. // + /////////////////////////////////////////////////////////////////////////////////////////////// + // throw (new QException("Unable to query filesystem table by field: " + criteria.getFieldName())); + return (true); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 5dde1ab6..19090262 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -41,6 +41,8 @@ import java.util.ArrayList; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -111,7 +113,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** List the files for this table. *******************************************************************************/ @Override - public List listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { try { @@ -130,7 +132,14 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction for(String matchedFile : matchedFiles) { - if(SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails)) + boolean isMatch = true; + if(StringUtils.hasContent(requestedPath)) + { + QQueryFilter filter = new QQueryFilter(new QFilterCriteria(tableBackendDetails.getFileNameFieldName(), QCriteriaOperator.EQUALS, requestedPath)); + isMatch = SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(matchedFile, filter, tableBackendDetails); + } + + if(isMatch) { rs.add(new File(fullPath + File.separatorChar + matchedFile)); } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 4c5e7b8c..3693d32a 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -33,7 +33,6 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -163,7 +162,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { S3BackendMetaData s3BackendMetaData = getBackendMetaData(S3BackendMetaData.class, backendBase); AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); @@ -175,7 +174,7 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, QQueryFilter filter, AbstractFilesystemTableBackendDetails tableDetails) throws QException + public List listObjectsInBucketMatchingGlob(String bucketName, String path, String glob, String requestedPath, AbstractFilesystemTableBackendDetails tableDetails) throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////// // s3 list requests find nothing if the path starts with a /, so strip away any leading slashes // @@ -96,38 +92,20 @@ public class S3Utils prefix = prefix.substring(0, prefix.indexOf('*')); } - /////////////////////////////////////////////////////////////////////////////////////////////////////// - // for a file-per-record (ONE) table, we may need to apply the filter to listing. // - // but for MANY tables, the filtering would be done on the records after they came out of the files. // - /////////////////////////////////////////////////////////////////////////////////////////////////////// - boolean useQQueryFilter = false; - if(tableDetails != null && Cardinality.ONE.equals(tableDetails.getCardinality())) + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // optimization, to avoid listing whole bucket, for use-case where less than a whole bucket is requested // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(requestedPath)) { - useQQueryFilter = true; - } - - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // if there's a filter for single file, make that file name the "prefix" that we send to s3, so we just get back that 1 file. // - // as this will be a common case. // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter) - { - if(filter.getCriteria() != null && filter.getCriteria().size() == 1) + if(!prefix.isEmpty()) { - QFilterCriteria criteria = filter.getCriteria().get(0); - if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) - { - if(!prefix.isEmpty()) - { - /////////////////////////////////////////////////////// - // remember, a prefix starting with / finds nothing! // - /////////////////////////////////////////////////////// - prefix += "/"; - } - - prefix += criteria.getValues().get(0); - } + /////////////////////////////////////////////////////// + // remember, a prefix starting with / finds nothing! // + /////////////////////////////////////////////////////// + prefix += "/"; } + + prefix += requestedPath; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -203,27 +181,7 @@ public class S3Utils continue; } - /////////////////////////////////////////////////////////////////////////////////// - // if we're a file-per-record table, and we have a filter, compare the key to it // - /////////////////////////////////////////////////////////////////////////////////// - if(!SharedFilesystemBackendModuleUtils.doesFilePathMatchFilter(key, filter, tableDetails)) - { - continue; - } - rs.add(objectSummary); - - ///////////////////////////////////////////////////////////////// - // if we have a limit, and we've hit it, break out of the loop // - ///////////////////////////////////////////////////////////////// - if(filter != null && useQQueryFilter && filter.getLimit() != null) - { - if(rs.size() >= filter.getLimit()) - { - break; - } - } - } } while(listObjectsV2Result.isTruncated()); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 18cb6c70..1bb8a303 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -34,15 +34,13 @@ import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; @@ -267,23 +265,14 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, QQueryFilter filter) throws QException + public List listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { try { String fullPath = getFullBasePath(table, backendBase); - - // todo - move somewhere shared - // todo - should all do this? - if(filter != null) + if(StringUtils.hasContent(requestedPath)) { - for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) - { - if(isPathEqualsCriteria(criteria)) - { - fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); - } - } + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar); } List rs = new ArrayList<>(); diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java index 26cb44ea..117e2cc6 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/FilesystemBackendModuleTest.java @@ -26,9 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -41,7 +38,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -97,55 +93,59 @@ public class FilesystemBackendModuleTest ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertEquals("BLOB-2.txt", files.get(0).getName()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertEquals("BLOB-1.txt", files.get(0).getName()); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = abstractFilesystemAction.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = abstractFilesystemAction.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .rootCause() - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> abstractFilesystemAction.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .rootCause() - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index c18ce4bc..faec900e 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -26,9 +26,6 @@ import java.util.List; import java.util.UUID; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -38,7 +35,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -77,53 +73,59 @@ public class S3BackendModuleTest extends BaseS3Test ///////////////////////////////////////// // filter for a file name that's found // ///////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-2.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-2.txt"); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"))); + files = actionBase.listFiles(table, backend, "BLOB-1.txt"); assertEquals(1, files.size()); assertThat(files.get(0).getKey()).contains("BLOB-1.txt"); - /////////////////////////////////// - // filter for 2 names that exist // - /////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); - assertEquals(2, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // /////////////////////////////////// + // // filter for 2 names that exist // + // /////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-1.txt", "BLOB-2.txt"))); + // assertEquals(2, files.size()); ///////////////////////////////////////////// // filter for a file name that isn't found // ///////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "NOT-FOUND.txt"))); + files = actionBase.listFiles(table, backend, "NOT-FOUND.txt"); assertEquals(0, files.size()); - files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); - assertEquals(1, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IN, "BLOB-2.txt", "NOT-FOUND.txt"))); + // assertEquals(1, files.size()); - //////////////////////////////////////////////////// - // 2 criteria, and'ed, and can't match, so find 0 // - //////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); - assertEquals(0, files.size()); + /////////////////////////// + // not supported anymore // + /////////////////////////// + // //////////////////////////////////////////////////// + // // 2 criteria, and'ed, and can't match, so find 0 // + // //////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt"))); + // assertEquals(0, files.size()); - ////////////////////////////////////////////////// - // 2 criteria, or'ed, and both match, so find 2 // - ////////////////////////////////////////////////// - files = actionBase.listFiles(table, backend, new QQueryFilter( - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), - new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) - .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); - assertEquals(2, files.size()); + // ////////////////////////////////////////////////// + // // 2 criteria, or'ed, and both match, so find 2 // + // ////////////////////////////////////////////////// + // files = actionBase.listFiles(table, backend, new QQueryFilter( + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-1.txt"), + // new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "BLOB-2.txt")) + // .withBooleanOperator(QQueryFilter.BooleanOperator.OR)); + // assertEquals(2, files.size()); - ////////////////////////////////////// - // ensure unsupported filters throw // - ////////////////////////////////////// - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("foo", QCriteriaOperator.GREATER_THAN, 42)))) - .hasMessageContaining("Unable to query filesystem table by field"); - assertThatThrownBy(() -> actionBase.listFiles(table, backend, new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.IS_BLANK)))) - .hasMessageContaining("Unable to query filename field using operator"); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // note that we used to try unsupported filters here, expecting them to throw - but those are // + // more-or-less now implemented in the base class's query method, so, no longer expected to throw here. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 9708f68e..092ea987 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -23,15 +23,11 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; import java.util.List; -import java.util.Map; import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; -import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -62,69 +58,6 @@ class SFTPQueryActionTest extends BaseSFTPTest - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryWithPath() throws Exception - { - String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; - try - { - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); - - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE) - .withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query"); - } - finally - { - rmrfInContainer(subfolderPath); - } - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testQueryWithPathAndNameLike() throws Exception - { - String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/"; - try - { - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt"); - copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt"); - - Map patternExpectedCountMap = Map.of( - "%.txt", 3, - "sub%", 2, - "%1%", 1, - "%", 3, - "*", 0 - ); - - for(Map.Entry entry : patternExpectedCountMap.entrySet()) - { - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() - .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) - .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); - QueryOutput queryOutput = new QueryAction().execute(queryInput); - Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); - } - } - finally - { - rmrfInContainer(subfolderPath); - } - } - - - /******************************************************************************* ** *******************************************************************************/ From 44236f430943d598014ad49f6b745cdc7251c35d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 11:42:23 -0600 Subject: [PATCH 161/209] change to not include createDate field for s3 (where it's not supported); changed file-name field used on the download adornment to be baseName by default, but configurable --- .../FilesystemTableMetaDataBuilder.java | 106 +++++++++++++----- 1 file changed, 80 insertions(+), 26 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java index 3c33eb39..7e61300f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/FilesystemTableMetaDataBuilder.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; +import java.util.ArrayList; +import java.util.List; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; import com.kingsrook.qqq.backend.module.filesystem.local.FilesystemBackendModule; @@ -59,6 +62,8 @@ public class FilesystemTableMetaDataBuilder private String basePath; private String glob; + private String contentsAdornmentFileNameField = "baseName"; + /******************************************************************************* @@ -66,46 +71,64 @@ public class FilesystemTableMetaDataBuilder *******************************************************************************/ public QTableMetaData buildStandardCardinalityOneTable() { + boolean includeCreateDate = true; AbstractFilesystemTableBackendDetails tableBackendDetails = switch(backend.getBackendType()) { - case S3BackendModule.BACKEND_TYPE -> new S3TableBackendDetails(); + case S3BackendModule.BACKEND_TYPE -> + { + includeCreateDate = false; + yield new S3TableBackendDetails(); + } case FilesystemBackendModule.BACKEND_TYPE -> new FilesystemTableBackendDetails(); case SFTPBackendModule.BACKEND_TYPE -> new SFTPTableBackendDetails(); default -> throw new IllegalStateException("Unexpected value: " + backend.getBackendType()); }; + List fields = new ArrayList<>(); + + fields.add((new QFieldMetaData("fileName", QFieldType.STRING))); + fields.add((new QFieldMetaData("baseName", QFieldType.STRING))); + fields.add((new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS))); + fields.add((new QFieldMetaData("modifyDate", QFieldType.DATE_TIME))); + fields.add((new QFieldMetaData("contents", QFieldType.BLOB) + .withIsHeavy(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, contentsAdornmentFileNameField + )))); + + QFieldSection t3Section = SectionFactory.defaultT3("modifyDate"); + + AbstractFilesystemTableBackendDetails backendDetails = tableBackendDetails + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withBaseNameFieldName("baseName") + .withContentsFieldName("contents") + .withSizeFieldName("size") + .withModifyDateFieldName("modifyDate") + .withBasePath(basePath) + .withGlob(glob); + + if(includeCreateDate) + { + fields.add((new QFieldMetaData("createDate", QFieldType.DATE_TIME))); + backendDetails.setCreateDateFieldName("createDate"); + + ArrayList t3FieldNames = new ArrayList<>(t3Section.getFieldNames()); + t3FieldNames.add(0, "createDate"); + t3Section.setFieldNames(t3FieldNames); + } + return new QTableMetaData() .withName(name) .withIsHidden(true) .withBackendName(backend.getName()) .withPrimaryKeyField("fileName") - - .withField(new QFieldMetaData("fileName", QFieldType.STRING)) - .withField(new QFieldMetaData("baseName", QFieldType.STRING)) - .withField(new QFieldMetaData("size", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS)) - .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME)) - .withField(new QFieldMetaData("contents", QFieldType.BLOB) - .withIsHeavy(true) - .withFieldAdornment(new FieldAdornment(AdornmentType.FILE_DOWNLOAD) - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "%s") - .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName") - )) - + .withFields(fields) .withSection(SectionFactory.defaultT1("fileName")) .withSection(SectionFactory.defaultT2("baseName", "contents", "size")) - .withSection(SectionFactory.defaultT3("createDate", "modifyDate")) - - .withBackendDetails(tableBackendDetails - .withCardinality(Cardinality.ONE) - .withFileNameFieldName("fileName") - .withBaseNameFieldName("baseName") - .withContentsFieldName("contents") - .withSizeFieldName("size") - .withCreateDateFieldName("createDate") - .withModifyDateFieldName("modifyDate") - .withBasePath(basePath) - .withGlob(glob)); + .withSection(t3Section) + .withBackendDetails(backendDetails); } @@ -232,4 +255,35 @@ public class FilesystemTableMetaDataBuilder return (this); } + + /******************************************************************************* + ** Getter for contentsAdornmentFileNameField + *******************************************************************************/ + public String getContentsAdornmentFileNameField() + { + return (this.contentsAdornmentFileNameField); + } + + + + /******************************************************************************* + ** Setter for contentsAdornmentFileNameField + *******************************************************************************/ + public void setContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + } + + + + /******************************************************************************* + ** Fluent setter for contentsAdornmentFileNameField + *******************************************************************************/ + public FilesystemTableMetaDataBuilder withContentsAdornmentFileNameField(String contentsAdornmentFileNameField) + { + this.contentsAdornmentFileNameField = contentsAdornmentFileNameField; + return (this); + } + + } From d401cc9ae183d3cb0ea4e371897c4ccb13f78f3b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 14:29:08 -0600 Subject: [PATCH 162/209] Implement and test DeleteAction functionality - Unified `deleteFile` API across storage modules by removing unused `QInstance` parameter. - Added implementations for S3, SFTP, and local filesystem deleteAction. --- .../actions/AbstractBaseFilesystemAction.java | 61 ++++++++++++++- .../actions/AbstractFilesystemAction.java | 2 +- .../local/actions/FilesystemDeleteAction.java | 27 +++---- .../basic/BasicETLCleanupSourceFilesStep.java | 2 +- .../importer/FilesystemImporterStep.java | 2 +- .../s3/actions/AbstractS3Action.java | 5 +- .../filesystem/s3/actions/S3DeleteAction.java | 27 +++---- .../sftp/actions/AbstractSFTPAction.java | 11 ++- .../sftp/actions/SFTPDeleteAction.java | 27 +++---- .../local/FilesystemBackendModuleTest.java | 4 +- .../actions/FilesystemDeleteActionTest.java | 29 ++++++-- .../filesystem/s3/S3BackendModuleTest.java | 6 +- .../s3/actions/S3DeleteActionTest.java | 48 +++++++++++- .../module/filesystem/sftp/BaseSFTPTest.java | 7 +- .../sftp/actions/SFTPDeleteActionTest.java | 74 ++++++++++++++++++- 15 files changed, 259 insertions(+), 73 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 76a28706..30999153 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -40,6 +40,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -57,6 +59,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSett import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; import com.kingsrook.qqq.backend.core.model.statusmessages.SystemErrorStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; @@ -138,7 +142,7 @@ public abstract class AbstractBaseFilesystemAction ** ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ - public abstract void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException; + public abstract void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException; /******************************************************************************* ** Move a file from a source path, to a destination path. @@ -285,7 +289,7 @@ public abstract class AbstractBaseFilesystemAction QueryOutput queryOutput = new QueryOutput(queryInput); String requestedPath = null; - QQueryFilter filter = queryInput.getFilter(); + QQueryFilter filter = queryInput.getFilter(); if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) { if(filter.getCriteria() != null && filter.getCriteria().size() == 1) @@ -670,4 +674,57 @@ public abstract class AbstractBaseFilesystemAction } } + + + /******************************************************************************* + ** + *******************************************************************************/ + protected DeleteOutput executeDelete(DeleteInput deleteInput) throws QException + { + try + { + preAction(deleteInput.getBackend()); + + DeleteOutput output = new DeleteOutput(); + output.setRecordsWithErrors(new ArrayList<>()); + + QTableMetaData table = deleteInput.getTable(); + QBackendMetaData backend = deleteInput.getBackend(); + + AbstractFilesystemTableBackendDetails tableDetails = getTableBackendDetails(AbstractFilesystemTableBackendDetails.class, table); + if(tableDetails.getCardinality().equals(Cardinality.ONE)) + { + int deletedCount = 0; + for(Serializable primaryKey : deleteInput.getPrimaryKeys()) + { + try + { + deleteFile(table, stripDuplicatedSlashes(getFullBasePath(table, backend) + "/" + primaryKey)); + deletedCount++; + } + catch(Exception e) + { + String message = ObjectUtils.tryElse(() -> ExceptionUtils.getRootException(e).getMessage(), "Message not available"); + output.addRecordWithError(new QRecord().withValue(table.getPrimaryKeyField(), primaryKey).withError(new SystemErrorStatusMessage("Error deleting file: " + message))); + } + } + output.setDeletedRecordCount(deletedCount); + } + else + { + throw (new NotImplementedException("Delete is not implemented for filesystem tables with cardinality: " + tableDetails.getCardinality())); + } + + return (output); + } + catch(Exception e) + { + throw new QException("Error executing delete: " + e.getMessage(), e); + } + finally + { + postAction(); + } + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java index 19090262..cb7d9255 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/AbstractFilesystemAction.java @@ -230,7 +230,7 @@ public class AbstractFilesystemAction extends AbstractBaseFilesystemAction ** @throws FilesystemException if the delete is known to have failed, and the file is thought to still exit *******************************************************************************/ @Override - public void deleteFile(QInstance instance, QTableMetaData table, String fileReference) throws FilesystemException + public void deleteFile(QTableMetaData table, String fileReference) throws FilesystemException { File file = new File(fileReference); if(!file.exists()) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java index 6d304f00..6dda158d 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteAction.java @@ -26,13 +26,12 @@ import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; -import org.apache.commons.lang.NotImplementedException; /******************************************************************************* ** *******************************************************************************/ -public class FilesystemDeleteAction implements DeleteInterface +public class FilesystemDeleteAction extends AbstractFilesystemAction implements DeleteInterface { /******************************************************************************* @@ -40,21 +39,19 @@ public class FilesystemDeleteAction implements DeleteInterface *******************************************************************************/ public DeleteOutput execute(DeleteInput deleteInput) throws QException { - throw new NotImplementedException("Filesystem delete not implemented"); - /* - try - { - DeleteResult rs = new DeleteResult(); - QTableMetaData table = deleteRequest.getTable(); + return (executeDelete(deleteInput)); + } - // return rs; - } - catch(Exception e) - { - throw new QException("Error executing delete: " + e.getMessage(), e); - } - */ + + /******************************************************************************* + ** Specify whether this particular module's update action can & should fetch + ** records before updating them, e.g., for audits or "not-found-checks" + *******************************************************************************/ + @Override + public boolean supportsPreFetchQuery() + { + return (false); } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java index 68a319ca..604d4c2b 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java @@ -94,7 +94,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep if(VALUE_DELETE.equals(moveOrDelete)) { LOG.info("Deleting ETL source file: " + sourceFile); - actionBase.deleteFile(QContext.getQInstance(), table, sourceFile); + actionBase.deleteFile(table, sourceFile); } else if(VALUE_MOVE.equals(moveOrDelete)) { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java index f41ceb6b..bc0c3ec2 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/importer/FilesystemImporterStep.java @@ -319,7 +319,7 @@ public class FilesystemImporterStep implements BackendStep { String fullBasePath = sourceActionBase.getFullBasePath(sourceTable, sourceBackend); LOG.info("Removing source file", logPair("path", fullBasePath + "/" + sourceFileName), logPair("sourceTable", sourceTable.getName())); - sourceActionBase.deleteFile(QContext.getQInstance(), sourceTable, fullBasePath + "/" + sourceFileName); + sourceActionBase.deleteFile(sourceTable, fullBasePath + "/" + sourceFileName); } else { diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 3693d32a..3b8bcb09 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -31,6 +31,7 @@ import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -247,9 +248,9 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, filesBeforeDelete.get(0).getAbsolutePath()); + filesystemBackendModule.getActionBase().deleteFile(table, filesBeforeDelete.get(0).getAbsolutePath()); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size() - 1, filesAfterDelete.size(), @@ -191,7 +191,7 @@ public class FilesystemBackendModuleTest List filesBeforeDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); FilesystemBackendModule filesystemBackendModule = new FilesystemBackendModule(); - filesystemBackendModule.getActionBase().deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + filesystemBackendModule.getActionBase().deleteFile(table, PATH_THAT_WONT_EXIST); List filesAfterDelete = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName())); Assertions.assertEquals(filesBeforeDelete.size(), filesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java index e43a667e..ff113a24 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemDeleteActionTest.java @@ -22,11 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; -import org.apache.commons.lang.NotImplementedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -34,14 +42,25 @@ import static org.junit.jupiter.api.Assertions.assertThrows; *******************************************************************************/ public class FilesystemDeleteActionTest extends FilesystemActionTest { - /******************************************************************************* ** *******************************************************************************/ @Test - public void test() throws QException + public void testSuccessfulDeleteMultiple() throws QException { - assertThrows(NotImplementedException.class, () -> new FilesystemDeleteAction().execute(new DeleteInput())); + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount(); + + String filename1 = "A.txt"; + String filename2 = "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_BLOB_LOCAL_FS)).getCount()); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java index faec900e..b3b187de 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/S3BackendModuleTest.java @@ -147,7 +147,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, s3ObjectSummariesBeforeDelete.get(0).getKey()); + actionBase.deleteFile(table, s3ObjectSummariesBeforeDelete.get(0).getKey()); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -176,7 +176,7 @@ public class S3BackendModuleTest extends BaseS3Test AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); String path = "//" + s3ObjectSummariesBeforeDelete.get(0).getKey().replaceAll("/", "//"); - actionBase.deleteFile(qInstance, table, "//" + path); + actionBase.deleteFile(table, "//" + path); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size() - 1, s3ObjectSummariesAfterDelete.size(), @@ -203,7 +203,7 @@ public class S3BackendModuleTest extends BaseS3Test S3BackendModule s3BackendModule = new S3BackendModule(); AbstractS3Action actionBase = (AbstractS3Action) s3BackendModule.getActionBase(); actionBase.setS3Utils(getS3Utils()); - actionBase.deleteFile(qInstance, table, PATH_THAT_WONT_EXIST); + actionBase.deleteFile(table, PATH_THAT_WONT_EXIST); List s3ObjectSummariesAfterDelete = getS3Utils().listObjectsInBucketMatchingGlob(BUCKET_NAME, TEST_FOLDER, ""); Assertions.assertEquals(s3ObjectSummariesBeforeDelete.size(), s3ObjectSummariesAfterDelete.size(), diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java index 6b1ba2fa..ce01677f 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3DeleteActionTest.java @@ -22,12 +22,19 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions; +import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test; -import org.apache.commons.lang.NotImplementedException; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -42,7 +49,42 @@ public class S3DeleteActionTest extends BaseS3Test @Test public void test() throws QException { - assertThrows(NotImplementedException.class, () -> new S3DeleteAction().execute(new DeleteInput())); + QInstance qInstance = TestUtils.defineInstance(); + + int initialCount = count(TestUtils.TABLE_NAME_BLOB_S3); + + InsertInput insertInput = new InsertInput(); + insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3); + insertInput.setRecords(List.of( + new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob."))); + + S3InsertAction insertAction = new S3InsertAction(); + insertAction.setS3Utils(getS3Utils()); + insertAction.execute(insertInput); + + assertEquals(initialCount + 1, count(TestUtils.TABLE_NAME_BLOB_S3)); + + S3DeleteAction deleteAction = new S3DeleteAction(); + deleteAction.setS3Utils(getS3Utils()); + DeleteOutput deleteOutput = deleteAction.execute(new DeleteInput(TestUtils.TABLE_NAME_BLOB_S3).withPrimaryKeys(List.of("file2.txt"))); + assertEquals(1, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + + assertEquals(initialCount, count(TestUtils.TABLE_NAME_BLOB_S3)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + private Integer count(String tableName) throws QException + { + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + S3CountAction s3CountAction = new S3CountAction(); + s3CountAction.setS3Utils(getS3Utils()); + CountOutput countOutput = s3CountAction.execute(countInput); + return countOutput.getCount(); } } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java index 42c968be..6c71ea42 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -43,8 +43,8 @@ public class BaseSFTPTest extends BaseTest public static final String TABLE_FOLDER = "files"; public static final String REMOTE_DIR = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER; - private static GenericContainer sftpContainer; - private static Integer currentPort; + protected static GenericContainer sftpContainer; + private static Integer currentPort; @@ -71,6 +71,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ @@ -80,6 +81,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ @@ -89,6 +91,7 @@ public class BaseSFTPTest extends BaseTest } + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java index 3a992fbf..15d74d3b 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPDeleteActionTest.java @@ -22,12 +22,23 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; -import org.apache.commons.lang.NotImplementedException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -35,14 +46,69 @@ import static org.junit.jupiter.api.Assertions.assertThrows; *******************************************************************************/ public class SFTPDeleteActionTest extends BaseSFTPTest { + private String filesBasename = "delete-test-"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() throws Exception + { + rmrfInContainer(REMOTE_DIR + "/" + filesBasename + "*"); + } + + /******************************************************************************* ** *******************************************************************************/ @Test - public void test() throws QException + public void testSuccessfulDeleteMultiple() throws QException { - assertThrows(NotImplementedException.class, () -> new SFTPDeleteAction().execute(new DeleteInput())); + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "A.txt"; + String filename2 = filesBasename + "B.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + assertEquals(2, deleteOutput.getDeletedRecordCount()); + assertEquals(0, deleteOutput.getRecordsWithErrors().size()); + assertEquals(initialCount, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testFailedDelete() throws Exception + { + int initialCount = new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount(); + + String filename1 = filesBasename + "C.txt"; + String filename2 = filesBasename + "D.txt"; + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SFTP_FILE).withRecords(List.of( + new QRecord().withValue("fileName", filename1).withValue("contents", "bytes"), + new QRecord().withValue("fileName", filename2).withValue("contents", "bytes")))); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); + + sftpContainer.execInContainer("chmod", "000", REMOTE_DIR); + DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(TestUtils.TABLE_NAME_SFTP_FILE).withPrimaryKeys(List.of(filename1, filename2))); + sftpContainer.execInContainer("chmod", "777", REMOTE_DIR); + + assertEquals(0, deleteOutput.getDeletedRecordCount()); + assertEquals(2, deleteOutput.getRecordsWithErrors().size()); + assertThat(deleteOutput.getRecordsWithErrors().get(0).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertThat(deleteOutput.getRecordsWithErrors().get(1).getErrorsAsString()).contains("Error deleting file: Permission denied"); + assertEquals(initialCount + 2, new CountAction().execute(new CountInput(TestUtils.TABLE_NAME_SFTP_FILE)).getCount()); } } \ No newline at end of file From 05bb0ef363d4778d9652558f5844531da652076a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 20 Feb 2025 15:35:52 -0600 Subject: [PATCH 163/209] Fix withoutCapabilities(Set) - was calling with, not without :( --- .../qqq/backend/core/model/metadata/tables/QTableMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 1df589d1..0d0d6152 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -1056,7 +1056,7 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData { for(Capability disabledCapability : disabledCapabilities) { - withCapability(disabledCapability); + withoutCapability(disabledCapability); } return (this); } From a659dc7a020f9c6a1719f811e13805ce9aef4cab Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 20 Feb 2025 23:25:35 -0600 Subject: [PATCH 164/209] CE-2261: updated call to set table name to not be escaped --- .../qqq/backend/module/rdbms/actions/RDBMSInsertAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index d6b89246..d2dbb8f9 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -132,6 +133,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(getTableName(table)); + } outputRecords.add(outputRecord); } continue; From 3114812e34b8f9c69123d5343c67de0be69b266d Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Fri, 21 Feb 2025 09:44:29 -0600 Subject: [PATCH 165/209] CE-2261: qqq updates to table name --- .../module/rdbms/actions/RDBMSInsertAction.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java index d2dbb8f9..d2a78560 100644 --- a/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java +++ b/qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java @@ -93,8 +93,8 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(List page : CollectionUtils.getPages(insertInput.getRecords(), getActionStrategy().getPageSize(insertInput))) { - String tableName = escapeIdentifier(getTableName(table)); - sql = new StringBuilder("INSERT INTO ").append(tableName).append("(").append(columns).append(") VALUES"); + String backendTableName = escapeIdentifier(getTableName(table)); + sql = new StringBuilder("INSERT INTO ").append(backendTableName).append("(").append(columns).append(") VALUES"); params = new ArrayList<>(); int recordIndex = 0; @@ -135,7 +135,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte QRecord outputRecord = new QRecord(record); if(!StringUtils.hasContent(outputRecord.getTableName())) { - outputRecord.setTableName(getTableName(table)); + outputRecord.setTableName(table.getName()); } outputRecords.add(outputRecord); } @@ -156,6 +156,10 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte for(QRecord record : page) { QRecord outputRecord = new QRecord(record); + if(!StringUtils.hasContent(outputRecord.getTableName())) + { + outputRecord.setTableName(table.getName()); + } outputRecords.add(outputRecord); if(CollectionUtils.nullSafeIsEmpty(record.getErrors())) From b2c8c075fdf050bd2d8b6fb52b2656e71eef1536 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Fri, 21 Feb 2025 12:09:36 -0600 Subject: [PATCH 166/209] CE-2260: added utility method for getting oath access key which will handle variants properly --- .../module/api/actions/BaseAPIActionUtil.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 209285e0..6a2b227a 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -845,7 +845,7 @@ public class BaseAPIActionUtil /******************************************************************************* ** *******************************************************************************/ - public String getOAuth2Token() throws OAuthCredentialsException, QException + String getOAuth2AccessTokenKey() throws QException { ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // define the key that will be used in the backend's customValues map, to stash the access token. // @@ -857,6 +857,17 @@ public class BaseAPIActionUtil Serializable variantId = getVariantId(); accessTokenKey = accessTokenKey + ":" + variantId; } + return (accessTokenKey); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public String getOAuth2Token() throws QException + { + String accessTokenKey = getOAuth2AccessTokenKey(); //////////////////////////////////////////////////////////////////////////////////////////////////////////////// // check for the access token in the backend meta data. if it's not there, then issue a request for a token. // @@ -1245,6 +1256,8 @@ public class BaseAPIActionUtil if(!caughtAnOAuthExpiredToken) { LOG.info("OAuth Expired token for [" + table.getName() + "] - retrying"); + // String accessTokenKey = getOAuth2AccessTokenKey(); + // backendMetaData.withCustomValue(accessTokenKey, null); backendMetaData.withCustomValue("accessToken", null); caughtAnOAuthExpiredToken = true; } From e2b81e46b9220be68d36e250514c12a5e1f449c5 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Fri, 21 Feb 2025 12:36:40 -0600 Subject: [PATCH 167/209] CE-2260: fixes to oath with variants --- .../qqq/backend/module/api/actions/BaseAPIActionUtil.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java index 6a2b227a..40206565 100644 --- a/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java +++ b/qqq-backend-module-api/src/main/java/com/kingsrook/qqq/backend/module/api/actions/BaseAPIActionUtil.java @@ -1256,9 +1256,8 @@ public class BaseAPIActionUtil if(!caughtAnOAuthExpiredToken) { LOG.info("OAuth Expired token for [" + table.getName() + "] - retrying"); - // String accessTokenKey = getOAuth2AccessTokenKey(); - // backendMetaData.withCustomValue(accessTokenKey, null); - backendMetaData.withCustomValue("accessToken", null); + String accessTokenKey = getOAuth2AccessTokenKey(); + backendMetaData.withCustomValue(accessTokenKey, null); caughtAnOAuthExpiredToken = true; } else From 693dfb2d5b968285f19e39f9d487ec8efea65bfb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:02:29 -0600 Subject: [PATCH 168/209] Update getExistingRecordQueryFilter to convert sourceKeyList to be in the destination foreign key field's type --- .../AbstractTableSyncTransformStep.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java index b14847f7..c8f6b82e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/tablesync/AbstractTableSyncTransformStep.java @@ -173,8 +173,21 @@ public abstract class AbstractTableSyncTransformStep extends AbstractTransformSt *******************************************************************************/ protected QQueryFilter getExistingRecordQueryFilter(RunBackendStepInput runBackendStepInput, List sourceKeyList) { - String destinationTableForeignKeyField = getSyncProcessConfig().destinationTableForeignKey; - return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyField, QCriteriaOperator.IN, sourceKeyList)); + String destinationTableForeignKeyFieldName = getSyncProcessConfig().destinationTableForeignKey; + String destinationTableName = getSyncProcessConfig().destinationTable; + QFieldMetaData destinationForeignKeyField = QContext.getQInstance().getTable(destinationTableName).getField(destinationTableForeignKeyFieldName); + + List sourceKeysInDestinationKeyTypeList = null; + if(sourceKeyList != null) + { + sourceKeysInDestinationKeyTypeList = new ArrayList<>(); + for(Serializable sourceKey : sourceKeyList) + { + sourceKeysInDestinationKeyTypeList.add(ValueUtils.getValueAsFieldType(destinationForeignKeyField.getType(), sourceKey)); + } + } + + return new QQueryFilter().withCriteria(new QFilterCriteria(destinationTableForeignKeyFieldName, QCriteriaOperator.IN, sourceKeysInDestinationKeyTypeList)); } From df530b70b85021e78c326badde29e2af91ec9bca Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:04:02 -0600 Subject: [PATCH 169/209] Add static wrapper --- .../core/actions/tables/CountAction.java | 17 +++++++++++++++++ .../core/actions/tables/CountActionTest.java | 18 ++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java index 5a25c396..497d6f34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/CountAction.java @@ -32,6 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; @@ -82,6 +83,22 @@ public class CountAction + /******************************************************************************* + ** shorthand way to call for the most common use-case, when you just want the + ** count to be returned, and you just want to pass in a table name and filter. + *******************************************************************************/ + public static Integer execute(String tableName, QQueryFilter filter) throws QException + { + CountAction countAction = new CountAction(); + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + countInput.setFilter(filter); + CountOutput countOutput = countAction.execute(countInput); + return (countOutput.getCount()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java index ba1c7c9b..5a93f970 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/CountActionTest.java @@ -23,10 +23,14 @@ package com.kingsrook.qqq.backend.core.actions.tables; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -50,4 +54,18 @@ class CountActionTest extends BaseTest CountOutput result = new CountAction().execute(request); assertNotNull(result); } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStaticWrapper() throws QException + { + TestUtils.insertDefaultShapes(QContext.getQInstance()); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, null)); + assertEquals(3, CountAction.execute(TestUtils.TABLE_NAME_SHAPE, new QQueryFilter())); + } + } From 0395e0d02cfdd77c75aec5def34f094cb0696dbd Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:04:23 -0600 Subject: [PATCH 170/209] Add warning if input primaryKey is a filter (because that's probably not what you wanted!) --- .../kingsrook/qqq/backend/core/actions/tables/GetAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java index 24c6e983..c98e31f3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java @@ -238,6 +238,11 @@ public class GetAction *******************************************************************************/ public static QRecord execute(String tableName, Serializable primaryKey) throws QException { + if(primaryKey instanceof QQueryFilter) + { + LOG.warn("Unexpected use of QQueryFilter instead of primary key in GetAction call"); + } + GetAction getAction = new GetAction(); GetInput getInput = new GetInput(tableName).withPrimaryKey(primaryKey); return getAction.executeForRecord(getInput); From f4f2f3c80e67f208f7253ed9bc41e54d9cf56877 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 15:05:11 -0600 Subject: [PATCH 171/209] Refactor a findProducers method out of processAllMetaDataProducersInPackage, for more flexibility (e.g., in QBitProducers) --- .../metadata/MetaDataProducerHelper.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index d4600787..487c6dda 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -106,14 +106,10 @@ public class MetaDataProducerHelper } - - /******************************************************************************* - ** Recursively find all classes in the given package, that implement MetaDataProducerInterface - ** run them, and add their output to the given qInstance. + /*************************************************************************** ** - ** Note - they'll be sorted by the sortOrder they provide. - *******************************************************************************/ - public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + ***************************************************************************/ + public static List> findProducers(String packageName) throws QException { List> classesInPackage; try @@ -196,6 +192,20 @@ public class MetaDataProducerHelper } })); + return (producers); + } + + + /******************************************************************************* + ** Recursively find all classes in the given package, that implement MetaDataProducerInterface + ** run them, and add their output to the given qInstance. + ** + ** Note - they'll be sorted by the sortOrder they provide. + *******************************************************************************/ + public static void processAllMetaDataProducersInPackage(QInstance instance, String packageName) throws QException + { + List> producers = findProducers(packageName); + /////////////////////////////////////////////////////////////////////////// // execute each one (if enabled), adding their meta data to the instance // /////////////////////////////////////////////////////////////////////////// From 001860fc91e743f0ca221849501695b24bb2e13a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:24:30 -0600 Subject: [PATCH 172/209] Initial checkin of QBits, mostly. --- .../metadata/MetaDataProducerMultiOutput.java | 49 +++- .../core/model/metadata/QInstance.java | 67 +++++ .../metadata/processes/QProcessMetaData.java | 46 +++- .../qbits/ProvidedOrSuppliedTableConfig.java | 122 +++++++++ .../qbits/QBitComponentMetaDataProducer.java | 71 ++++++ .../core/model/metadata/qbits/QBitConfig.java | 110 ++++++++ .../qbits/QBitConfigValidationException.java | 44 ++++ .../model/metadata/qbits/QBitMetaData.java | 237 ++++++++++++++++++ .../model/metadata/qbits/QBitProducer.java | 117 +++++++++ .../model/metadata/qbits/SourceQBitAware.java | 77 ++++++ .../model/metadata/tables/QTableMetaData.java | 39 ++- .../metadata/qbits/QBitProducerTest.java | 151 +++++++++++ .../qbits/testqbit/TestQBitConfig.java | 181 +++++++++++++ .../qbits/testqbit/TestQBitProducer.java | 92 +++++++ .../metadata/OtherTableMetaDataProducer.java | 69 +++++ .../metadata/SomeTableMetaDataProducer.java | 68 +++++ 16 files changed, 1534 insertions(+), 6 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java index d6797a2a..32c446a1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerMultiOutput.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.model.metadata; import java.util.ArrayList; import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -31,10 +32,12 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; ** Output object for a MetaDataProducer, which contains multiple meta-data ** objects. *******************************************************************************/ -public class MetaDataProducerMultiOutput implements MetaDataProducerOutput +public class MetaDataProducerMultiOutput implements MetaDataProducerOutput, SourceQBitAware { private List contents; + private String sourceQBitName; + /******************************************************************************* @@ -98,4 +101,48 @@ public class MetaDataProducerMultiOutput implements MetaDataProducerOutput return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + + ///////////////////////////////////////////// + // propagate the name down to the children // + ///////////////////////////////////////////// + for(MetaDataProducerOutput content : contents) + { + if(content instanceof SourceQBitAware aware) + { + aware.setSourceQBitName(sourceQBitName); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public MetaDataProducerMultiOutput withSourceQBitName(String sourceQBitName) + { + setSourceQBitName(sourceQBitName); + return this; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java index a61971c8..def809b9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java @@ -56,6 +56,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRule import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueMetaData; import com.kingsrook.qqq.backend.core.model.metadata.queues.QQueueProviderMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; @@ -89,6 +90,7 @@ public class QInstance //////////////////////////////////////////////////////////////////////////////////////////// // Important to use LinkedHashmap here, to preserve the order in which entries are added. // //////////////////////////////////////////////////////////////////////////////////////////// + private Map qBits = new LinkedHashMap<>(); private Map tables = new LinkedHashMap<>(); private Map joins = new LinkedHashMap<>(); private Map possibleValueSources = new LinkedHashMap<>(); @@ -1489,6 +1491,7 @@ public class QInstance } + /******************************************************************************* ** Getter for metaDataFilter *******************************************************************************/ @@ -1519,4 +1522,68 @@ public class QInstance } + + /******************************************************************************* + ** + *******************************************************************************/ + public void addQBit(QBitMetaData qBitMetaData) + { + List missingParts = new ArrayList<>(); + if(!StringUtils.hasContent(qBitMetaData.getGroupId())) + { + missingParts.add("groupId"); + } + if(!StringUtils.hasContent(qBitMetaData.getArtifactId())) + { + missingParts.add("artifactId"); + } + if(!StringUtils.hasContent(qBitMetaData.getVersion())) + { + missingParts.add("version"); + + } + if(!missingParts.isEmpty()) + { + throw (new IllegalArgumentException("Attempted to add a qBit without a " + StringUtils.joinWithCommasAndAnd(missingParts))); + } + + String name = qBitMetaData.getName(); + if(this.qBits.containsKey(name)) + { + throw (new IllegalArgumentException("Attempted to add a second qBit with name (formed from 'groupId:artifactId:version[:namespace]'): " + name)); + } + this.qBits.put(name, qBitMetaData); + } + + + + /******************************************************************************* + ** Getter for qBits + *******************************************************************************/ + public Map getQBits() + { + return (this.qBits); + } + + + + /******************************************************************************* + ** Setter for qBits + *******************************************************************************/ + public void setQBits(Map qBits) + { + this.qBits = qBits; + } + + + + /******************************************************************************* + ** Fluent setter for qBits + *******************************************************************************/ + public QInstance withQBits(Map qBits) + { + this.qBits = qBits; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java index 582b36e2..21112267 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QProcessMetaData.java @@ -37,6 +37,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -46,11 +47,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; ** Meta-Data to define a process in a QQQ instance. ** *******************************************************************************/ -public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { - private String name; - private String label; - private String tableName; + private String name; + private String label; + private String tableName; + + private String sourceQBitName; + private boolean isHidden = false; private BasepullConfiguration basepullConfiguration; private QPermissionRules permissionRules; @@ -870,6 +874,7 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi } + /******************************************************************************* ** Getter for processTracerCodeReference *******************************************************************************/ @@ -900,4 +905,37 @@ public class QProcessMetaData implements QAppChildMetaData, MetaDataWithPermissi } + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QProcessMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java new file mode 100644 index 00000000..3383a4c6 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/ProvidedOrSuppliedTableConfig.java @@ -0,0 +1,122 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +/*************************************************************************** + ** Common (maybe)? qbit config pattern, where the qbit may be able to provide + ** a particular table, or, the application may supply it itself. + ** + ** If the qbit provides it, then we need to be told (by the application) + ** what backendName to use for the table. + ** + ** Else if the application supplies it, it needs to tell the qBit what the + ** tableName is. + ***************************************************************************/ +public class ProvidedOrSuppliedTableConfig +{ + private boolean doProvideTable; + private String backendName; + private String tableName; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public ProvidedOrSuppliedTableConfig(boolean doProvideTable, String backendName, String tableName) + { + this.doProvideTable = doProvideTable; + this.backendName = backendName; + this.tableName = tableName; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig provideTableUsingBackendNamed(String backendName) + { + return (new ProvidedOrSuppliedTableConfig(true, backendName, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static ProvidedOrSuppliedTableConfig useSuppliedTaleNamed(String tableName) + { + return (new ProvidedOrSuppliedTableConfig(false, null, tableName)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getEffectiveTableName(String tableNameIfProviding) + { + if (getDoProvideTable()) + { + return tableNameIfProviding; + } + else + { + return getTableName(); + } + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Getter for doProvideTable + ** + *******************************************************************************/ + public boolean getDoProvideTable() + { + return doProvideTable; + } + + + + /******************************************************************************* + ** Getter for backendName + ** + *******************************************************************************/ + public String getBackendName() + { + return backendName; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java new file mode 100644 index 00000000..346633f8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitComponentMetaDataProducer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; + + +/******************************************************************************* + ** extension of MetaDataProducerInterface, designed for producing meta data + ** within a (java-defined, at this time) QBit. + ** + ** Specifically exists to accept the QBitConfig as a type parameter and a value, + ** easily accessed in the producer's methods as getQBitConfig() + *******************************************************************************/ +public abstract class QBitComponentMetaDataProducer implements MetaDataProducerInterface +{ + private C qBitConfig = null; + + + + /******************************************************************************* + ** Getter for qBitConfig + *******************************************************************************/ + public C getQBitConfig() + { + return (this.qBitConfig); + } + + + + /******************************************************************************* + ** Setter for qBitConfig + *******************************************************************************/ + public void setQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for qBitConfig + *******************************************************************************/ + public QBitComponentMetaDataProducer withQBitConfig(C qBitConfig) + { + this.qBitConfig = qBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java new file mode 100644 index 00000000..bb4ea5c1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfig.java @@ -0,0 +1,110 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Interface for configuration settings used both in the production of meta-data + ** for a QBit, but also at runtime, e.g., to be aware of exactly how the qbit + ** has been incorporated into an application. + ** + ** For example: + ** - should the QBit define certain tables, or will they be supplied by the application? + ** - what other meta-data names should the qbit reference (backends, schedulers) + ** - what meta-data-customizer(s) should be used? + ** + ** When implementing a QBit, you'll implement this interface - adding whatever + ** (if any) properties you need, and if you have any rules, then overriding + ** the validate method (ideally the one that takes the List-of-String errors) + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** and pass it through to the QBit producer. + *******************************************************************************/ +public interface QBitConfig extends Serializable +{ + QLogger LOG = QLogger.getLogger(QBitConfig.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance) throws QBitConfigValidationException + { + List errors = new ArrayList<>(); + + try + { + validate(qInstance, errors); + } + catch(Exception e) + { + LOG.warn("Error validating QBitConfig: " + this.getClass().getName(), e); + } + + if(!errors.isEmpty()) + { + throw (new QBitConfigValidationException(this, errors)); + } + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default void validate(QInstance qInstance, List errors) + { + ///////////////////////////////////// + // nothing to validate by default! // + ///////////////////////////////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default boolean assertCondition(boolean condition, String message, List errors) + { + if(!condition) + { + errors.add(message); + } + return (condition); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (null); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java new file mode 100644 index 00000000..058442c5 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitConfigValidationException.java @@ -0,0 +1,44 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** thrown by QBitConfig.validate() if there's an issue. + *******************************************************************************/ +public class QBitConfigValidationException extends QException +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public QBitConfigValidationException(QBitConfig qBitConfig, List errors) + { + super("Validation failed for QBitConfig: " + qBitConfig.getClass().getName() + ":\n" + StringUtils.join("\n", errors)); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java new file mode 100644 index 00000000..feacb969 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitMetaData.java @@ -0,0 +1,237 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Meta-data to define an active QBit in a QQQ Instance. + ** + ** The unique "name" for the QBit is composed of its groupId and artifactId + ** (maven style). There is also a version - but it is not part of the unique + ** name. But - there is also a namespace attribute, which IS part of the + ** unique name. This will (eventually?) allow us to have multiple instances + ** of the same qbit in a qInstance at the same time (e.g., 2 versions of some + ** table, which should be namespace-prefixed); + ** + ** QBitMetaData also retains the QBitConfig that was used to produce the QBit. + ** + ** Some meta-data objects are aware of the fact that they may have come from a + ** QBit - see SourceQBitAware interface. These objects can get their source + ** QBitMetaData (this object) and its config,via that interface. + *******************************************************************************/ +public class QBitMetaData implements TopLevelMetaDataInterface +{ + private String groupId; + private String artifactId; + private String version; + private String namespace; + + private QBitConfig config; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getName() + { + String name = groupId + ":" + artifactId; + if(StringUtils.hasContent(namespace)) + { + name += ":" + namespace; + } + return name; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void addSelfToInstance(QInstance qInstance) + { + qInstance.addQBit(this); + } + + + + /******************************************************************************* + ** Getter for config + *******************************************************************************/ + public QBitConfig getConfig() + { + return (this.config); + } + + + + /******************************************************************************* + ** Setter for config + *******************************************************************************/ + public void setConfig(QBitConfig config) + { + this.config = config; + } + + + + /******************************************************************************* + ** Fluent setter for config + *******************************************************************************/ + public QBitMetaData withConfig(QBitConfig config) + { + this.config = config; + return (this); + } + + + + /******************************************************************************* + ** Getter for groupId + *******************************************************************************/ + public String getGroupId() + { + return (this.groupId); + } + + + + /******************************************************************************* + ** Setter for groupId + *******************************************************************************/ + public void setGroupId(String groupId) + { + this.groupId = groupId; + } + + + + /******************************************************************************* + ** Fluent setter for groupId + *******************************************************************************/ + public QBitMetaData withGroupId(String groupId) + { + this.groupId = groupId; + return (this); + } + + + + /******************************************************************************* + ** Getter for artifactId + *******************************************************************************/ + public String getArtifactId() + { + return (this.artifactId); + } + + + + /******************************************************************************* + ** Setter for artifactId + *******************************************************************************/ + public void setArtifactId(String artifactId) + { + this.artifactId = artifactId; + } + + + + /******************************************************************************* + ** Fluent setter for artifactId + *******************************************************************************/ + public QBitMetaData withArtifactId(String artifactId) + { + this.artifactId = artifactId; + return (this); + } + + + + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public QBitMetaData withVersion(String version) + { + this.version = version; + return (this); + } + + + + /******************************************************************************* + ** Getter for namespace + *******************************************************************************/ + public String getNamespace() + { + return (this.namespace); + } + + + + /******************************************************************************* + ** Setter for namespace + *******************************************************************************/ + public void setNamespace(String namespace) + { + this.namespace = namespace; + } + + + + /******************************************************************************* + ** Fluent setter for namespace + *******************************************************************************/ + public QBitMetaData withNamespace(String namespace) + { + this.namespace = namespace; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java new file mode 100644 index 00000000..7fef2174 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducer.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** interface for how a QBit's meta-data gets produced and added to a QInstance. + ** + ** When implementing a QBit, you'll implement this interface: + ** - adding a QBitConfig subclass as a property + ** - overriding the produce(qInstance, namespace) method - where you'll: + ** -- create and add your QBitMetaData + ** -- call MetaDataProducerHelper.findProducers + ** -- hand off to finishProducing() in this interface + ** + ** When using a QBit, you'll create an instance of the QBit's config object, + ** pass it in to the producer, then call produce, ala: + ** + ** new SomeQBitProducer() + ** .withQBitConfig(someQBitConfig) + ** .produce(qInstance); + ** + *******************************************************************************/ +public interface QBitProducer +{ + QLogger LOG = QLogger.getLogger(QBitProducer.class); + + + /*************************************************************************** + ** + ***************************************************************************/ + default void produce(QInstance qInstance) throws QException + { + produce(qInstance, null); + } + + /*************************************************************************** + ** + ***************************************************************************/ + void produce(QInstance qInstance, String namespace) throws QException; + + + /*************************************************************************** + * + ***************************************************************************/ + default void finishProducing(QInstance qInstance, QBitMetaData qBitMetaData, C qBitConfig, List> producers) throws QException + { + qBitConfig.validate(qInstance); + + /////////////////////////////// + // todo - move to base class // + /////////////////////////////// + for(MetaDataProducerInterface producer : producers) + { + if(producer instanceof QBitComponentMetaDataProducer) + { + QBitComponentMetaDataProducer qBitComponentMetaDataProducer = (QBitComponentMetaDataProducer) producer; + qBitComponentMetaDataProducer.setQBitConfig(qBitConfig); + } + + if(!producer.isEnabled()) + { + LOG.debug("Not using producer which is not enabled", logPair("producer", producer.getClass().getSimpleName())); + continue; + } + + MetaDataProducerOutput output = producer.produce(qInstance); + + ///////////////////////////////////////// + // apply table customizer, if provided // + ///////////////////////////////////////// + if(qBitConfig.getTableMetaDataCustomizer() != null && output instanceof QTableMetaData table) + { + output = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table); + } + + ///////////////////////////////////////////////// + // set source qbit, if output is aware of such // + ///////////////////////////////////////////////// + if(output instanceof SourceQBitAware sourceQBitAware) + { + sourceQBitAware.setSourceQBitName(qBitMetaData.getName()); + } + + output.addSelfToInstance(qInstance); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java new file mode 100644 index 00000000..c4d4be7f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/SourceQBitAware.java @@ -0,0 +1,77 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import com.kingsrook.qqq.backend.core.context.QContext; + + +/******************************************************************************* + ** interface for meta data objects that may have come from a qbit, and where we + ** might want to get data about that qbit (e.g., config or meta-data). + *******************************************************************************/ +public interface SourceQBitAware +{ + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + String getSourceQBitName(); + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + void setSourceQBitName(String sourceQBitName); + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + Object withSourceQBitName(String sourceQBitName); + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitMetaData getSourceQBit() + { + String qbitName = getSourceQBitName(); + return (QContext.getQInstance().getQBits().get(qbitName)); + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default QBitConfig getSourceQBitConfig() + { + QBitMetaData sourceQBit = getSourceQBit(); + if(sourceQBit == null) + { + return null; + } + else + { + return sourceQBit.getConfig(); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java index 0d0d6152..b2318de5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableMetaData.java @@ -50,6 +50,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware; import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.automation.QTableAutomationDetails; @@ -62,7 +63,7 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; ** Meta-Data to define a table in a QQQ instance. ** *******************************************************************************/ -public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface +public class QTableMetaData implements QAppChildMetaData, Serializable, MetaDataWithPermissionRules, TopLevelMetaDataInterface, SourceQBitAware { private static final QLogger LOG = QLogger.getLogger(QTableMetaData.class); @@ -73,6 +74,8 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData private String primaryKeyField; private boolean isHidden = false; + private String sourceQBitName; + private Map fields; private List uniqueKeys; private List associations; @@ -1554,4 +1557,38 @@ public class QTableMetaData implements QAppChildMetaData, Serializable, MetaData QInstanceHelpContentManager.removeHelpContentByRoleSetFromList(roles, listForSlot); } + + /******************************************************************************* + ** Getter for sourceQBitName + *******************************************************************************/ + @Override + public String getSourceQBitName() + { + return (this.sourceQBitName); + } + + + + /******************************************************************************* + ** Setter for sourceQBitName + *******************************************************************************/ + @Override + public void setSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + } + + + + /******************************************************************************* + ** Fluent setter for sourceQBitName + *******************************************************************************/ + @Override + public QTableMetaData withSourceQBitName(String sourceQBitName) + { + this.sourceQBitName = sourceQBitName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java new file mode 100644 index 00000000..de56fbfc --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/QBitProducerTest.java @@ -0,0 +1,151 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits; + + +import java.util.LinkedHashMap; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.OtherTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata.SomeTableMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for QBitProducer + *******************************************************************************/ +class QBitProducerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed(TestUtils.MEMORY_BACKEND_NAME)) + .withIsSomeTableEnabled(true) + .withSomeSetting("yes") + .withTableMetaDataCustomizer((i, table) -> + { + if(table.getBackendName() == null) + { + table.setBackendName(TestUtils.DEFAULT_BACKEND_NAME); + } + + table.addField(new QFieldMetaData("custom", QFieldType.STRING)); + + return (table); + }); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // OtherTable should have been provided by the qbit, with the backend name we told it above (MEMORY) // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNotNull(otherTable); + assertEquals(TestUtils.MEMORY_BACKEND_NAME, otherTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + QBitMetaData sourceQBit = otherTable.getSourceQBit(); + assertEquals("testQBit", sourceQBit.getArtifactId()); + + //////////////////////////////////////////////////////////////////////////////// + // SomeTable should have been provided, w/ backend name set by the customizer // + //////////////////////////////////////////////////////////////////////////////// + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNotNull(someTable); + assertEquals(TestUtils.DEFAULT_BACKEND_NAME, someTable.getBackendName()); + assertNotNull(otherTable.getField("custom")); + + TestQBitConfig qBitConfig = (TestQBitConfig) someTable.getSourceQBitConfig(); + assertEquals("yes", qBitConfig.getSomeSetting()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDisableThings() throws QException + { + TestQBitConfig config = new TestQBitConfig() + .withOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)) + .withIsSomeTableEnabled(false); + + QInstance qInstance = QContext.getQInstance(); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + + ////////////////////////////////////// + // neither table should be produced // + ////////////////////////////////////// + QTableMetaData otherTable = qInstance.getTable(OtherTableMetaDataProducer.NAME); + assertNull(otherTable); + + QTableMetaData someTable = qInstance.getTable(SomeTableMetaDataProducer.NAME); + assertNull(someTable); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidationErrors() throws QException + { + QInstance qInstance = QContext.getQInstance(); + TestQBitConfig config = new TestQBitConfig(); + + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set") + .hasMessageContaining("isSomeTableEnabled must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setIsSomeTableEnabled(true); + assertThatThrownBy(() -> new TestQBitProducer().withTestQBitConfig(config).produce(qInstance)) + .isInstanceOf(QBitConfigValidationException.class) + .hasMessageContaining("otherTableConfig must be set"); + qInstance.setQBits(new LinkedHashMap<>()); + + config.setOtherTableConfig(ProvidedOrSuppliedTableConfig.useSuppliedTaleNamed(TestUtils.TABLE_NAME_PERSON_MEMORY)); + new TestQBitProducer().withTestQBitConfig(config).produce(qInstance); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java new file mode 100644 index 00000000..05ac39a9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitConfig.java @@ -0,0 +1,181 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitConfig implements QBitConfig +{ + private MetaDataCustomizerInterface tableMetaDataCustomizer; + + private Boolean isSomeTableEnabled; + private ProvidedOrSuppliedTableConfig otherTableConfig; + + private String someSetting; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QInstance qInstance, List errors) + { + assertCondition(otherTableConfig != null, "otherTableConfig must be set", errors); + assertCondition(isSomeTableEnabled != null, "isSomeTableEnabled must be set", errors); + } + + + + /******************************************************************************* + ** Getter for otherTableConfig + *******************************************************************************/ + public ProvidedOrSuppliedTableConfig getOtherTableConfig() + { + return (this.otherTableConfig); + } + + + + /******************************************************************************* + ** Setter for otherTableConfig + *******************************************************************************/ + public void setOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + } + + + + /******************************************************************************* + ** Fluent setter for otherTableConfig + *******************************************************************************/ + public TestQBitConfig withOtherTableConfig(ProvidedOrSuppliedTableConfig otherTableConfig) + { + this.otherTableConfig = otherTableConfig; + return (this); + } + + + + /******************************************************************************* + ** Getter for isSomeTableEnabled + *******************************************************************************/ + public Boolean getIsSomeTableEnabled() + { + return (this.isSomeTableEnabled); + } + + + + /******************************************************************************* + ** Setter for isSomeTableEnabled + *******************************************************************************/ + public void setIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + } + + + + /******************************************************************************* + ** Fluent setter for isSomeTableEnabled + *******************************************************************************/ + public TestQBitConfig withIsSomeTableEnabled(Boolean isSomeTableEnabled) + { + this.isSomeTableEnabled = isSomeTableEnabled; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableMetaDataCustomizer + *******************************************************************************/ + public MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (this.tableMetaDataCustomizer); + } + + + + /******************************************************************************* + ** Setter for tableMetaDataCustomizer + *******************************************************************************/ + public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + + + + /******************************************************************************* + ** Fluent setter for tableMetaDataCustomizer + *******************************************************************************/ + public TestQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + return (this); + } + + + + /******************************************************************************* + ** Getter for someSetting + *******************************************************************************/ + public String getSomeSetting() + { + return (this.someSetting); + } + + + + /******************************************************************************* + ** Setter for someSetting + *******************************************************************************/ + public void setSomeSetting(String someSetting) + { + this.someSetting = someSetting; + } + + + + /******************************************************************************* + ** Fluent setter for someSetting + *******************************************************************************/ + public TestQBitConfig withSomeSetting(String someSetting) + { + this.someSetting = someSetting; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java new file mode 100644 index 00000000..14c1d5d1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/TestQBitProducer.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestQBitProducer implements QBitProducer +{ + private TestQBitConfig testQBitConfig; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void produce(QInstance qInstance, String namespace) throws QException + { + QBitMetaData qBitMetaData = new QBitMetaData() + .withGroupId("test.com.kingsrook.qbits") + .withArtifactId("testQBit") + .withVersion("0.1.0") + .withNamespace(namespace) + .withConfig(testQBitConfig); + qInstance.addQBit(qBitMetaData); + + List> producers = MetaDataProducerHelper.findProducers(getClass().getPackageName() + ".metadata"); + finishProducing(qInstance, qBitMetaData, testQBitConfig, producers); + } + + + + /******************************************************************************* + ** Getter for testQBitConfig + *******************************************************************************/ + public TestQBitConfig getTestQBitConfig() + { + return (this.testQBitConfig); + } + + + + /******************************************************************************* + ** Setter for testQBitConfig + *******************************************************************************/ + public void setTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + } + + + + /******************************************************************************* + ** Fluent setter for testQBitConfig + *******************************************************************************/ + public TestQBitProducer withTestQBitConfig(TestQBitConfig testQBitConfig) + { + this.testQBitConfig = testQBitConfig; + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java new file mode 100644 index 00000000..7660758c --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/OtherTableMetaDataProducer.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for OtherTable + *******************************************************************************/ +public class OtherTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "otherTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getOtherTableConfig().getDoProvideTable()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withBackendName(getQBitConfig().getOtherTableConfig().getBackendName()) + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java new file mode 100644 index 00000000..3ef79554 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/qbits/testqbit/metadata/SomeTableMetaDataProducer.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.metadata; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.testqbit.TestQBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Meta Data Producer for SomeTable + *******************************************************************************/ +public class SomeTableMetaDataProducer extends QBitComponentMetaDataProducer +{ + public static final String NAME = "someTable"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (getQBitConfig().getIsSomeTableEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)); + + return (qTableMetaData); + } + +} From 6fe04e65df78aa5356839a40dfa852cf150bdd98 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:10 -0600 Subject: [PATCH 173/209] Add getValueFromRecordOrOldRecord --- .../RecordCustomizerUtilityInterface.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java index fa91e9ab..53947578 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/customizers/RecordCustomizerUtilityInterface.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; @@ -160,4 +161,18 @@ public interface RecordCustomizerUtilityInterface return (oldRecordMap); } + + /*************************************************************************** + ** + ***************************************************************************/ + static T getValueFromRecordOrOldRecord(String fieldName, QRecord record, Serializable primaryKey, Optional> oldRecordMap) + { + T value = (T) record.getValue(fieldName); + if(value == null && primaryKey != null && oldRecordMap.isPresent() && oldRecordMap.get().containsKey(primaryKey)) + { + value = (T) oldRecordMap.get().get(primaryKey).getValue(fieldName); + } + return value; + } + } From 46a1a77d1b61730cb7146de3cf323a74f510c6b4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:19 -0600 Subject: [PATCH 174/209] Add method getProcess --- .../model/actions/processes/RunBackendStepInput.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index 84da02f7..ca066eb3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface; @@ -617,4 +618,14 @@ public class RunBackendStepInput extends AbstractActionInput } } } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public QProcessMetaData getProcess() + { + return (QContext.getQInstance().getProcess(getProcessName())); + } } From 2b9181b22ec251ba52560809b11e97764f80e701 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:26:54 -0600 Subject: [PATCH 175/209] Remove block that was adding fileName to requestedPath, idk, wasn't good --- .../actions/AbstractBaseFilesystemAction.java | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 30999153..db3d3d05 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -62,7 +62,6 @@ import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.Back import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.ObjectUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.lambdas.UnsafeSupplier; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; @@ -288,21 +287,8 @@ public abstract class AbstractBaseFilesystemAction QueryOutput queryOutput = new QueryOutput(queryInput); - String requestedPath = null; - QQueryFilter filter = queryInput.getFilter(); - if(filter != null && tableDetails.getCardinality().equals(Cardinality.ONE)) - { - if(filter.getCriteria() != null && filter.getCriteria().size() == 1) - { - QFilterCriteria criteria = filter.getCriteria().get(0); - if(tableDetails.getFileNameFieldName().equals(criteria.getFieldName()) && criteria.getOperator().equals(QCriteriaOperator.EQUALS)) - { - requestedPath = ValueUtils.getValueAsString(criteria.getValues().get(0)); - } - } - } - - List files = listFiles(table, queryInput.getBackend(), requestedPath); + String requestedPath = null; + List files = listFiles(table, queryInput.getBackend(), requestedPath); switch(tableDetails.getCardinality()) { From cddc42db5b7c506ae8325d81b758ebc37698d0fb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 21 Feb 2025 16:27:17 -0600 Subject: [PATCH 176/209] add testSimpleQueryForOneFile --- .../sftp/actions/SFTPQueryActionTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java index 092ea987..35f87644 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java @@ -28,6 +28,9 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -58,6 +61,20 @@ class SFTPQueryActionTest extends BaseSFTPTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testSimpleQueryForOneFile() throws QException + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE); + queryInput.setFilter(new QQueryFilter(new QFilterCriteria("fileName", QCriteriaOperator.EQUALS, "testfile-1.txt"))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(1, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); + } + + + /******************************************************************************* ** *******************************************************************************/ From 35c4049174e15d5525dbc1cac2ba3ec39a133ebe Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 10:23:02 -0600 Subject: [PATCH 177/209] Add LinkValues.TO_RECORD_FROM_TABLE_DYNAMIC and FileDownloadValues.DOWNLOAD_URL_DYNAMIC --- .../model/metadata/fields/AdornmentType.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index 29982aad..9fb76f41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -23,8 +23,12 @@ package com.kingsrook.qqq.backend.core.model.metadata.fields; import java.io.Serializable; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Objects; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; @@ -48,14 +52,14 @@ public enum AdornmentType ////////////////////////////////////////////////////////////////////////// - /******************************************************************************* ** *******************************************************************************/ public interface LinkValues { - String TARGET = "target"; - String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TARGET = "target"; + String TO_RECORD_FROM_TABLE = "toRecordFromTable"; + String TO_RECORD_FROM_TABLE_DYNAMIC = "toRecordFromTableDynamic"; } @@ -72,6 +76,8 @@ public enum AdornmentType String SUPPLEMENTAL_PROCESS_NAME = "supplementalProcessName"; String SUPPLEMENTAL_CODE_REFERENCE = "supplementalCodeReference"; + String DOWNLOAD_URL_DYNAMIC = "downloadUrlDynamic"; + //////////////////////////////////////////////////// // use these two together, as in: // // FILE_NAME_FORMAT = "Order %s Packing Slip.pdf" // @@ -79,6 +85,17 @@ public enum AdornmentType //////////////////////////////////////////////////// String FILE_NAME_FORMAT = "fileNameFormat"; String FILE_NAME_FORMAT_FIELDS = "fileNameFormatFields"; + + /*************************************************************************** + ** + ***************************************************************************/ + static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName) + { + return ("/data/" + tableName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8) + "/" + + fieldName + "/" + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + } } From 80c286ab00ae47d859fc7873e6773598db6c56d8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 10:46:39 -0600 Subject: [PATCH 178/209] update setBlobValuesToDownloadUrls to not do that if the field is set to use a downloadUrlDyanmic. --- .../core/actions/values/QValueFormatter.java | 16 +-- .../actions/values/QValueFormatterTest.java | 102 ++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java index 6129b409..876b76ae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java @@ -23,8 +23,6 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -34,7 +32,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -48,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -488,6 +486,8 @@ public class QValueFormatter String fileNameFormat = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT)); String defaultExtension = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION)); + Boolean downloadUrlDynamic = ValueUtils.getValueAsBoolean(adornmentValues.get(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC)); + for(QRecord record : records) { if(!doesFieldHaveValue(field, record)) @@ -495,6 +495,11 @@ public class QValueFormatter continue; } + if(BooleanUtils.isTrue(downloadUrlDynamic)) + { + continue; + } + Serializable primaryKey = record.getValue(table.getPrimaryKeyField()); String fileName = null; @@ -544,10 +549,7 @@ public class QValueFormatter || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_CODE_REFERENCE) || adornmentValues.containsKey(AdornmentType.FileDownloadValues.SUPPLEMENTAL_PROCESS_NAME)) { - record.setValue(field.getName(), "/data/" + table.getName() + "/" - + URLEncoder.encode(ValueUtils.getValueAsString(primaryKey), StandardCharsets.UTF_8) + "/" - + field.getName() + "/" - + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + record.setValue(field.getName(), AdornmentType.FileDownloadValues.makeFieldDownloadUrl(table.getName(), primaryKey, field.getName(), fileName)); } record.setDisplayValue(field.getName(), fileName); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index cd678974..c4eac300 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -30,19 +30,23 @@ import java.time.LocalTime; import java.time.Month; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; import com.kingsrook.qqq.backend.core.model.metadata.fields.DateTimeDisplayValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -237,4 +241,102 @@ class QValueFormatterTest extends BaseTest assertEquals("2024-04-04 02:12:00 PM CDT", record.getDisplayValue("createDate")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBlobValuesToDownloadUrls() + { + byte[] blobBytes = "hello".getBytes(); + { + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT, "blob-%s.txt") + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT_FIELDS, new ArrayList<>(List.of("id"))))); + + ////////////////////////////////////////////////////////////////// + // verify display value gets set to formated file-name + fields // + // and raw value becomes URL for downloading the byte // + ////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/blob-47.txt", record.getValueString("blobField")); + assertEquals("blob-47.txt", record.getDisplayValue("blobField")); + + //////////////////////////////////////////////////////// + // verify that w/ no blob value, we don't do anything // + //////////////////////////////////////////////////////// + QRecord recordWithoutBlobValue = new QRecord().withValue("id", 47); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(recordWithoutBlobValue)); + assertNull(recordWithoutBlobValue.getValue("blobField")); + assertNull(recordWithoutBlobValue.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD) + .withValue(AdornmentType.FileDownloadValues.FILE_NAME_FIELD, "fileName"); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB) + .withFieldAdornment(adornment)); + + //////////////////////////////////////////////////// + // here get the file name directly from one field // + //////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/myBlob.txt", record.getValueString("blobField")); + assertEquals("myBlob.txt", record.getDisplayValue("blobField")); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // switch to use dynamic url, rerun, and assert we get the values as they were on the record before the call // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, true); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes).withValue("fileName", "myBlob.txt") + .withDisplayValue("blobField:" + AdornmentType.FileDownloadValues.DOWNLOAD_URL_DYNAMIC, "/something-custom/") + .withDisplayValue("blobField", "myDisplayValue"); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertArrayEquals(blobBytes, record.getValueByteArray("blobField")); + assertEquals("myDisplayValue", record.getDisplayValue("blobField")); + } + + { + FieldAdornment adornment = new FieldAdornment().withType(AdornmentType.FILE_DOWNLOAD); + + QTableMetaData table = new QTableMetaData() + .withName("testTable") + .withLabel("Test Table") + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("blobField", QFieldType.BLOB).withLabel("Blob").withFieldAdornment(adornment)); + + /////////////////////////////////////////////////////////////////////////////////////////// + // w/o file name format or whatever, generate a file name from table & id & field labels // + /////////////////////////////////////////////////////////////////////////////////////////// + QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField")); + + //////////////////////////////////////// + // add a default extension and re-run // + //////////////////////////////////////// + adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html"); + record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); + QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); + assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob.html", record.getValueString("blobField")); + assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField")); + } + } + } \ No newline at end of file From 77cc2724259b7e9d3822278cb2a74adadda768ac Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 11:07:22 -0600 Subject: [PATCH 179/209] Initial checkin --- .../widgets/RecordListWidgetRenderer.java | 251 ++++++++++++++++++ .../widgets/RecordListWidgetRendererTest.java | 188 +++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java new file mode 100644 index 00000000..ffd6710e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRenderer.java @@ -0,0 +1,251 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceValidatorPluginInterface; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.FilterUseCase; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.AbstractWidgetMetaDataBuilder; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Generic widget to display a list of records. + ** + ** Note, closely related to (and copied from ChildRecordListRenderer. + ** opportunity to share more code with that in the future?? + *******************************************************************************/ +public class RecordListWidgetRenderer extends AbstractWidgetRenderer +{ + private static final QLogger LOG = QLogger.getLogger(RecordListWidgetRenderer.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static Builder widgetMetaDataBuilder(String widgetName) + { + return (new Builder(new QWidgetMetaData() + .withName(widgetName) + .withIsCard(true) + .withCodeReference(new QCodeReference(RecordListWidgetRenderer.class)) + .withType(WidgetType.CHILD_RECORD_LIST.getType()) + .withValidatorPlugin(new RecordListWidgetValidator()) + )); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class Builder extends AbstractWidgetMetaDataBuilder + { + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public Builder(QWidgetMetaData widgetMetaData) + { + super(widgetMetaData); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withLabel(String label) + { + widgetMetaData.setLabel(label); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withMaxRows(Integer maxRows) + { + widgetMetaData.withDefaultValue("maxRows", maxRows); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withTableName(String tableName) + { + widgetMetaData.withDefaultValue("tableName", tableName); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public Builder withFilter(QQueryFilter filter) + { + widgetMetaData.withDefaultValue("filter", filter); + return (this); + } + + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + try + { + Integer maxRows = null; + if(StringUtils.hasContent(input.getQueryParams().get("maxRows"))) + { + maxRows = ValueUtils.getValueAsInteger(input.getQueryParams().get("maxRows")); + } + else if(input.getWidgetMetaData().getDefaultValues().containsKey("maxRows")) + { + maxRows = ValueUtils.getValueAsInteger(input.getWidgetMetaData().getDefaultValues().get("maxRows")); + } + + QQueryFilter filter = ((QQueryFilter) input.getWidgetMetaData().getDefaultValues().get("filter")).clone(); + filter.interpretValues(new HashMap<>(input.getQueryParams()), FilterUseCase.DEFAULT); + filter.setLimit(maxRows); + + String tableName = ValueUtils.getValueAsString(input.getWidgetMetaData().getDefaultValues().get("tableName")); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(tableName); + queryInput.setShouldTranslatePossibleValues(true); + queryInput.setShouldGenerateDisplayValues(true); + queryInput.setFilter(filter); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + + QValueFormatter.setBlobValuesToDownloadUrls(table, queryOutput.getRecords()); + + int totalRows = queryOutput.getRecords().size(); + if(maxRows != null && (queryOutput.getRecords().size() == maxRows)) + { + ///////////////////////////////////////////////////////////////////////////////////// + // if the input said to only do some max, and the # of results we got is that max, // + // then do a count query, for displaying 1-n of // + ///////////////////////////////////////////////////////////////////////////////////// + CountInput countInput = new CountInput(); + countInput.setTableName(tableName); + countInput.setFilter(filter); + totalRows = new CountAction().execute(countInput).getCount(); + } + + String tablePath = QContext.getQInstance().getTablePath(tableName); + String viewAllLink = tablePath == null ? null : (tablePath + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), Charset.defaultCharset())); + + ChildRecordListData widgetData = new ChildRecordListData(input.getQueryParams().get("widgetLabel"), queryOutput, table, tablePath, viewAllLink, totalRows); + + return (new RenderWidgetOutput(widgetData)); + } + catch(Exception e) + { + LOG.warn("Error rendering record list widget", e, logPair("widgetName", () -> input.getWidgetMetaData().getName())); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static class RecordListWidgetValidator implements QInstanceValidatorPluginInterface + { + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void validate(QWidgetMetaDataInterface widgetMetaData, QInstance qInstance, QInstanceValidator qInstanceValidator) + { + String prefix = "Widget " + widgetMetaData.getName() + ": "; + + ////////////////////////////////////////////// + // make sure table name is given and exists // + ////////////////////////////////////////////// + QTableMetaData table = null; + String tableName = ValueUtils.getValueAsString(CollectionUtils.nonNullMap(widgetMetaData.getDefaultValues()).get("tableName")); + if(qInstanceValidator.assertCondition(StringUtils.hasContent(tableName), prefix + "defaultValue for tableName must be given")) + { + //////////////////////////// + // make sure table exists // + //////////////////////////// + table = qInstance.getTable(tableName); + qInstanceValidator.assertCondition(table != null, prefix + "No table named " + tableName + " exists in the instance"); + } + + //////////////////////////////////////////////////////////////////////////////////// + // make sure filter is given and is valid (only check that if table is given too) // + //////////////////////////////////////////////////////////////////////////////////// + QQueryFilter filter = ((QQueryFilter) widgetMetaData.getDefaultValues().get("filter")); + if(qInstanceValidator.assertCondition(filter != null, prefix + "defaultValue for filter must be given") && table != null) + { + qInstanceValidator.validateQueryFilter(qInstance, prefix, table, filter, null); + } + } + + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java new file mode 100644 index 00000000..64246384 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/dashboard/widgets/RecordListWidgetRendererTest.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.actions.dashboard.widgets; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for RecordListWidgetRenderer + *******************************************************************************/ +class RecordListWidgetRendererTest extends BaseTest +{ + + /*************************************************************************** + ** + ***************************************************************************/ + private QWidgetMetaData defineWidget() + { + return RecordListWidgetRenderer.widgetMetaDataBuilder("testRecordListWidget") + .withTableName(TestUtils.TABLE_NAME_SHAPE) + .withMaxRows(20) + .withLabel("Some Shapes") + .withFilter(new QQueryFilter() + .withCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, "${input.maxShapeId}") + .withCriteria("name", QCriteriaOperator.NOT_EQUALS, "Square") + .withOrderBy(new QFilterOrderBy("id", false)) + ).getWidgetMetaData(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidation() throws QInstanceValidationException + { + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + widgetMetaData.getDefaultValues().remove("tableName"); + widgetMetaData.getDefaultValues().remove("filter"); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("defaultValue for filter must be given") + .hasMessageContaining("defaultValue for tableName must be given"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + QQueryFilter filter = (QQueryFilter) widgetMetaData.getDefaultValues().get("filter"); + filter.addCriteria(new QFilterCriteria("noField", QCriteriaOperator.EQUALS, "noValue")); + qInstance.addWidget(widgetMetaData); + + assertThatThrownBy(() -> new QInstanceValidator().validate(qInstance)) + .isInstanceOf(QInstanceValidationException.class) + .hasMessageContaining("Criteria fieldName noField is not a field in this table"); + } + + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + ////////////////////////////////// + // make sure valid setup passes // + ////////////////////////////////// + new QInstanceValidator().validate(qInstance); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRender() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + QWidgetMetaData widgetMetaData = defineWidget(); + qInstance.addWidget(widgetMetaData); + + TestUtils.insertDefaultShapes(qInstance); + TestUtils.insertExtraShapes(qInstance); + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "1")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(1, widgetData.getTotalRows()); + assertEquals(1, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + } + + { + RecordListWidgetRenderer recordListWidgetRenderer = new RecordListWidgetRenderer(); + RenderWidgetInput input = new RenderWidgetInput(); + input.setWidgetMetaData(widgetMetaData); + input.setQueryParams(Map.of("maxShapeId", "4")); + RenderWidgetOutput output = recordListWidgetRenderer.render(input); + + ChildRecordListData widgetData = (ChildRecordListData) output.getWidgetData(); + assertEquals(3, widgetData.getTotalRows()); + + ///////////////////////////////////////////////////////////////////////// + // id=2,name=Square was skipped due to NOT_EQUALS Square in the filter // + // max-shape-id applied we don't get id=5 or 6 // + // and they're ordered as specified in the filter (id desc) // + ///////////////////////////////////////////////////////////////////////// + assertEquals(4, widgetData.getQueryOutput().getRecords().get(0).getValue("id")); + assertEquals("Rectangle", widgetData.getQueryOutput().getRecords().get(0).getValue("name")); + + assertEquals(3, widgetData.getQueryOutput().getRecords().get(1).getValue("id")); + assertEquals("Circle", widgetData.getQueryOutput().getRecords().get(1).getValue("name")); + + assertEquals(1, widgetData.getQueryOutput().getRecords().get(2).getValue("id")); + assertEquals("Triangle", widgetData.getQueryOutput().getRecords().get(2).getValue("name")); + } + } + +} \ No newline at end of file From a0d12eade7389eab69d03e5f7a0ed102cf0df908 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 11:14:56 -0600 Subject: [PATCH 180/209] Make validateQueryFilter public --- .../qqq/backend/core/instances/QInstanceValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index a214ee6b..b5be0c57 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -1905,7 +1905,7 @@ public class QInstanceValidator /******************************************************************************* ** *******************************************************************************/ - private void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) + public void validateQueryFilter(QInstance qInstance, String context, QTableMetaData table, QQueryFilter queryFilter, List queryJoins) { for(QFilterCriteria criterion : CollectionUtils.nonNullList(queryFilter.getCriteria())) { From b984959aa736bdcf4ce245532061e5255869c4fa Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 14:25:30 -0600 Subject: [PATCH 181/209] A little more flexibility in filter validation, for context w/o a joinContext --- .../core/instances/QInstanceValidator.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index b5be0c57..b123858e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -142,6 +142,8 @@ public class QInstanceValidator private static ListingHash, QInstanceValidatorPluginInterface> validatorPlugins = new ListingHash<>(); + private JoinGraph joinGraph = null; + private List errors = new ArrayList<>(); @@ -169,8 +171,7 @@ public class QInstanceValidator // the enricher will build a join graph (if there are any joins). we'd like to only do that // // once, during the enrichment/validation work, so, capture it, and store it back in the instance. // ///////////////////////////////////////////////////////////////////////////////////////////////////// - JoinGraph joinGraph = null; - long start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); try { ///////////////////////////////////////////////////////////////////////////////////////////////// @@ -179,7 +180,7 @@ public class QInstanceValidator // TODO - possible point of customization (use a different enricher, or none, or pass it options). QInstanceEnricher qInstanceEnricher = new QInstanceEnricher(qInstance); qInstanceEnricher.enrich(); - joinGraph = qInstanceEnricher.getJoinGraph(); + this.joinGraph = qInstanceEnricher.getJoinGraph(); } catch(Exception e) { @@ -1949,7 +1950,8 @@ public class QInstanceValidator { if(fieldName.contains(".")) { - String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String fieldNameAfterDot = fieldName.substring(fieldName.lastIndexOf(".") + 1); + String tableNameBeforeDot = fieldName.substring(0, fieldName.lastIndexOf(".")); if(CollectionUtils.nullSafeHasContents(queryJoins)) { @@ -1973,11 +1975,32 @@ public class QInstanceValidator } else { - errors.add("QInstanceValidator does not yet support finding a field that looks like a join field, but isn't associated with a query."); - return (true); - // todo! for(QJoinMetaData join : CollectionUtils.nonNullMap(qInstance.getJoins()).values()) - // { - // } + if(this.joinGraph != null) + { + Set joinConnections = joinGraph.getJoinConnections(table.getName()); + for(JoinGraph.JoinConnectionList joinConnectionList : joinConnections) + { + JoinGraph.JoinConnection joinConnection = joinConnectionList.list().get(joinConnectionList.list().size() - 1); + if(tableNameBeforeDot.equals(joinConnection.joinTable())) + { + QTableMetaData joinTable = qInstance.getTable(tableNameBeforeDot); + if(joinTable.getFields().containsKey(fieldNameAfterDot)) + { + ///////////////////////// + // mmm, looks valid... // + ///////////////////////// + return (true); + } + } + } + } + + ////////////////////////////////////////////////////////////////////////////////////// + // todo - not sure how vulnerable we are to ongoing issues here... // + // idea: let a filter (or any object?) be opted out of validation, some version of // + // a static map of objects we can check at the top of various validate methods... // + ////////////////////////////////////////////////////////////////////////////////////// + errors.add("Failed to find field named: " + fieldName); } } } From 21c4434831646be3c17c6b6b8e86c6d40dfb68e8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 19:57:07 -0600 Subject: [PATCH 182/209] Add support for public-key based authentication --- .../sftp/actions/AbstractSFTPAction.java | 52 +++++++++++++++---- .../actions/SFTPTestConnectionAction.java | 36 ++++++++++++- .../model/metadata/SFTPBackendMetaData.java | 32 ++++++++++++ .../metadata/SFTPBackendVariantSetting.java | 3 +- .../module/filesystem/sftp/BaseSFTPTest.java | 11 ++++ .../actions/SFTPTestConnectionActionTest.java | 33 ++++++++++++ .../src/test/resources/README.md | 11 ++++ .../src/test/resources/test-only-key | 52 +++++++++++++++++++ .../src/test/resources/test-only-key.pub | 1 + 9 files changed, 219 insertions(+), 12 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/test/resources/README.md create mode 100644 qqq-backend-module-filesystem/src/test/resources/test-only-key create mode 100644 qqq-backend-module-filesystem/src/test/resources/test-only-key.pub diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 1ef137af..64a77632 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -26,6 +26,11 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -48,6 +53,7 @@ import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBacke import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPBackendVariantSetting; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.session.ClientSession; +import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.sftp.client.SftpClient; import org.apache.sshd.sftp.client.SftpClientFactory; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -60,6 +66,8 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction 0) + { + PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(privateKeyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + PrivateKey privateKey = keyFactory.generatePrivate(keySpec); + PublicKey publicKey = KeyUtils.recoverPublicKey(privateKey); + clientSession.addPublicKeyIdentity(new KeyPair(publicKey, privateKey)); + } + + ////////////////////////////////////////////////// + // if we have a password, add password identity // + ////////////////////////////////////////////////// + if(StringUtils.hasContent(password)) + { + clientSession.addPasswordIdentity(password); + } + clientSession.auth().verify(); this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java index 53b93ccf..d31fbcbe 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionAction.java @@ -37,7 +37,7 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction ***************************************************************************/ public SFTPTestConnectionTestOutput testConnection(SFTPTestConnectionTestInput input) { - try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword())) + try(SftpClient sftpClient = super.makeConnection(input.getUsername(), input.getHostName(), input.getPort(), input.getPassword(), input.getPrivateKey())) { SFTPTestConnectionTestOutput output = new SFTPTestConnectionTestOutput().withIsConnectionSuccess(true); @@ -80,6 +80,7 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction private Integer port; private String password; private String basePath; + private byte[] privateKey; @@ -251,6 +252,39 @@ public class SFTPTestConnectionAction extends AbstractSFTPAction return (this); } + + + /******************************************************************************* + ** Getter for privateKey + ** + *******************************************************************************/ + public byte[] getPrivateKey() + { + return privateKey; + } + + + + /******************************************************************************* + ** Setter for privateKey + ** + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + ** + *******************************************************************************/ + public SFTPTestConnectionTestInput withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java index c423f99e..28ed667f 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendMetaData.java @@ -34,6 +34,7 @@ public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData private String username; private String password; private String hostName; + private byte[] privateKey; private Integer port; @@ -195,4 +196,35 @@ public class SFTPBackendMetaData extends AbstractFilesystemBackendMetaData return (this); } + + + /******************************************************************************* + ** Getter for privateKey + *******************************************************************************/ + public byte[] getPrivateKey() + { + return (this.privateKey); + } + + + + /******************************************************************************* + ** Setter for privateKey + *******************************************************************************/ + public void setPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + } + + + + /******************************************************************************* + ** Fluent setter for privateKey + *******************************************************************************/ + public SFTPBackendMetaData withPrivateKey(byte[] privateKey) + { + this.privateKey = privateKey; + return (this); + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java index b5205cb9..2a702fc8 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/model/metadata/SFTPBackendVariantSetting.java @@ -34,5 +34,6 @@ public enum SFTPBackendVariantSetting implements BackendVariantSetting PASSWORD, HOSTNAME, PORT, - BASE_PATH + BASE_PATH, + PRIVATE_KEY } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java index 6c71ea42..e7093402 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java @@ -67,6 +67,17 @@ public class BaseSFTPTest extends BaseTest grantUploadFilesDirWritePermission(); + /////////////////////////////////////////////////// + // add our test-only public key to the container // + /////////////////////////////////////////////////// + String sshDir = "/home/" + USERNAME + "/.ssh"; + sftpContainer.execInContainer("mkdir", sshDir); + sftpContainer.execInContainer("chmod", "700", sshDir); + sftpContainer.execInContainer("chown", USERNAME, sshDir); + copyFileToContainer("test-only-key.pub", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chmod", "600", sshDir + "/authorized_keys"); + sftpContainer.execInContainer("chown", USERNAME, sshDir + "/authorized_keys"); + currentPort = sftpContainer.getMappedPort(22); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java index b1439cbf..5b023735 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPTestConnectionActionTest.java @@ -22,7 +22,12 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.module.filesystem.sftp.BaseSFTPTest; +import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -157,6 +162,7 @@ class SFTPTestConnectionActionTest extends BaseSFTPTest } + /******************************************************************************* ** *******************************************************************************/ @@ -175,4 +181,31 @@ class SFTPTestConnectionActionTest extends BaseSFTPTest assertNull(output.getListBasePathErrorMessage()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testConnectViaPublicKey() throws Exception + { + try(InputStream resourceAsStream = getClass().getResourceAsStream("/test-only-key")) + { + String pem = IOUtils.readLines(resourceAsStream, StandardCharsets.UTF_8).stream() + .filter(s -> !s.startsWith("----")) + .collect(Collectors.joining("")); + + byte[] privateKeyBytes = Base64.getDecoder().decode(pem); + + SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() + .withUsername(BaseSFTPTest.USERNAME) + .withPrivateKey(privateKeyBytes) + .withPort(BaseSFTPTest.getCurrentPort()) + .withHostName(BaseSFTPTest.HOST_NAME); + SFTPTestConnectionAction.SFTPTestConnectionTestOutput output = new SFTPTestConnectionAction().testConnection(input); + assertTrue(output.getIsConnectionSuccess()); + assertNull(output.getConnectionErrorMessage()); + } + } + } \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/README.md b/qqq-backend-module-filesystem/src/test/resources/README.md new file mode 100644 index 00000000..abaabf7a --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/README.md @@ -0,0 +1,11 @@ +The `test-only-key` / `test-only-key.pub` key pair in this directory was generated via: + +```shell +ssh-keygen -t rsa -b 4096 -m PEM -f test-only-key +openssl pkcs8 -topk8 -inform PEM -in test-only-key -outform PEM -nocrypt -out test-only-key-kpcs8.pem +cp test-only-key-kpcs8.pem .../src/test/resources/test-only-key +``` + +It is NOT meant to be used as a secure key in ANY environment. + +It is included in this repo ONLY to be used for basic unit testing. \ No newline at end of file diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key b/qqq-backend-module-filesystem/src/test/resources/test-only-key new file mode 100644 index 00000000..c3ca2a4d --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDGEYxHZMPM6A+4 +OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+X +Se+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDR +yrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8O +qXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0 +hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnic +t3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWya +WzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepON +sVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6k +QNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9 +SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0g +VM5q55bFgxUp32G+HKUT8TI98ZrbEwIDAQABAoICAQC1f2MCEO3zGDs/YHtYmimo +Er917+W3tY7jJljZHAbCniFvAxt4kAdYhCxjNrzwumIqS8W/vgPCHlDCu3eleVV4 +umgIZoL8l3akVJN/t2AtEbroPMdNoZN9sYRkMHRqW6B9bXTdBoiHTnKLS6kWzRoZ +uqr2Ft0YkwcQIyT3VwFTSXwRVI1At8nkal6gd5mYEqU8OC7eoaG+Ued9cftDNtTB +8csGaKGwneTG7fay/t56bM6HFrbJn11YLFbFYEGMq49/+tlSG5sIeCP5tFOjCFd3 +iMS61ndmpfVibZJ1Wnjz5WeZNUN91Za2lhvhxEA++dtsXmnKhktoJkgTo21fj4Ry +1DbdysR5W0vz+HQ5usT/fWAypKj4KSlP6rsL/zkqmlxW1qE/nwzK6UN6kVWvO1Sm +6EcZIQKTRpkGFXluOJL/Bkqg+4Ayl9G3veLON5yTOB367m1haI/oIgr+jOKrkzdG +PWWNZ78USCT9YD/P2vHkwVQ/uzF3kHhLL6n8hzunFpJk+5eScXsyXTUjDCn4HurE +A8Z1FHLOeu4EEKODEdg1PCa34/8Z0K+88G5G/sTDWLf2MjywdCSDzKxYiSpQW0gt +5I5WgQG8CiGHbysBWMnqjoG1J+QpAUDzT29CuLhVbn4i0uiTe0AZ0xLjravhd5dW +dlItzvOY1rj5jHPs5hiWkQKCAQEA97jAuCzPw2Ats2QUfpnqEd/ITKRYTtK6zo+c +sdc1JH0RnzxCeBr2BApSrfMbqAMrPaiUKk5FUqpF5q47AXGrYFlaAPe7cq+Lr97N +LRi7NHM9RdF1myollCghwgKkq8qUe64eNAypMtGOdjrMh7kC879udESYXLjqgT5H +wQbHF5IRvg5vBkVKxzLrg37lX1f0MVGdB8VxA+QRQ4egFuQApPvMLFVenYZL165r +u3OIpOQWcJ91L5VzN5jJMR1x8VWFR6iD4PxachdmD8qaNxfwWKTTWxkpwTXfW1EI +68NZ2s9RmuRbJEfOtiznfzdVL+lAibeMe7dvUMzIevr0hPzMNwKCAQEAzLADz691 +9bJniRVFUkHlbXRV7k0sRxAXNzGmar8xpnG3wJaW+zSsj8Nnr+aSBGgGWmmldEn7 +tiHlozrjWUgUEsRJKObiCrdGsTC7+0flQLs1bukLfgZSnP3oMDYAgq15Vaw49oCU +M4KxRGfkPEhwP/DZClHsYkPr6HegT2/21z8AFHTAxknGjWWGJJAFxwaog7Akugdy +gXLb7lU4SjCJdb5tR1c7aDEUOvDDu1iffhWtt5Tp9BL0dKlN4M+6XZvqSNiVlN6P +BB5gDuSa0qEewIbMWiT4rcvE7gCSXFEWPnGbtHU7QcI4Wx1F75Y4CRgs2rRlnj9j +bVAsRNIOTqkSBQKCAQEAliEL+xJ9X6TcTYnrucZBy09aLsizFCI2QJVcm5MXi+OY +WG7Gwc9lJZG0BeP98Nbqz9Vo5jLFZJH5BxK0g+2FtUCxgUCiA6FMAOwAYMJKQkFM +8xE8OytR1vZzbwb3EX4WetZNS7IYoMnLku+ToPWJSnvLzv77b8ZJqMY76knXQvut +cQeCVcSMyyia/vhavmupfHI/vsPz+C2yIMEDTpwjn9lSJdQfIUyQjkgQ1mvwdi4d +Q2gANzRVvW4FEJUNxvrTaVhBhIqrrdVsb0mUKKuDZ9WMmfsoCQZDNS5pP6kGvctD +Y6HdcqFqL5ILQlggcobkLBJnO1syRT+2iIGqyyYCBQKCAQAbwy/xJnJQbe8/F6R8 +YLW2n9Xb6Zm81cDgWpqg1eftFHWA6Kv3zJAvO6i/of1iHZ3m+3dWi4ZZkMVt21nk +zTLzzK3Dn3U/UNaEyABnN7wviHTZ40AMyty/sGyixWBSWScg6KgdPxla1zol9hVt +28Fl2swFa1EtjtrbgAY9YAlR7pibLa7L9ku49/E22lX+RbfrjKOem837ItITxHlL +DsRGNRrrVziWjDmbOPbDXWTcnCIgyVDmKv//JsuKV4KGmdQwJzg6pekt/NS4kGcz +dGkQYfgrreIQ6JeAVJGFdfYXaB9fXZs48xfju9e1hGF7Uk0bKOazjRN2Sy6F8xu/ +rYzlAoIBAEjY7u3Jmntn1AYsbuy9wTblKl1IaZP4ST+X0/dLtvW8ZLsx0jPGwMXx +xmOku5OGqPjCn5i8Ws2KS8O6O+7lGm/CHXvmDpozD3wpjnJ64SgoLnjrT8R78TEJ +UjsGQfR7ofSj4heR7TgEPp+n0SXse3qERd6VZ5YPuzGva1iVJogErwI58QU2QaxQ +0ONV6F8oZuXjUs9KRhXQ8W0i87m0P7/ZumhqPaQqY/MeAYF/ED5C6ETKISxaDqs/ +zd/jf6uPZL6P4DPWcw7cSk5/aNZZ0P+/BkEX33WHBDSdVyHC+ydMcYZBrrlWKoSt +sNTITZbKrQB4hwHdawpMHxh+5mRXLk0= +-----END PRIVATE KEY----- diff --git a/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub new file mode 100644 index 00000000..02a25a74 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/resources/test-only-key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDGEYxHZMPM6A+4OtayotUrB94AqYow1PcK5HzY8psKuHYejurugwFBbrFHq+dOso6EkLKcSpjvEX+XSe+m3CDIC6qVuLBS6fmWe6wo73fhlqyEP6rslz//v5+Di8fhCPSfa8y4X5RLbkDRyrL/ilmIEHUcmCU82SFFp9C10W+Ir9SZbjAoiRHT+X/WWmXYnMrfLHl38xxdWr8OqXEmLaVxmSaLnWAmMXRr4Nqrk7r7VVq899CZSJMVLSfqeiJPqyLkwOPcYjzOKft0hYAhIXCUsEK4Wv7R1t9SooC7nqE2mY/htBcrPPyDhTT4Kl0zAD2MNloNUUQsTnict3FCUy0pB9zr26bZ4YHBIvdReTWcx6IlH2U28V9ktxWRJe/ESqay4RTfif24aWyaWzD9+yTowTdPJbMaQZI5XGYJBfFtXr8aC8Di+VuILbriiuLk1iPuj1p2R3JXepONsVRDBm10SmHdETcsZXB3glqqiV+mFGmHguPXSo1WZmP79+kj/phwtn2wFKRLwi6kQNeqOxLbamRtj7y4M2sphEC7Cih1w4J+CzjZE8PC7+Bck+ldvmdtW6+wkkhLgli9SvDwe2pM8eTWTznzs1Qj9My5p1v0uvIo0xADtpITzb3kQtl9+2C7uqscjJLIpc0gVM5q55bFgxUp32G+HKUT8TI98ZrbEw== test-only-key From cdc6df214048c71badca9d6eb59835a54eb9ad67 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 24 Feb 2025 20:10:26 -0600 Subject: [PATCH 183/209] Removing call to remove all writeCapabilities from RenderedReport table... not entirely clear that's wanted anyway, and it's a change in behavior now, since this overload of withoutCapabilities was fixed... --- .../core/model/savedreports/SavedReportsMetaDataProvider.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java index 2b40bd86..13084a42 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/SavedReportsMetaDataProvider.java @@ -50,7 +50,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -351,8 +350,7 @@ public class SavedReportsMetaDataProvider .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedReportId", "renderedReportStatusId"))) .withSection(new QFieldSection("input", new QIcon().withName("input"), Tier.T2, List.of("userId", "reportFormat"))) .withSection(new QFieldSection("output", new QIcon().withName("output"), Tier.T2, List.of("jobUuid", "resultPath", "rowCount", "errorMessage", "startTime", "endTime"))) - .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))) - .withoutCapabilities(Capability.allWriteCapabilities()); + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); table.getField("renderedReportStatusId").setAdornments(List.of(new FieldAdornment(AdornmentType.CHIP) .withValues(AdornmentType.ChipValues.iconAndColorValues(RenderedReportStatus.RUNNING.getId(), "pending", AdornmentType.ChipValues.COLOR_SECONDARY)) From eae24e3ebaac17ae55b2b05e7e28cfa9828eaa03 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 25 Feb 2025 08:45:48 -0600 Subject: [PATCH 184/209] Add method pemStringToDecodedBytes --- .../sftp/actions/AbstractSFTPAction.java | 16 ++++++++++++++++ .../actions/SFTPTestConnectionActionTest.java | 9 ++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 64a77632..f98582e8 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -33,6 +33,7 @@ import java.security.PublicKey; import java.security.spec.PKCS8EncodedKeySpec; import java.time.Instant; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -409,4 +410,19 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction !s.startsWith("----")) - .collect(Collectors.joining("")); - - byte[] privateKeyBytes = Base64.getDecoder().decode(pem); + byte[] privateKeyBytes = AbstractSFTPAction.pemStringToDecodedBytes(StringUtils.join("", IOUtils.readLines(resourceAsStream, StandardCharsets.UTF_8))); SFTPTestConnectionAction.SFTPTestConnectionTestInput input = new SFTPTestConnectionAction.SFTPTestConnectionTestInput() .withUsername(BaseSFTPTest.USERNAME) From 4b585cde459b524d5debae0ffcf9d91421ad8e1d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 25 Feb 2025 11:47:09 -0600 Subject: [PATCH 185/209] convert paths starting with / to be ./ instead --- .../filesystem/sftp/actions/AbstractSFTPAction.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index f98582e8..93a5adaf 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -310,6 +310,16 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction rs = new ArrayList<>(); + ///////////////////////////////////////////////////////////////////////////////////// + // at least in some cases, listing / seems to be interpreted by the server as // + // a listing from the root of the system, not just the user's dir. so, converting // + // paths starting with / to instead be ./ is giving us better results. // + ///////////////////////////////////////////////////////////////////////////////////// + if(fullPath.startsWith("/")) + { + fullPath = "." + fullPath; + } + for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) { if(".".equals(dirEntry.getFilename()) || "..".equals(dirEntry.getFilename())) From 366f5d96000ed0d84c03c0486ad8961e136d5923 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:47:11 -0600 Subject: [PATCH 186/209] Initial checkin --- .../backend/core/scheduler/CronDescriber.java | 344 ++++++++++++++++++ .../CronExpressionTooltipFieldBehavior.java | 90 +++++ .../core/scheduler/CronDescriberTest.java | 68 ++++ ...ronExpressionTooltipFieldBehaviorTest.java | 67 ++++ 4 files changed, 569 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java new file mode 100644 index 00000000..2b2a2c56 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java @@ -0,0 +1,344 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/******************************************************************************* + ** class to give a human-friendly descriptive string from a cron expression. + ** (written in half by my friend Mr. Chatty G) + *******************************************************************************/ +public class CronDescriber +{ + private static final Map DAY_OF_WEEK_MAP = new HashMap<>(); + private static final Map MONTH_MAP = new HashMap<>(); + + static + { + DAY_OF_WEEK_MAP.put("1", "Sunday"); + DAY_OF_WEEK_MAP.put("2", "Monday"); + DAY_OF_WEEK_MAP.put("3", "Tuesday"); + DAY_OF_WEEK_MAP.put("4", "Wednesday"); + DAY_OF_WEEK_MAP.put("5", "Thursday"); + DAY_OF_WEEK_MAP.put("6", "Friday"); + DAY_OF_WEEK_MAP.put("7", "Saturday"); + + //////////////////////////////// + // Quartz also allows SUN-SAT // + //////////////////////////////// + DAY_OF_WEEK_MAP.put("SUN", "Sunday"); + DAY_OF_WEEK_MAP.put("MON", "Monday"); + DAY_OF_WEEK_MAP.put("TUE", "Tuesday"); + DAY_OF_WEEK_MAP.put("WED", "Wednesday"); + DAY_OF_WEEK_MAP.put("THU", "Thursday"); + DAY_OF_WEEK_MAP.put("FRI", "Friday"); + DAY_OF_WEEK_MAP.put("SAT", "Saturday"); + + MONTH_MAP.put("1", "January"); + MONTH_MAP.put("2", "February"); + MONTH_MAP.put("3", "March"); + MONTH_MAP.put("4", "April"); + MONTH_MAP.put("5", "May"); + MONTH_MAP.put("6", "June"); + MONTH_MAP.put("7", "July"); + MONTH_MAP.put("8", "August"); + MONTH_MAP.put("9", "September"); + MONTH_MAP.put("10", "October"); + MONTH_MAP.put("11", "November"); + MONTH_MAP.put("12", "December"); + + //////////////////////////////// + // Quartz also allows JAN-DEC // + //////////////////////////////// + MONTH_MAP.put("JAN", "January"); + MONTH_MAP.put("FEB", "February"); + MONTH_MAP.put("MAR", "March"); + MONTH_MAP.put("APR", "April"); + MONTH_MAP.put("MAY", "May"); + MONTH_MAP.put("JUN", "June"); + MONTH_MAP.put("JUL", "July"); + MONTH_MAP.put("AUG", "August"); + MONTH_MAP.put("SEP", "September"); + MONTH_MAP.put("OCT", "October"); + MONTH_MAP.put("NOV", "November"); + MONTH_MAP.put("DEC", "December"); + } + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getDescription(String cronExpression) throws ParseException + { + String[] parts = cronExpression.split("\\s+"); + if(parts.length < 6 || parts.length > 7) + { + throw new ParseException("Invalid cron expression: " + cronExpression, 0); + } + + String seconds = parts[0]; + String minutes = parts[1]; + String hours = parts[2]; + String dayOfMonth = parts[3]; + String month = parts[4]; + String dayOfWeek = parts[5]; + String year = parts.length == 7 ? parts[6] : "*"; + + StringBuilder description = new StringBuilder(); + + description.append("At "); + description.append(describeTime(seconds, minutes, hours)); + description.append(", on "); + description.append(describeDayOfMonth(dayOfMonth)); + description.append(" of "); + description.append(describeMonth(month)); + description.append(", "); + description.append(describeDayOfWeek(dayOfWeek)); + if(!year.equals("*")) + { + description.append(", in ").append(year); + } + description.append("."); + + return description.toString(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeTime(String seconds, String minutes, String hours) + { + return String.format("%s, %s, %s", describePart(seconds, "second"), describePart(minutes, "minute"), describePart(hours, "hour")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfMonth(String dayOfMonth) + { + if(dayOfMonth.equals("?")) + { + return "every day"; + } + else if(dayOfMonth.equals("L")) + { + return "the last day"; + } + else if(dayOfMonth.contains("W")) + { + return "the nearest weekday to day " + dayOfMonth.replace("W", ""); + } + else + { + return (describePart(dayOfMonth, "day")); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeMonth(String month) + { + if(month.equals("*")) + { + return "every month"; + } + else + { + String[] months = month.split(","); + StringBuilder result = new StringBuilder(); + for(String m : months) + { + result.append(MONTH_MAP.getOrDefault(m, m)).append(", "); + } + return result.substring(0, result.length() - 2); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describeDayOfWeek(String dayOfWeek) + { + if(dayOfWeek.equals("?")) + { + return "every day of the week"; + } + else if(dayOfWeek.equals("L")) + { + return "the last day of the week"; + } + else if(dayOfWeek.contains("#")) + { + String[] parts = dayOfWeek.split("#"); + return String.format("the %s %s of the month", ordinal(parts[1]), DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0])); + } + else if(dayOfWeek.contains("-")) + { + String[] parts = dayOfWeek.split("-"); + return String.format("from %s to %s", DAY_OF_WEEK_MAP.getOrDefault(parts[0], parts[0]), DAY_OF_WEEK_MAP.getOrDefault(parts[1], parts[1])); + } + else + { + String[] days = dayOfWeek.split(","); + StringBuilder result = new StringBuilder(); + for(String d : days) + { + result.append(DAY_OF_WEEK_MAP.getOrDefault(d, d)).append(", "); + } + return result.substring(0, result.length() - 2); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String describePart(String part, String label) + { + if(part.equals("*")) + { + return "every " + label; + } + else if(part.contains("/")) + { + String[] parts = part.split("/"); + if(parts[0].equals("*")) + { + parts[0] = "0"; + } + return String.format("every %s " + label + "s starting at %s", parts[1], parts[0]); + } + else if(part.contains(",")) + { + if(label.equals("hour")) + { + String[] parts = part.split(","); + List partList = Arrays.stream(parts).map(p -> hourToAmPm(p)).toList(); + return String.join(", ", partList); + } + else + { + if(label.equals("day")) + { + return "days " + part.replace(",", ", "); + } + else + { + return part.replace(",", ", ") + " " + label + "s"; + } + } + } + else if(part.contains("-")) + { + String[] parts = part.split("-"); + if(label.equals("day")) + { + return String.format("%ss from %s to %s", label, parts[0], parts[1]); + } + else if(label.equals("hour")) + { + return String.format("from %s to %s", hourToAmPm(parts[0]), hourToAmPm(parts[1])); + } + else + { + return String.format("from %s to %s %s", parts[0], parts[1], label + "s"); + } + } + else + { + if(label.equals("day")) + { + return label + " " + part; + } + if(label.equals("hour")) + { + return hourToAmPm(part); + } + else + { + return part + " " + label + "s"; + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String hourToAmPm(String part) + { + try + { + int hour = Integer.parseInt(part); + return switch(hour) + { + case 0 -> "midnight"; + case 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 -> hour + " AM"; + case 12 -> "noon"; + case 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23 -> (hour - 12) + " PM"; + default -> hour + " hours"; + }; + } + catch(Exception e) + { + return part + " hours"; + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static String ordinal(String number) + { + int n = Integer.parseInt(number); + if(n >= 11 && n <= 13) + { + return n + "th"; + } + + return switch(n % 10) + { + case 1 -> n + "st"; + case 2 -> n + "nd"; + case 3 -> n + "rd"; + default -> n + "th"; + }; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java new file mode 100644 index 00000000..bd77f6b4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehavior.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** Field display behavior, to add a human-redable tooltip to cron-expressions. + *******************************************************************************/ +public class CronExpressionTooltipFieldBehavior implements FieldDisplayBehavior +{ + + /*************************************************************************** + ** Add both this behavior, and the tooltip adornment to a field + ** Note, if either was already there, then that part is left alone. + ***************************************************************************/ + public static void addToField(QFieldMetaData fieldMetaData) + { + CronExpressionTooltipFieldBehavior existingBehavior = fieldMetaData.getBehaviorOnlyIfSet(CronExpressionTooltipFieldBehavior.class); + if(existingBehavior == null) + { + fieldMetaData.withBehavior(new CronExpressionTooltipFieldBehavior()); + } + + if(fieldMetaData.getAdornment(AdornmentType.TOOLTIP).isEmpty()) + { + fieldMetaData.withFieldAdornment((new FieldAdornment(AdornmentType.TOOLTIP) + .withValue(AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, true))); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : recordList) + { + try + { + String cronExpression = record.getValueString(field.getName()); + if(StringUtils.hasContent(cronExpression)) + { + String description = CronDescriber.getDescription(cronExpression); + record.setDisplayValue(field.getName() + ":" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC, description); + } + } + catch(Exception e) + { + ///////////////////// + // just leave null // + ///////////////////// + } + } + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java new file mode 100644 index 00000000..b5801d01 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import java.text.ParseException; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CronDescriber + *******************************************************************************/ +class CronDescriberTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws ParseException + { + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?")); + assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?")); + assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?")); + assertEquals("At 0 seconds, 0, 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?")); + assertEquals("At 0 seconds, 0 minutes, noon, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 12 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?")); + assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days 10, 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?")); + assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?")); + assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?")); + assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?")); + assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); + assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); + assertEquals("??", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java new file mode 100644 index 00000000..26dc8879 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronExpressionTooltipFieldBehaviorTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.scheduler; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for CronExpressionTooltipFieldBehavior + *******************************************************************************/ +class CronExpressionTooltipFieldBehaviorTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QFieldMetaData field = new QFieldMetaData("cronExpression", QFieldType.STRING); + QContext.getQInstance().getTable(TestUtils.TABLE_NAME_SHAPE) + .addField(field); + + CronExpressionTooltipFieldBehavior.addToField(field); + + new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_SHAPE).withRecord( + new QRecord().withValue("name", "Square").withValue("cronExpression", "* * * * * ?"))); + + QRecord record = new GetAction().executeForRecord(new GetInput(TestUtils.TABLE_NAME_SHAPE).withPrimaryKey(1).withShouldGenerateDisplayValues(true)); + assertThat(record.getDisplayValue("cronExpression:" + AdornmentType.TooltipValues.TOOLTIP_DYNAMIC)) + .contains("every second"); + } + +} \ No newline at end of file From 27c816d627d5e6331c5031e447b2cdcee2901d86 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:53:42 -0600 Subject: [PATCH 187/209] Add a root exception --- .../qqq/backend/core/actions/reporting/CsvExportStreamer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java index fac33e13..d36305f4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/CsvExportStreamer.java @@ -108,7 +108,7 @@ public class CsvExportStreamer implements ExportStreamerInterface } catch(Exception e) { - throw (new QReportingException("Error starting CSV report")); + throw (new QReportingException("Error starting CSV report", e)); } } From 428832f4ec22b54c9baea41298a38e9074e60a2c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:54:00 -0600 Subject: [PATCH 188/209] Add discoverAndAddPluginsInPackage --- .../core/instances/QInstanceEnricher.java | 25 ++++++++++ .../core/instances/QInstanceEnricherTest.java | 35 ++++++++------ .../testplugins/TestEnricherPlugin.java | 48 +++++++++++++++++++ 3 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 809508cd..a234bfe1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -95,6 +95,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; +import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -1482,6 +1483,30 @@ public class QInstanceEnricher + /*************************************************************************** + ** + ***************************************************************************/ + public static void discoverAndAddPluginsInPackage(String packageName) throws QException + { + try + { + for(Class aClass : ClassPathUtils.getClassesInPackage(packageName)) + { + if(QInstanceEnricherPluginInterface.class.isAssignableFrom(aClass)) + { + QInstanceEnricherPluginInterface plugin = (QInstanceEnricherPluginInterface) aClass.getConstructor().newInstance(); + addEnricherPlugin(plugin); + } + } + } + catch(Exception e) + { + throw (new QException("Error discovering and adding enricher plugins in package [" + packageName + "]", e)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 1817b57c..58e7ccca 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -27,7 +27,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.enrichment.testplugins.TestEnricherPlugin; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -595,27 +596,31 @@ class QInstanceEnricherTest extends BaseTest { QInstance qInstance = TestUtils.defineInstance(); - QInstanceEnricher.addEnricherPlugin(new QInstanceEnricherPluginInterface() - { - /*************************************************************************** - * - ***************************************************************************/ - @Override - public void enrich(QFieldMetaData field, QInstance qInstance) - { - if(field != null) - { - field.setLabel(field.getLabel() + " Plugged"); - } - } - }); + QInstanceEnricher.addEnricherPlugin(new TestEnricherPlugin()); new QInstanceEnricher(qInstance).enrich(); qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDiscoverAndAddPlugins() throws QException + { + QInstance qInstance = TestUtils.defineInstance(); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).doesNotEndWith("Plugged"))); + + qInstance = TestUtils.defineInstance(); + QInstanceEnricher.discoverAndAddPluginsInPackage(getClass().getPackageName() + ".enrichment.testplugins"); + new QInstanceEnricher(qInstance).enrich(); + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java new file mode 100644 index 00000000..9e940284 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/enrichment/testplugins/TestEnricherPlugin.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.enrichment.testplugins; + + +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TestEnricherPlugin implements QInstanceEnricherPluginInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void enrich(QFieldMetaData field, QInstance qInstance) + { + if(field != null) + { + field.setLabel(field.getLabel() + " Plugged"); + } + } + +} From 2703f06b2351f83246b01c2b5b54cf3d4cc21589 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:55:07 -0600 Subject: [PATCH 189/209] Add TOOLTIP type adornment; also, update url-encoding in FileDownload adornment to .replace("+", "%20") --- .../model/metadata/fields/AdornmentType.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index 9fb76f41..4cf6e42b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -46,6 +46,7 @@ public enum AdornmentType REVEAL, FILE_DOWNLOAD, FILE_UPLOAD, + TOOLTIP, ERROR; ////////////////////////////////////////////////////////////////////////// // keep these values in sync with AdornmentType.ts in qqq-frontend-core // @@ -92,9 +93,9 @@ public enum AdornmentType static String makeFieldDownloadUrl(String tableName, Serializable primaryKey, String fieldName, String fileName) { return ("/data/" + tableName + "/" - + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8) + "/" + + URLEncoder.encode(Objects.requireNonNullElse(ValueUtils.getValueAsString(primaryKey), ""), StandardCharsets.UTF_8).replace("+", "%20") + "/" + fieldName + "/" - + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8)); + + URLEncoder.encode(Objects.requireNonNullElse(fileName, ""), StandardCharsets.UTF_8).replace("+", "%20")); } } @@ -246,4 +247,15 @@ public enum AdornmentType } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public interface TooltipValues + { + String STATIC_TEXT = "staticText"; + String TOOLTIP_DYNAMIC = "tooltipDynamic"; + } + } From 1354755372aab7429949e85b52497811bb5dc854 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 14:56:05 -0600 Subject: [PATCH 190/209] Make some of hard-coded table & field names optionally come from widget input, for more flexible usage (e.g., by sftp-data-integration qbit's report export setup) --- ...ReportValuesDynamicFormWidgetRenderer.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java index 3a448eda..f71da271 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedreports/ReportValuesDynamicFormWidgetRenderer.java @@ -56,7 +56,16 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* ** Note - exists under 2 names, for the RenderSavedReport process, and for the - ** ScheduledReport table + ** ScheduledReport table (and can be used in your custom code too: + * + ** by default, in qqq backend core, we'll assume this widget is being used on the + ** view screen for a ScheduledReport, with field names that we know from that table. + ** But, allow it to be used on a different table (optionally with different field names), + ** coming from the input map. + ** + ** e.g., that one may set in widget metaData as: + ** .withDefaultValue("tableName", "myTable") + ** .withDefaultValue("fieldNameId", "identifier"), etc. *******************************************************************************/ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRenderer { @@ -88,11 +97,16 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } else if(input.getQueryParams().containsKey("id")) { - QRecord scheduledReportRecord = new GetAction().executeForRecord(new GetInput(ScheduledReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get("id")))); - QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(scheduledReportRecord.getValueInteger("savedReportId")))); + String tableName = input.getQueryParams().getOrDefault("tableName", ScheduledReport.TABLE_NAME); + String fieldNameId = input.getQueryParams().getOrDefault("fieldNameId", "id"); + String fieldNameSavedReportId = input.getQueryParams().getOrDefault("fieldNameSavedReportId", "savedReportId"); + String fieldNameInputValues = input.getQueryParams().getOrDefault("fieldNameInputValues", "inputValues"); + + QRecord hostRecord = new GetAction().executeForRecord(new GetInput(tableName).withPrimaryKey(ValueUtils.getValueAsInteger(input.getQueryParams().get(fieldNameId)))); + QRecord record = new GetAction().executeForRecord(new GetInput(SavedReport.TABLE_NAME).withPrimaryKey(ValueUtils.getValueAsInteger(hostRecord.getValueInteger(fieldNameSavedReportId)))); savedReport = new SavedReport(record); - String inputValues = scheduledReportRecord.getValueString("inputValues"); + String inputValues = hostRecord.getValueString(fieldNameInputValues); if(StringUtils.hasContent(inputValues)) { JSONObject jsonObject = JsonUtils.toJSONObject(inputValues); @@ -197,8 +211,8 @@ public class ReportValuesDynamicFormWidgetRenderer extends AbstractWidgetRendere } catch(Exception e) { - LOG.warn("Error rendering scheduled report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); - throw (new QException("Error rendering scheduled report values dynamic form widget", e)); + LOG.warn("Error rendering report values dynamic form widget", e, logPair("queryParams", String.valueOf(input.getQueryParams()))); + throw (new QException("Error rendering report values dynamic form widget", e)); } } From b87fb6bd4a2ed65d89c4b4e5a6c09dfb244cb06e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:11:11 -0600 Subject: [PATCH 191/209] Adjust inserted-ids process summary line for when only 1 record was inserted --- .../bulk/insert/BulkInsertLoadStep.java | 5 + .../insert/BulkInsertFullProcessTest.java | 205 ++++++++++++------ 2 files changed, 147 insertions(+), 63 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java index b35bb24b..30f98c91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertLoadStep.java @@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; @@ -110,6 +111,10 @@ public class BulkInsertLoadStep extends LoadViaInsertStep implements ProcessSumm if(field.getType().isNumeric()) { ProcessSummaryLine idsLine = new ProcessSummaryLine(Status.INFO, "Inserted " + field.getLabel() + " values between " + firstInsertedPrimaryKey + " and " + lastInsertedPrimaryKey); + if(Objects.equals(firstInsertedPrimaryKey, lastInsertedPrimaryKey)) + { + idsLine.setMessage("Inserted " + field.getLabel() + " " + firstInsertedPrimaryKey); + } idsLine.setCount(null); processSummary.add(idsLine); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 2ab31159..91972a91 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -22,24 +22,21 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; @@ -61,6 +58,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; *******************************************************************************/ class BulkInsertFullProcessTest extends BaseTest { + private static final String defaultEmail = "noone@kingsrook.com"; + + /******************************************************************************* ** @@ -116,48 +116,20 @@ class BulkInsertFullProcessTest extends BaseTest @Test void test() throws Exception { - String defaultEmail = "noone@kingsrook.com"; - - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - QInstance qInstance = QContext.getQInstance(); - String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); - ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // ///////////////////////////////////////////////////////// - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(processName); - runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); String processUUID = runProcessOutput.getProcessUUID(); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); - ////////////////////////////// - // simulate the file upload // - ////////////////////////////// - String storageReference = UUID.randomUUID() + ".csv"; - StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); - try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) - { - outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - } - catch(IOException e) - { - throw (e); - } - ////////////////////////// // continue post-upload // ////////////////////////// - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(2)); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); @@ -176,29 +148,10 @@ class BulkInsertFullProcessTest extends BaseTest assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); - //////////////////////////////////////////////////////////////////////////////// - // all subsequent steps will want these data - so set up a lambda to set them // - //////////////////////////////////////////////////////////////////////////////// - Consumer addProfileToRunProcessInput = (RunProcessInput input) -> - { - input.addValue("version", "v1"); - input.addValue("layout", "FLAT"); - input.addValue("hasHeaderRow", "true"); - input.addValue("fieldListJSON", JsonUtils.toJson(List.of( - new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), - new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), - new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), - new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), - new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) - ))); - }; - //////////////////////////////// // continue post file-mapping // //////////////////////////////// - runProcessInput.setStartAfterStep("fileMapping"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostFileMapping(runProcessInput); Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); @@ -211,23 +164,20 @@ class BulkInsertFullProcessTest extends BaseTest ///////////////////////////////// // continue post value-mapping // ///////////////////////////////// - runProcessInput.setStartAfterStep("valueMapping"); - runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostValueMapping(runProcessInput); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); ///////////////////////////////// // continue post review screen // ///////////////////////////////// - runProcessInput.setStartAfterStep("review"); - addProfileToRunProcessInput.accept(runProcessInput); - runProcessOutput = new RunProcessAction().execute(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); assertThat(runProcessOutput.getRecords()).hasSize(2); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); assertThat(runProcessOutput.getException()).isEmpty(); + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id values between 1 and 2"); + //////////////////////////////////// // query for the inserted records // //////////////////////////////////// @@ -249,4 +199,133 @@ class BulkInsertFullProcessTest extends BaseTest assertNull(records.get(1).getValue("noOfShoes")); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOneRow() throws Exception + { + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + RunProcessInput runProcessInput = new RunProcessInput(); + RunProcessOutput runProcessOutput = startProcess(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + + continueProcessPostUpload(runProcessInput, processUUID, simulateFileUpload(1)); + continueProcessPostFileMapping(runProcessInput); + continueProcessPostValueMapping(runProcessInput); + runProcessOutput = continueProcessPostReviewScreen(runProcessInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // all that just so we can make sure this message is right (because it was wrong when we first wrote it, lol) // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + ProcessSummaryAssert.assertThat(runProcessOutput).hasLineWithMessageContaining("Inserted Id 1"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostReviewScreen(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostValueMapping(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput(runProcessInput); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException + { + RunProcessOutput runProcessOutput; + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput continueProcessPostUpload(RunProcessInput runProcessInput, String processUUID, StorageInput storageInput) throws QException + { + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return (runProcessOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static StorageInput simulateFileUpload(int noOfRows) throws Exception + { + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + (noOfRows == 2 ? getPersonCsvRow2() : "")).getBytes()); + } + return storageInput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static RunProcessOutput startProcess(RunProcessInput runProcessInput) throws QException + { + runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + return runProcessOutput; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addProfileToRunProcessInput(RunProcessInput input) + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + } + } \ No newline at end of file From 2a0bc03337c074a102dffa5d7fe10398df140544 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:14:47 -0600 Subject: [PATCH 192/209] Accept storageReference (file path) as optional input --- .../RenderSavedReportExecuteStep.java | 27 +++++++++++-------- .../RenderSavedReportMetaDataProducer.java | 2 ++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java index f91a9000..900dc4da 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportExecuteStep.java @@ -92,16 +92,21 @@ public class RenderSavedReportExecuteStep implements BackendStep //////////////////////////////// // read inputs, set up params // //////////////////////////////// - String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); - String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); - String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); - String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); - ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); - String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); - String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); - SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); - String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); - String storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + String sesProviderName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.SES_PROVIDER_NAME); + String fromEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FROM_EMAIL_ADDRESS); + String replyToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.REPLY_TO_EMAIL_ADDRESS); + String storageTableName = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_TABLE_NAME); + ReportFormat reportFormat = ReportFormat.fromString(runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_REPORT_FORMAT)); + String sendToEmailAddress = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_ADDRESS); + String emailSubject = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_EMAIL_SUBJECT); + SavedReport savedReport = new SavedReport(runBackendStepInput.getRecords().get(0)); + + String downloadFileBaseName = getDownloadFileBaseName(runBackendStepInput, savedReport); + String storageReference = runBackendStepInput.getValueString(RenderSavedReportMetaDataProducer.FIELD_NAME_STORAGE_REFERENCE); + if(!StringUtils.hasContent(storageReference)) + { + storageReference = LocalDate.now() + "/" + LocalTime.now().toString().replaceAll(":", "").replaceFirst("\\..*", "") + "/" + UUID.randomUUID() + "/" + downloadFileBaseName + "." + reportFormat.getExtension(); + } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // if sending an email (or emails), validate the addresses before doing anything so user gets error and can fix // @@ -241,7 +246,7 @@ public class RenderSavedReportExecuteStep implements BackendStep /******************************************************************************* ** *******************************************************************************/ - private String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) + public static String getDownloadFileBaseName(RunBackendStepInput runBackendStepInput, SavedReport report) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-HHmm").withZone(ZoneId.systemDefault()); String datePart = formatter.format(Instant.now()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java index c8693195..4b25c61c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedreports/RenderSavedReportMetaDataProducer.java @@ -56,6 +56,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf public static final String FROM_EMAIL_ADDRESS = "fromEmailAddress"; public static final String REPLY_TO_EMAIL_ADDRESS = "replyToEmailAddress"; public static final String FIELD_NAME_STORAGE_TABLE_NAME = "storageTableName"; + public static final String FIELD_NAME_STORAGE_REFERENCE = "storageReference"; public static final String FIELD_NAME_REPORT_FORMAT = "reportFormat"; public static final String FIELD_NAME_EMAIL_ADDRESS = "reportDestinationEmailAddress"; public static final String FIELD_NAME_EMAIL_SUBJECT = "emailSubject"; @@ -81,6 +82,7 @@ public class RenderSavedReportMetaDataProducer implements MetaDataProducerInterf .withField(new QFieldMetaData(FROM_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(REPLY_TO_EMAIL_ADDRESS, QFieldType.STRING)) .withField(new QFieldMetaData(FIELD_NAME_STORAGE_TABLE_NAME, QFieldType.STRING)) + .withField(new QFieldMetaData(FIELD_NAME_STORAGE_REFERENCE, QFieldType.STRING)) .withRecordListMetaData(new QRecordListMetaData().withTableName(SavedReport.TABLE_NAME))) .withCode(new QCodeReference(RenderSavedReportPreStep.class))) From 92f0bd38467e8310c3bd58afef1a1d6e06d6391a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:15:26 -0600 Subject: [PATCH 193/209] Try to bubble more useful exceptions out --- .../filesystem/sftp/actions/AbstractSFTPAction.java | 13 ++++++++++++- .../filesystem/sftp/utils/SFTPOutputStream.java | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 93a5adaf..8d5330ff 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -36,9 +36,11 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -46,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; @@ -57,6 +60,7 @@ import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.config.keys.KeyUtils; import org.apache.sshd.sftp.client.SftpClient; import org.apache.sshd.sftp.client.SftpClientFactory; +import org.apache.sshd.sftp.common.SftpException; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -300,9 +304,10 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction listFiles(QTableMetaData table, QBackendMetaData backendBase, String requestedPath) throws QException { + String fullPath = null; try { - String fullPath = getFullBasePath(table, backendBase); + fullPath = getFullBasePath(table, backendBase); if(StringUtils.hasContent(requestedPath)) { fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + requestedPath + File.separatorChar); @@ -340,6 +345,12 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction putFuture; @@ -62,12 +65,14 @@ public class SFTPOutputStream extends PipedOutputStream pipedInputStream = new PipedInputStream(this, 32 * 1024); this.sftpClient = sftpClient; + this.path = path; putFuture = Executors.newSingleThreadExecutor().submit(() -> { try { started.set(true); + LOG.debug("Starting sftp put", logPair("path", path)); sftpClient.put(pipedInputStream, path); } catch(Exception e) @@ -105,6 +110,10 @@ public class SFTPOutputStream extends PipedOutputStream { if(putException.get() != null) { + if(putException.get() instanceof SftpException sftpException) + { + throw new IOException("Error performing SFTP put for path [" + path + "]: " + sftpException.getMessage()); + } throw new IOException("Error performing SFTP put", putException.get()); } From 3ae5f90cc8396512a5f548b8a19eaf180f26f629 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:18:31 -0600 Subject: [PATCH 194/209] Checkstyle --- .../implementations/bulk/insert/BulkInsertFullProcessTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 91972a91..2fe422fd 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -257,6 +257,9 @@ class BulkInsertFullProcessTest extends BaseTest + /*************************************************************************** + ** + ***************************************************************************/ private static RunProcessOutput continueProcessPostFileMapping(RunProcessInput runProcessInput) throws QException { RunProcessOutput runProcessOutput; From 2808b3fcc4fb7cc24547ba8f457f020b34bd8b54 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 15:22:49 -0600 Subject: [PATCH 195/209] test fixes --- .../qqq/backend/core/actions/values/QValueFormatterTest.java | 4 ++-- .../qqq/backend/core/scheduler/CronDescriberTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java index c4eac300..05d24862 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatterTest.java @@ -325,7 +325,7 @@ class QValueFormatterTest extends BaseTest /////////////////////////////////////////////////////////////////////////////////////////// QRecord record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); - assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob", record.getValueString("blobField")); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob", record.getValueString("blobField")); assertEquals("Test Table 47 Blob", record.getDisplayValue("blobField")); //////////////////////////////////////// @@ -334,7 +334,7 @@ class QValueFormatterTest extends BaseTest adornment.withValue(AdornmentType.FileDownloadValues.DEFAULT_EXTENSION, "html"); record = new QRecord().withValue("id", 47).withValue("blobField", blobBytes); QValueFormatter.setBlobValuesToDownloadUrls(table, List.of(record)); - assertEquals("/data/testTable/47/blobField/Test+Table+47+Blob.html", record.getValueString("blobField")); + assertEquals("/data/testTable/47/blobField/Test%20Table%2047%20Blob.html", record.getValueString("blobField")); assertEquals("Test Table 47 Blob.html", record.getDisplayValue("blobField")); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java index b5801d01..35f1e3bf 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -62,7 +62,7 @@ class CronDescriberTest extends BaseTest assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); - assertEquals("??", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, 52 minutes, every hour, on every day of January, March, September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); } } \ No newline at end of file From 425d18e6df1dd76f6387fb5b1357b92cf4204357 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 16:03:10 -0600 Subject: [PATCH 196/209] Remove TOOLTIP from FieldAdornment values --- .../javalin/specs/v1/responses/components/FieldAdornment.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java index 2f59ee63..aeebca15 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java @@ -82,6 +82,7 @@ public class FieldAdornment implements ToSchema { EnumSet subSet = EnumSet.allOf(AdornmentType.class); subSet.remove(AdornmentType.FILE_UPLOAD); // todo - remove for next version! + subSet.remove(AdornmentType.TOOLTIP); // todo - remove for next version! FieldAdornmentSubSet.subSet = subSet; } From 7efd8264faafbb364a0725c99b4f93ba67d31181 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 16:56:19 -0600 Subject: [PATCH 197/209] Change tables PVS to be custom type, respecting session permissions; refactor some PVS search logic to make custom implementations a little better --- .../values/QCustomPossibleValueProvider.java | 24 +++ .../SearchPossibleValueSourceAction.java | 140 ++++++++++------ .../TablesCustomPossibleValueProvider.java | 88 ++++++++++ ...esPossibleValueSourceMetaDataProvider.java | 24 +-- ...TablesCustomPossibleValueProviderTest.java | 157 ++++++++++++++++++ 5 files changed, 360 insertions(+), 73 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java index 09e786c5..09bbd74e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java @@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.core.actions.values; import java.io.Serializable; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; @@ -74,6 +75,29 @@ public interface QCustomPossibleValueProvider } + /*************************************************************************** + * + ***************************************************************************/ + default List> _defaultSearch(SearchPossibleValueSourceInput input, List> possibleValues) + { + SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input); + + List> rs = new ArrayList<>(); + + for(QPossibleValue possibleValue : possibleValues) + { + if(possibleValue != null && SearchPossibleValueSourceAction.doesPossibleValueMatchSearchInput(possibleValue, preparedInput)) + { + rs.add(possibleValue); + } + } + + rs.sort(Comparator.nullsLast(Comparator.comparing((QPossibleValue pv) -> pv.getLabel()))); + + return (rs); + } + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 97bcae1c..2063f39f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -51,7 +52,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -108,60 +108,54 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** record to store "computed" values as part of a possible-value search - + ** e.g., ids type-convered, and lower-cased labels. + ***************************************************************************/ + public record PreparedSearchPossibleValueSourceInput(Collection inputIdsAsCorrectType, Collection lowerCaseLabels, String searchTerm) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static PreparedSearchPossibleValueSourceInput prepareSearchPossibleValueSourceInput(SearchPossibleValueSourceInput input) + { + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(input.getPossibleValueSourceName()); + List inputIdsAsCorrectType = convertInputIdsToPossibleValueSourceIdType(possibleValueSource, input.getIdList()); + + Set lowerCaseLabels = null; + if(input.getLabelList() != null) + { + lowerCaseLabels = input.getLabelList().stream() + .filter(Objects::nonNull) + .map(l -> l.toLowerCase()) + .collect(Collectors.toSet()); + } + + return (new PreparedSearchPossibleValueSourceInput(inputIdsAsCorrectType, lowerCaseLabels, input.getSearchTerm())); + } + + + /******************************************************************************* ** *******************************************************************************/ private SearchPossibleValueSourceOutput searchPossibleValueEnum(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) { + PreparedSearchPossibleValueSourceInput preparedSearchPossibleValueSourceInput = prepareSearchPossibleValueSourceInput(input); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceOutput(); List matchingIds = new ArrayList<>(); - List inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList()); - Set labels = null; - for(QPossibleValue possibleValue : possibleValueSource.getEnumValues()) { - boolean match = false; - - if(input.getIdList() != null) - { - if(inputIdsAsCorrectType.contains(possibleValue.getId())) - { - match = true; - } - } - else if(input.getLabelList() != null) - { - if(labels == null) - { - labels = input.getLabelList().stream().filter(Objects::nonNull).map(l -> l.toLowerCase()).collect(Collectors.toSet()); - } - - if(labels.contains(possibleValue.getLabel().toLowerCase())) - { - match = true; - } - } - else - { - if(StringUtils.hasContent(input.getSearchTerm())) - { - match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase()) - || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); - } - else - { - match = true; - } - } + boolean match = doesPossibleValueMatchSearchInput(possibleValue, preparedSearchPossibleValueSourceInput); if(match) { - matchingIds.add((Serializable) possibleValue.getId()); + matchingIds.add(possibleValue.getId()); } - - // todo - skip & limit? - // todo - default filter } List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, matchingIds); @@ -172,42 +166,84 @@ public class SearchPossibleValueSourceAction + /*************************************************************************** + ** + ***************************************************************************/ + public static boolean doesPossibleValueMatchSearchInput(QPossibleValue possibleValue, PreparedSearchPossibleValueSourceInput input) + { + boolean match = false; + + if(input.inputIdsAsCorrectType() != null) + { + if(input.inputIdsAsCorrectType().contains(possibleValue.getId())) + { + match = true; + } + } + else if(input.lowerCaseLabels() != null) + { + if(input.lowerCaseLabels().contains(possibleValue.getLabel().toLowerCase())) + { + match = true; + } + } + else + { + if(StringUtils.hasContent(input.searchTerm())) + { + match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.searchTerm().toLowerCase()) + || possibleValue.getLabel().toLowerCase().startsWith(input.searchTerm().toLowerCase())); + } + else + { + match = true; + } + } + return match; + } + + + /******************************************************************************* ** The input list of ids might come through as a type that isn't the same as ** the type of the ids in the enum (e.g., strings from a frontend, integers - ** in an enum). So, this method looks at the first id in the enum, and then - ** maps all the inputIds to be of the same type. + ** in an enum). So, this method type-converts them. *******************************************************************************/ - private List convertInputIdsToEnumIdType(QPossibleValueSource possibleValueSource, List inputIdList) + private static List convertInputIdsToPossibleValueSourceIdType(QPossibleValueSource possibleValueSource, List inputIdList) { List rs = new ArrayList<>(); - if(CollectionUtils.nullSafeIsEmpty(inputIdList)) + + if(inputIdList == null) + { + return (null); + } + else if(inputIdList.isEmpty()) { return (rs); } - Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId(); + QFieldType type = possibleValueSource.getIdType(); for(Serializable inputId : inputIdList) { Object properlyTypedId = null; try { - if(anIdFromTheEnum instanceof Integer) + if(type.equals(QFieldType.INTEGER)) { properlyTypedId = ValueUtils.getValueAsInteger(inputId); } - else if(anIdFromTheEnum instanceof String) + else if(type.isStringLike()) { properlyTypedId = ValueUtils.getValueAsString(inputId); } - else if(anIdFromTheEnum instanceof Boolean) + else if(type.equals(QFieldType.BOOLEAN)) { properlyTypedId = ValueUtils.getValueAsBoolean(inputId); } else { - LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + LOG.warn("Unexpected type [" + type + "] for ids in enum: " + possibleValueSource.getName()); } } catch(Exception e) @@ -215,7 +251,7 @@ public class SearchPossibleValueSourceAction LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId)); } - if (properlyTypedId != null) + if(properlyTypedId != null) { rs.add(properlyTypedId); } @@ -397,7 +433,7 @@ public class SearchPossibleValueSourceAction } catch(Exception e) { - String message = "Error sending searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; + String message = "Error searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; LOG.warn(message, e); throw (new QException(message)); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java new file mode 100644 index 00000000..93e97d83 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TablesCustomPossibleValueProvider implements QCustomPossibleValueProvider +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QPossibleValue getPossibleValue(Serializable idValue) + { + QTableMetaData table = QContext.getQInstance().getTable(ValueUtils.getValueAsString(idValue)); + if(table != null && !table.getIsHidden()) + { + PermissionCheckResult permissionCheckResult = PermissionsHelper.getPermissionCheckResult(new QueryInput(table.getName()), table); + if(PermissionCheckResult.ALLOW.equals(permissionCheckResult)) + { + return (new QPossibleValue<>(table.getName(), table.getLabel())); + } + } + + return null; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List> search(SearchPossibleValueSourceInput input) throws QException + { + ///////////////////////////////////////////////////////////////////////////////////// + // build all of the possible values (note, will be filtered by user's permissions) // + ///////////////////////////////////////////////////////////////////////////////////// + List> allPossibleValues = new ArrayList<>(); + for(QTableMetaData table : QContext.getQInstance().getTables().values()) + { + QPossibleValue possibleValue = getPossibleValue(table.getName()); + if(possibleValue != null) + { + allPossibleValues.add(possibleValue); + } + } + + return _defaultSearch(input, allPossibleValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java index 27cce6df..75087ae7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesPossibleValueSourceMetaDataProvider.java @@ -22,17 +22,11 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PVSValueFormatAndFields; -import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import org.apache.commons.lang.BooleanUtils; /******************************************************************************* @@ -51,22 +45,10 @@ public class TablesPossibleValueSourceMetaDataProvider { QPossibleValueSource possibleValueSource = new QPossibleValueSource() .withName(NAME) - .withType(QPossibleValueSourceType.ENUM) + .withType(QPossibleValueSourceType.CUSTOM) + .withCustomCodeReference(new QCodeReference(TablesCustomPossibleValueProvider.class)) .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY); - List> enumValues = new ArrayList<>(); - for(QTableMetaData table : qInstance.getTables().values()) - { - if(BooleanUtils.isNotTrue(table.getIsHidden())) - { - String label = StringUtils.hasContent(table.getLabel()) ? table.getLabel() : QInstanceEnricher.nameToLabel(table.getName()); - enumValues.add(new QPossibleValue<>(table.getName(), label)); - } - } - - enumValues.sort(Comparator.comparing(QPossibleValue::getLabel)); - - possibleValueSource.withEnumValues(enumValues); return (possibleValueSource); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java new file mode 100644 index 00000000..4bdbbc06 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProviderTest.java @@ -0,0 +1,157 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.metadata.tables; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TablesCustomPossibleValueProvider + *******************************************************************************/ +class TablesCustomPossibleValueProviderTest extends BaseTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void beforeEach() + { + QInstance qInstance = TestUtils.defineInstance(); + + qInstance.addTable(new QTableMetaData() + .withName("hidden") + .withIsHidden(true) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addTable(new QTableMetaData() + .withName("restricted") + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)) + .withBackendName(TestUtils.MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER))); + + qInstance.addPossibleValueSource(TablesPossibleValueSourceMetaDataProvider.defineTablesPossibleValueSource(qInstance)); + + QContext.init(qInstance, newSession()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetPossibleValue() + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + QPossibleValue possibleValue = provider.getPossibleValue(TestUtils.TABLE_NAME_PERSON); + assertEquals(TestUtils.TABLE_NAME_PERSON, possibleValue.getId()); + assertEquals("Person", possibleValue.getLabel()); + + assertNull(provider.getPossibleValue("no-such-table")); + assertNull(provider.getPossibleValue("hidden")); + assertNull(provider.getPossibleValue("restricted")); + + QContext.getQSession().withPermission("restricted.hasAccess"); + assertNotNull(provider.getPossibleValue("restricted")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPossibleValue() throws QException + { + TablesCustomPossibleValueProvider provider = new TablesCustomPossibleValueProvider(); + + List> list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).noneMatch(p -> p.getId().equals("no-such-table")); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + assertNull(provider.getPossibleValue("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withIdList(List.of(TestUtils.TABLE_NAME_PERSON, TestUtils.TABLE_NAME_SHAPE, "hidden"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("hidden")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(2, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).noneMatch(p -> p.getId().equals("restricted")); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(0, list.size()); + + ///////////////////////////////////////// + // add permission for restricted table // + ///////////////////////////////////////// + QContext.getQSession().withPermission("restricted.hasAccess"); + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withSearchTerm("restricted")); + assertEquals(1, list.size()); + + list = provider.search(new SearchPossibleValueSourceInput() + .withPossibleValueSourceName(TablesPossibleValueSourceMetaDataProvider.NAME) + .withLabelList(List.of("Person", "Shape", "Restricted"))); + assertEquals(3, list.size()); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_PERSON)); + assertThat(list).anyMatch(p -> p.getId().equals(TestUtils.TABLE_NAME_SHAPE)); + assertThat(list).anyMatch(p -> p.getId().equals("restricted")); + + } + +} \ No newline at end of file From 9fb53af0ba71d5599aa508d7c3f478d04ea89690 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 26 Feb 2025 17:01:40 -0600 Subject: [PATCH 198/209] Checkstyle! also rename new method --- .../core/actions/values/QCustomPossibleValueProvider.java | 6 ++++-- .../metadata/tables/TablesCustomPossibleValueProvider.java | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java index 09bbd74e..d32e4e79 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QCustomPossibleValueProvider.java @@ -76,9 +76,11 @@ public interface QCustomPossibleValueProvider /*************************************************************************** - * + ** meant to be protected (but interface...) - for a custom PVS implementation + ** to complete its search (e.g., after it generates the list of PVS objects, + ** let this method do the filtering). ***************************************************************************/ - default List> _defaultSearch(SearchPossibleValueSourceInput input, List> possibleValues) + default List> completeCustomPVSSearch(SearchPossibleValueSourceInput input, List> possibleValues) { SearchPossibleValueSourceAction.PreparedSearchPossibleValueSourceInput preparedInput = SearchPossibleValueSourceAction.prepareSearchPossibleValueSourceInput(input); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java index 93e97d83..df2b55a3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/TablesCustomPossibleValueProvider.java @@ -82,7 +82,7 @@ public class TablesCustomPossibleValueProvider implements QCustomPossibleValuePr } } - return _defaultSearch(input, allPossibleValues); + return completeCustomPVSSearch(input, allPossibleValues); } } From 99e282fcdfcf6421f880179dbeccc9a2b796e572 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:41:57 -0600 Subject: [PATCH 199/209] Add sourceClass attribute to MetaDataProducerInterface --- .../core/model/metadata/MetaDataProducer.java | 35 ++++++++++ .../metadata/MetaDataProducerHelper.java | 65 +++++++++++-------- .../metadata/MetaDataProducerInterface.java | 19 ++++++ ...omRecordEntityGenericMetaDataProducer.java | 34 ++++++++++ ...omRecordEntityGenericMetaDataProducer.java | 34 ++++++++++ ...ueSourceOfEnumGenericMetaDataProducer.java | 37 +++++++++++ ...eSourceOfTableGenericMetaDataProducer.java | 35 ++++++++++ ...dEntityToTableGenericMetaDataProducer.java | 34 ++++++++++ 8 files changed, 266 insertions(+), 27 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java index a4dbe375..401ecb0e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducer.java @@ -29,5 +29,40 @@ package com.kingsrook.qqq.backend.core.model.metadata; *******************************************************************************/ public abstract class MetaDataProducer implements MetaDataProducerInterface { + private Class sourceClass; + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + @Override + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + @Override + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public MetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java index 487c6dda..ec41e0b6 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerHelper.java @@ -239,17 +239,19 @@ public class MetaDataProducerHelper ** ***************************************************************************/ @SuppressWarnings("unchecked") - private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class aClass) + private static > MetaDataProducerInterface processMetaDataProducingPossibleValueEnum(Class sourceClass) { String warningPrefix = "Found a class annotated as @" + QMetaDataProducingPossibleValueEnum.class.getSimpleName(); - if(!PossibleValueEnum.class.isAssignableFrom(aClass)) + if(!PossibleValueEnum.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + PossibleValueEnum.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return null; } - PossibleValueEnum[] values = (PossibleValueEnum[]) aClass.getEnumConstants(); - return (new PossibleValueSourceOfEnumGenericMetaDataProducer(aClass.getSimpleName(), (PossibleValueEnum[]) values)); + PossibleValueEnum[] values = (PossibleValueEnum[]) sourceClass.getEnumConstants(); + PossibleValueSourceOfEnumGenericMetaDataProducer producer = new PossibleValueSourceOfEnumGenericMetaDataProducer<>(sourceClass.getSimpleName(), (PossibleValueEnum[]) values); + producer.setSourceClass(sourceClass); + return producer; } @@ -257,32 +259,32 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static List> processMetaDataProducingEntity(Class aClass) throws Exception + private static List> processMetaDataProducingEntity(Class sourceClass) throws Exception { List> rs = new ArrayList<>(); - QMetaDataProducingEntity qMetaDataProducingEntity = aClass.getAnnotation(QMetaDataProducingEntity.class); + QMetaDataProducingEntity qMetaDataProducingEntity = sourceClass.getAnnotation(QMetaDataProducingEntity.class); String warningPrefix = "Found a class annotated as @" + QMetaDataProducingEntity.class.getSimpleName(); /////////////////////////////////////////////////////////// // make sures class is QRecordEntity and cast it as such // /////////////////////////////////////////////////////////// - if(!QRecordEntity.class.isAssignableFrom(aClass)) + if(!QRecordEntity.class.isAssignableFrom(sourceClass)) { - LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but which is not a " + QRecordEntity.class.getSimpleName() + ", so it will not be used.", logPair("class", sourceClass.getSimpleName())); return (rs); } @SuppressWarnings("unchecked") // safe per the check above. - Class recordEntityClass = (Class) aClass; + Class recordEntityClass = (Class) sourceClass; //////////////////////////////////////////////// // get TABLE_NAME static field from the class // //////////////////////////////////////////////// - Field tableNameField = aClass.getDeclaredField("TABLE_NAME"); + Field tableNameField = recordEntityClass.getDeclaredField("TABLE_NAME"); if(!tableNameField.getType().equals(String.class)) { - LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", aClass.getSimpleName())); + LOG.warn(warningPrefix + ", but whose TABLE_NAME field is not a String, so it will not be used.", logPair("class", recordEntityClass.getSimpleName())); return (rs); } @@ -303,6 +305,7 @@ public class MetaDataProducerHelper } RecordEntityToTableGenericMetaDataProducer producer = new RecordEntityToTableGenericMetaDataProducer(tableNameValue, recordEntityClass, tableMetaDataProductionCustomizer); + producer.setSourceClass(recordEntityClass); if(tableMetaDataCustomizer != null) { @@ -322,7 +325,9 @@ public class MetaDataProducerHelper //////////////////////////////////////// if(qMetaDataProducingEntity.producePossibleValueSource()) { - rs.add(new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue)); + PossibleValueSourceOfTableGenericMetaDataProducer producer = new PossibleValueSourceOfTableGenericMetaDataProducer(tableNameValue); + producer.setSourceClass(recordEntityClass); + rs.add(producer); } ////////////////////////// @@ -333,11 +338,11 @@ public class MetaDataProducerHelper Class childEntityClass = childTable.childTableEntityClass(); if(childTable.childJoin().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildJoin(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildJoin(recordEntityClass, childTable)); if(childTable.childRecordListWidget().enabled()) { - CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(aClass, childTable)); + CollectionUtils.addIfNotNull(rs, processChildRecordListWidget(recordEntityClass, childTable)); } } else @@ -347,7 +352,7 @@ public class MetaDataProducerHelper ////////////////////////////////////////////////////////////////////////// // if not doing the join, can't do the child-widget, so warn about that // ////////////////////////////////////////////////////////////////////////// - LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", aClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); + LOG.warn(warningPrefix + " requested to produce a ChildRecordListWidget, but not produce a Join - which is not allowed (must do join to do widget). ", logPair("class", recordEntityClass.getSimpleName()), logPair("childEntityClass", childEntityClass.getSimpleName())); } } } @@ -360,14 +365,16 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildRecordListWidget(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildRecordListWidget(Class sourceClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(sourceClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); ChildRecordListWidget childRecordListWidget = childTable.childRecordListWidget(); - return (new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget)); + ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer producer = new ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, childRecordListWidget); + producer.setSourceClass(sourceClass); + return producer; } @@ -397,20 +404,22 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processChildJoin(Class aClass, ChildTable childTable) throws Exception + private static MetaDataProducerInterface processChildJoin(Class entityClass, ChildTable childTable) throws Exception { Class childEntityClass = childTable.childTableEntityClass(); - String parentTableName = getTableNameStaticFieldValue(aClass); + String parentTableName = getTableNameStaticFieldValue(entityClass); String childTableName = getTableNameStaticFieldValue(childEntityClass); String possibleValueFieldName = findPossibleValueField(childEntityClass, parentTableName); if(!StringUtils.hasContent(possibleValueFieldName)) { - LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + aClass.getSimpleName() + "]"); + LOG.warn("Could not find field in [" + childEntityClass.getSimpleName() + "] with possibleValueSource referencing table [" + entityClass.getSimpleName() + "]"); return (null); } - return (new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName)); + ChildJoinFromRecordEntityGenericMetaDataProducer producer = new ChildJoinFromRecordEntityGenericMetaDataProducer(childTableName, parentTableName, possibleValueFieldName); + producer.setSourceClass(entityClass); + return producer; } @@ -418,18 +427,20 @@ public class MetaDataProducerHelper /*************************************************************************** ** ***************************************************************************/ - private static MetaDataProducerInterface processMetaDataProducer(Class aClass) throws Exception + private static MetaDataProducerInterface processMetaDataProducer(Class sourceCClass) throws Exception { - for(Constructor constructor : aClass.getConstructors()) + for(Constructor constructor : sourceCClass.getConstructors()) { if(constructor.getParameterCount() == 0) { Object o = constructor.newInstance(); - return (MetaDataProducerInterface) o; + MetaDataProducerInterface producer = (MetaDataProducerInterface) o; + producer.setSourceClass(sourceCClass); + return producer; } } - LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", aClass.getSimpleName())); + LOG.warn("Found a class which implements MetaDataProducerInterface, but it does not have a no-arg constructor, so it cannot be used.", logPair("class", sourceCClass.getSimpleName())); return null; } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java index fa725451..6aef051e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/MetaDataProducerInterface.java @@ -73,4 +73,23 @@ public interface MetaDataProducerInterface return (true); } + + /*************************************************************************** + * + ***************************************************************************/ + default void setSourceClass(Class sourceClass) + { + ////////// + // noop // + ////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + default Class getSourceClass() + { + return null; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java index 76c892cd..c1bc51cf 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildJoinFromRecordEntityGenericMetaDataProducer.java @@ -62,6 +62,7 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat private String parentTableName; // e.g., order private String foreignKeyFieldName; // e.g., orderId + private Class sourceClass; /*************************************************************************** @@ -102,4 +103,37 @@ public class ChildJoinFromRecordEntityGenericMetaDataProducer implements MetaDat return (join); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildJoinFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java index d9d7858c..6dac1a10 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer.java @@ -57,6 +57,8 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem private ChildRecordListWidget childRecordListWidget; + private Class sourceClass; + /*************************************************************************** @@ -111,4 +113,36 @@ public class ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer implem return (widget); } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public ChildRecordListWidgetFromRecordEntityGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java index 8cb96ec2..f43818b5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfEnumGenericMetaDataProducer.java @@ -40,6 +40,10 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer[] values; + private Class sourceClass; + + + /******************************************************************************* @@ -62,4 +66,37 @@ public class PossibleValueSourceOfEnumGenericMetaDataProducer getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfEnumGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java index c1656f35..7d194f64 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/PossibleValueSourceOfTableGenericMetaDataProducer.java @@ -37,6 +37,7 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { private final String tableName; + private Class sourceClass; /******************************************************************************* @@ -58,4 +59,38 @@ public class PossibleValueSourceOfTableGenericMetaDataProducer implements MetaDa { return (QPossibleValueSource.newForTable(tableName)); } + + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public PossibleValueSourceOfTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java index 21565148..20cf5be1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/producers/RecordEntityToTableGenericMetaDataProducer.java @@ -48,6 +48,7 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu private static MetaDataCustomizerInterface defaultMetaDataCustomizer = null; + private Class sourceClass; /******************************************************************************* @@ -154,4 +155,37 @@ public class RecordEntityToTableGenericMetaDataProducer implements MetaDataProdu RecordEntityToTableGenericMetaDataProducer.defaultMetaDataCustomizer = defaultMetaDataCustomizer; } + + + /******************************************************************************* + ** Getter for sourceClass + ** + *******************************************************************************/ + public Class getSourceClass() + { + return sourceClass; + } + + + + /******************************************************************************* + ** Setter for sourceClass + ** + *******************************************************************************/ + public void setSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + } + + + /******************************************************************************* + ** Fluent setter for sourceClass + ** + *******************************************************************************/ + public RecordEntityToTableGenericMetaDataProducer withSourceClass(Class sourceClass) + { + this.sourceClass = sourceClass; + return (this); + } + } From 4b0d093a4a06f35e42baff4a0858b8523ca26be2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:42:40 -0600 Subject: [PATCH 200/209] Add clearKey(key) --- .../backend/core/utils/memoization/Memoization.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java index 4ed6cbce..ac02e21b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/memoization/Memoization.java @@ -247,6 +247,16 @@ public class Memoization + /******************************************************************************* + ** + *******************************************************************************/ + public void clearKey(K key) + { + this.map.remove(key); + } + + + /******************************************************************************* ** Setter for timeoutSeconds ** From 4cbcd0a1497c781efc0b45b8e7b807ebbfe4f21c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:45:01 -0600 Subject: [PATCH 201/209] better handling of some - ranges; upper-case input string to match month/day names; handle '*' day of week; day-names in , case; hour w/ AM/PM in , case; join with commas and and. --- .../backend/core/scheduler/CronDescriber.java | 41 +++++++++---------- .../core/scheduler/CronDescriberTest.java | 19 ++++++--- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java index 2b2a2c56..40099988 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriber.java @@ -27,6 +27,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -94,7 +95,7 @@ public class CronDescriber ***************************************************************************/ public static String getDescription(String cronExpression) throws ParseException { - String[] parts = cronExpression.split("\\s+"); + String[] parts = cronExpression.trim().toUpperCase().split("\\s+"); if(parts.length < 6 || parts.length > 7) { throw new ParseException("Invalid cron expression: " + cronExpression, 0); @@ -173,15 +174,16 @@ public class CronDescriber { return "every month"; } + else if(month.contains("-")) + { + String[] parts = month.split("-"); + return String.format("%s to %s", MONTH_MAP.getOrDefault(parts[0], parts[0]), MONTH_MAP.getOrDefault(parts[1], parts[1])); + } else { - String[] months = month.split(","); - StringBuilder result = new StringBuilder(); - for(String m : months) - { - result.append(MONTH_MAP.getOrDefault(m, m)).append(", "); - } - return result.substring(0, result.length() - 2); + String[] months = month.split(","); + List monthNames = Arrays.stream(months).map(m -> MONTH_MAP.getOrDefault(m, m)).toList(); + return StringUtils.joinWithCommasAndAnd(monthNames); } } @@ -192,7 +194,7 @@ public class CronDescriber ***************************************************************************/ private static String describeDayOfWeek(String dayOfWeek) { - if(dayOfWeek.equals("?")) + if(dayOfWeek.equals("?") || dayOfWeek.equals("*")) { return "every day of the week"; } @@ -212,13 +214,9 @@ public class CronDescriber } else { - String[] days = dayOfWeek.split(","); - StringBuilder result = new StringBuilder(); - for(String d : days) - { - result.append(DAY_OF_WEEK_MAP.getOrDefault(d, d)).append(", "); - } - return result.substring(0, result.length() - 2); + String[] days = dayOfWeek.split(","); + List dayNames = Arrays.stream(days).map(d -> DAY_OF_WEEK_MAP.getOrDefault(d, d)).toList(); + return StringUtils.joinWithCommasAndAnd(dayNames); } } @@ -244,21 +242,22 @@ public class CronDescriber } else if(part.contains(",")) { + List partsList = Arrays.stream(part.split(",")).toList(); + if(label.equals("hour")) { - String[] parts = part.split(","); - List partList = Arrays.stream(parts).map(p -> hourToAmPm(p)).toList(); - return String.join(", ", partList); + List hourNames = partsList.stream().map(p -> hourToAmPm(p)).toList(); + return StringUtils.joinWithCommasAndAnd(hourNames); } else { if(label.equals("day")) { - return "days " + part.replace(",", ", "); + return "days " + StringUtils.joinWithCommasAndAnd(partsList); } else { - return part.replace(",", ", ") + " " + label + "s"; + return StringUtils.joinWithCommasAndAnd(partsList) + " " + label + "s"; } } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java index 35f1e3bf..3b259ce9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/scheduler/CronDescriberTest.java @@ -43,7 +43,7 @@ class CronDescriberTest extends BaseTest assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * * * ?")); assertEquals("At 0 seconds, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 * * * * ?")); assertEquals("At 0 seconds, 0 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 * * * ?")); - assertEquals("At 0 seconds, 0, 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); + assertEquals("At 0 seconds, 0 and 30 minutes, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0,30 * * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 0 * * ?")); assertEquals("At 0 seconds, 0 minutes, 1 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 1 * * ?")); assertEquals("At 0 seconds, 0 minutes, 11 AM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 11 * * ?")); @@ -51,18 +51,25 @@ class CronDescriberTest extends BaseTest assertEquals("At 0 seconds, 0 minutes, 1 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 13 * * ?")); assertEquals("At 0 seconds, 0 minutes, 11 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 23 * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on day 10 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10 * ?")); - assertEquals("At 0 seconds, 0 minutes, midnight, on days 10, 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); + assertEquals("At 0 seconds, 0 minutes, midnight, on days 10 and 20 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10,20 * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on days from 10 to 15 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 10-15 * ?")); assertEquals("At from 10 to 15 seconds, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("10-15 0 0 * * ?")); assertEquals("At 30 seconds, 30 minutes, from 8 AM to 4 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("30 30 8-16 * * ?")); assertEquals("At 0 seconds, 0 minutes, midnight, on every 3 days starting at 0 of every month, every day of the week.", CronDescriber.getDescription("0 0 0 */3 * ?")); assertEquals("At every 5 seconds starting at 0, 0 minutes, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0/5 0 0 * * ?")); assertEquals("At 0 seconds, every 30 minutes starting at 3, midnight, on every day of every month, every day of the week.", CronDescriber.getDescription("0 3/30 0 * * ?")); - assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Monday, Wednesday, and Friday.", CronDescriber.getDescription("0 0 0 * * MON,WED,FRI")); assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, from Monday to Friday.", CronDescriber.getDescription("0 0 0 * * MON-FRI")); - assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday, Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); - assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); - assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, 52 minutes, every hour, on every day of January, March, September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + assertEquals("At 0 seconds, 0 minutes, midnight, on every day of every month, Sunday and Saturday.", CronDescriber.getDescription("0 0 0 * * 1,7")); + assertEquals("At 0 seconds, 0 minutes, 2 AM, 6 AM, noon, 4 PM, and 8 PM, on every day of every month, every day of the week.", CronDescriber.getDescription("0 0 2,6,12,16,20 * * ?")); + assertEquals("At every 5 seconds starting at 0, 14, 18, 3-39, and 52 minutes, every hour, on every day of January, March, and September, from Monday to Friday, in 2002-2010.", CronDescriber.getDescription("0/5 14,18,3-39,52 * ? JAN,MAR,SEP MON-FRI 2002-2010")); + + assertEquals("At every second, every minute, every hour, on every day of every month, every day of the week.", CronDescriber.getDescription("* * * ? * *")); + assertEquals("At every second, every minute, every hour, on every day of January to June, every day of the week.", CronDescriber.getDescription("* * * ? 1-6 *")); + assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * * 1,3,5 * *")); + // todo fix has 2-4 hours and 3 PM, s/b 2 AM to 4 AM and 3 PM assertEquals("At every second, every minute, every hour, on days 1, 3, and 5 of every month, every day of the week.", CronDescriber.getDescription("* * 2-4,15 1,3,5 * *")); + // hour failing on 3,2-7 (at least in TS side?) + // 3,2-7 makes 3,2 to July } } \ No newline at end of file From d4d20e2b2016f40f99c0f33ecaf751dd43b9891f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 28 Feb 2025 19:53:18 -0600 Subject: [PATCH 202/209] Fix this test that would never have worked on 3/1 of a non-leap year, i suppose --- .../actions/tables/query/expressions/NowWithOffsetTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java index 4655255d..affa9325 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/expressions/NowWithOffsetTest.java @@ -66,7 +66,7 @@ class NowWithOffsetTest extends BaseTest assertThat(twoWeeksFromNowMillis).isCloseTo(now + (14 * DAY_IN_MILLIS), allowedDiff); long oneMonthAgoMillis = ((Instant) NowWithOffset.minus(1, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); - assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusOneDay); + assertThat(oneMonthAgoMillis).isCloseTo(now - (30 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); // two days, to work on 3/1... long twoMonthsFromNowMillis = ((Instant) NowWithOffset.plus(2, ChronoUnit.MONTHS).evaluate(dateTimeField)).toEpochMilli(); assertThat(twoMonthsFromNowMillis).isCloseTo(now + (60 * DAY_IN_MILLIS), allowedDiffPlusTwoDays); From 3a8bfe5f486e02ffd8706748e201cdeac9028d75 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 3 Mar 2025 07:49:57 -0600 Subject: [PATCH 203/209] Minor cleanup from code review (comments, fixed a few exceptions); --- .../qqq/backend/core/actions/tables/InsertAction.java | 2 +- .../core/actions/values/SearchPossibleValueSourceAction.java | 2 +- .../qqq/backend/core/instances/QInstanceEnricher.java | 3 ++- .../metadata/tables/TablesCustomPossibleValueProvider.java | 3 ++- qqq-backend-module-filesystem/pom.xml | 1 + 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 846ed476..5ce2ce38 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -114,7 +114,7 @@ public class InsertAction extends AbstractQActionFunction { diff --git a/qqq-backend-module-filesystem/pom.xml b/qqq-backend-module-filesystem/pom.xml index 9344db4a..c389a018 100644 --- a/qqq-backend-module-filesystem/pom.xml +++ b/qqq-backend-module-filesystem/pom.xml @@ -75,6 +75,7 @@ test + net.java.dev.jna jna 5.7.0 From e3c89a80caed3f8fca77bffb348cafd6aef5e52c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 6 Mar 2025 11:40:09 -0600 Subject: [PATCH 204/209] Update qqq-frontend-material-dashboard to 0.24.0 [skip ci] --- qqq-sample-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index c80d1bbd..42224882 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,7 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - 0.24.0-SNAPSHOT + 0.24.0 com.h2database From 51c588d2def048c5b7fc4159b9a4fd2385ce057e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 6 Mar 2025 11:48:03 -0600 Subject: [PATCH 205/209] Update versions for release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c3f02707..17af52f5 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ - 0.24.0-SNAPSHOT + 0.24.0 UTF-8 UTF-8 From 1a5a374c4eb4ea9d9e4e5478408702a705b4af6c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 6 Mar 2025 11:48:05 -0600 Subject: [PATCH 206/209] Update for next development version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 17af52f5..bc43a583 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ - 0.24.0 + 0.25.0-SNAPSHOT UTF-8 UTF-8 From 04e13413efd6741a2637891dc50bf290b8465a88 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 6 Mar 2025 12:07:40 -0600 Subject: [PATCH 207/209] Updating to 0.25.0 --- qqq-dev-tools/CURRENT-SNAPSHOT-VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION index 2094a100..d21d277b 100644 --- a/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION +++ b/qqq-dev-tools/CURRENT-SNAPSHOT-VERSION @@ -1 +1 @@ -0.24.0 +0.25.0 From e0045bb212eb9528a03c00bf6e5160b1dca439d7 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Thu, 6 Mar 2025 16:28:51 -0600 Subject: [PATCH 208/209] updated ses sender to consider adding label to from if provided --- .../metadata/messaging/ses/SendSESAction.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java index 53e13f54..f3b3e2de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/messaging/ses/SendSESAction.java @@ -216,11 +216,16 @@ public class SendSESAction { LOG.warn("More than one FROM value was found, will send using the first one found [" + partyList.get(0).getAddress() + "]."); } + Party fromParty = partyList.get(0); + if(fromParty.getAddress() == null) + { + throw (new QException("Cannot send SES message because a FROM address was not provided.")); + } ///////////////////////////// // return the from address // ///////////////////////////// - return (partyList.get(0).getAddress()); + return (getFullEmailAddress(fromParty)); } @@ -267,15 +272,15 @@ public class SendSESAction { if(EmailPartyRole.CC.equals(party.getRole())) { - ccList.add(party.getAddress()); + ccList.add(getFullEmailAddress(party)); } else if(EmailPartyRole.BCC.equals(party.getRole())) { - bccList.add(party.getAddress()); + bccList.add(getFullEmailAddress(party)); } else if(party.getRole() == null || PartyRole.Default.DEFAULT.equals(party.getRole()) || EmailPartyRole.TO.equals(party.getRole())) { - toList.add(party.getAddress()); + toList.add(getFullEmailAddress(party)); } else { @@ -332,4 +337,22 @@ public class SendSESAction return amazonSES; } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String getFullEmailAddress(Party party) + { + if(party.getLabel() != null) + { + return (party.getLabel() + " <" + party.getAddress() + ">"); + } + + ///////////////////////////// + // return the from address // + ///////////////////////////// + return (party.getAddress()); + } } From 491fcd6d252b880742a520ce9441da9ce1e6a630 Mon Sep 17 00:00:00 2001 From: Tim Chamberlain Date: Fri, 7 Mar 2025 10:08:38 -0600 Subject: [PATCH 209/209] updated run backend step action to look for record id value string if no records in the input --- .../processes/RunBackendStepAction.java | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java index 119df559..bc0c205c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunBackendStepAction.java @@ -36,6 +36,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -46,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; @@ -188,21 +191,40 @@ public class RunBackendStepAction { if(CollectionUtils.nullSafeIsEmpty(runBackendStepInput.getRecords())) { - QueryInput queryInput = new QueryInput(); - queryInput.setTableName(inputMetaData.getRecordListMetaData().getTableName()); + QTableMetaData table = QContext.getQInstance().getTable(inputMetaData.getRecordListMetaData().getTableName()); + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(table.getName()); - // todo - handle this being async (e.g., http) - // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data - // then this step can re-run, hopefully with the needed data. - - QProcessCallback callback = runBackendStepInput.getCallback(); - if(callback == null) + ////////////////////////////////////////////////// + // look for record ids in the input data values // + ////////////////////////////////////////////////// + String recordIds = (String) runBackendStepInput.getValue("recordIds"); + if(recordIds == null) { - throw (new QUserFacingException("Missing input records.", - new QException("Function is missing input records, but no callback was present to request fields from a user"))); + recordIds = (String) runBackendStepInput.getValue("recordId"); } - queryInput.setFilter(callback.getQueryFilter()); + /////////////////////////////////////////////////////////// + // if records were found, add as criteria to query input // + /////////////////////////////////////////////////////////// + if(recordIds != null) + { + queryInput.setFilter(new QQueryFilter(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, recordIds.split(",")))); + } + else + { + // todo - handle this being async (e.g., http) + // seems like it just needs to throw, breaking this flow, and to send a response to the frontend, directing it to prompt the user for the needed data + // then this step can re-run, hopefully with the needed data. + QProcessCallback callback = runBackendStepInput.getCallback(); + if(callback == null) + { + throw (new QUserFacingException("Missing input records.", + new QException("Function is missing input records, but no callback was present to request fields from a user"))); + } + + queryInput.setFilter(callback.getQueryFilter()); + } ////////////////////////////////////////////////////////////////////////////////////////// // if process has a max-no of records, set a limit on the process of that number plus 1 // @@ -210,7 +232,7 @@ public class RunBackendStepAction ////////////////////////////////////////////////////////////////////////////////////////// if(process.getMaxInputRecords() != null) { - if(callback.getQueryFilter() == null) + if(queryInput.getFilter() == null) { queryInput.setFilter(new QQueryFilter()); }