From 73200b2fd2c5e27fe7e209a16368c40e784c1a23 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:13:12 -0600 Subject: [PATCH 01/84] CE-1955 Mark as Serializable --- .../core/model/actions/tables/storage/StorageInput.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java index 5407faeb..19d84ce3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java @@ -22,13 +22,14 @@ package com.kingsrook.qqq.backend.core.model.actions.tables.storage; +import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; /******************************************************************************* ** Input for Storage actions. *******************************************************************************/ -public class StorageInput extends AbstractTableActionInput +public class StorageInput extends AbstractTableActionInput implements Serializable { private String reference; From 7d058530d53fdb382565e409fde7e5f208712d90 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:13:34 -0600 Subject: [PATCH 02/84] CE-1955 Initial checkin --- .../SavedBulkLoadProfile.java | 285 ++++++++++++++++++ .../SavedBulkLoadProfileMetaDataProvider.java | 159 ++++++++++ .../SharedSavedBulkLoadProfile.java | 268 ++++++++++++++++ 3 files changed, 712 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java new file mode 100644 index 00000000..65f07c77 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfile.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.tables.TablesPossibleValueSourceMetaDataProvider; + + +/******************************************************************************* + ** Entity bean for the savedBulkLoadProfile table + *******************************************************************************/ +public class SavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "savedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(isRequired = true, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.TRUNCATE_ELLIPSIS, label = "Profile Name") + private String label; + + @QField(possibleValueSourceName = TablesPossibleValueSourceMetaDataProvider.NAME, maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, label = "Table", isRequired = true) + private String tableName; + + @QField(maxLength = 250, valueTooLongBehavior = ValueTooLongBehavior.ERROR, dynamicDefaultValueBehavior = DynamicDefaultValueBehavior.USER_ID, label = "Owner") + private String userId; + + @QField(label = "Mapping JSON") + private String mappingJson; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public Integer getId() + { + return id; + } + + + + /******************************************************************************* + ** Setter for id + ** + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Getter for createDate + ** + *******************************************************************************/ + public Instant getCreateDate() + { + return createDate; + } + + + + /******************************************************************************* + ** Setter for createDate + ** + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Getter for modifyDate + ** + *******************************************************************************/ + public Instant getModifyDate() + { + return modifyDate; + } + + + + /******************************************************************************* + ** Setter for modifyDate + ** + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** Setter for label + ** + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + ** + *******************************************************************************/ + public SavedBulkLoadProfile withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + ** + *******************************************************************************/ + public String getTableName() + { + return tableName; + } + + + + /******************************************************************************* + ** Setter for tableName + ** + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + ** + *******************************************************************************/ + public SavedBulkLoadProfile withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for userId + ** + *******************************************************************************/ + public String getUserId() + { + return userId; + } + + + + /******************************************************************************* + ** Setter for userId + ** + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + ** + *******************************************************************************/ + public SavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + + /******************************************************************************* + ** Getter for mappingJson + *******************************************************************************/ + public String getMappingJson() + { + return (this.mappingJson); + } + + + + /******************************************************************************* + ** Setter for mappingJson + *******************************************************************************/ + public void setMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + } + + + + /******************************************************************************* + ** Fluent setter for mappingJson + *******************************************************************************/ + public SavedBulkLoadProfile withMappingJson(String mappingJson) + { + this.mappingJson = mappingJson; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java new file mode 100644 index 00000000..d8d376e1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -0,0 +1,159 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableAudienceType; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareableTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileMetaDataProvider +{ + public static final String SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE = "sharedSavedBulkLoadProfileJoinSavedBulkLoadProfile"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void defineAll(QInstance instance, String recordTablesBackendName, Consumer backendDetailEnricher) throws QException + { + instance.addTable(defineSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addPossibleValueSource(QPossibleValueSource.newForTable(SavedBulkLoadProfile.TABLE_NAME)); + + ///////////////////////////////////// + // todo - param to enable sharing? // + ///////////////////////////////////// + instance.addTable(defineSharedSavedBulkLoadProfileTable(recordTablesBackendName, backendDetailEnricher)); + instance.addJoin(defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile()); + if(instance.getPossibleValueSource(ShareScopePossibleValueMetaDataProducer.NAME) == null) + { + instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private QJoinMetaData defineSharedSavedBulkLoadProfileJoinSavedBulkLoadProfile() + { + return (new QJoinMetaData() + .withName(SHARED_SAVED_BULK_LOAD_PROFILE_JOIN_SAVED_BULK_LOAD_PROFILE) + .withLeftTable(SharedSavedBulkLoadProfile.TABLE_NAME) + .withRightTable(SavedBulkLoadProfile.TABLE_NAME) + .withType(JoinType.MANY_TO_ONE) + .withJoinOn(new JoinOn("savedBulkLoadProfileId", "id"))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SavedBulkLoadProfile.TABLE_NAME) + .withLabel("Bulk Load Profile") + .withIcon(new QIcon().withName("drive_folder_upload")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("label") + .withBackendName(backendName) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SavedBulkLoadProfile.class) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("mappingJson")).withIsHidden(true)) + .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId")).withIsHidden(true)) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + // todo - want one of these? + // table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + + table.withShareableTableMetaData(new ShareableTableMetaData() + .withSharedRecordTableName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withAssetIdFieldName("savedBulkLoadProfileId") + .withScopeFieldName("scope") + .withThisTableOwnerIdFieldName("userId") + .withAudienceType(new ShareableAudienceType().withName("user").withFieldName("userId"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public QTableMetaData defineSharedSavedBulkLoadProfileTable(String backendName, Consumer backendDetailEnricher) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(SharedSavedBulkLoadProfile.TABLE_NAME) + .withLabel("Shared Bulk Load Profile") + .withIcon(new QIcon().withName("share")) + .withRecordLabelFormat("%s") + .withRecordLabelFields("savedBulkLoadProfileId") + .withBackendName(backendName) + .withUniqueKey(new UniqueKey("savedBulkLoadProfileId", "userId")) + .withPrimaryKeyField("id") + .withFieldsFromEntity(SharedSavedBulkLoadProfile.class) + // todo - security key + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) + .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "savedBulkLoadProfileId", "userId"))) + .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("scope"))) + .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + if(backendDetailEnricher != null) + { + backendDetailEnricher.accept(table); + } + + return (table); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java new file mode 100644 index 00000000..a02e5c3b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SharedSavedBulkLoadProfile.java @@ -0,0 +1,268 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.time.Instant; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.sharing.ShareScopePossibleValueMetaDataProducer; + + +/******************************************************************************* + ** Entity bean for the shared saved bulk load profile table + *******************************************************************************/ +public class SharedSavedBulkLoadProfile extends QRecordEntity +{ + public static final String TABLE_NAME = "sharedSavedBulkLoadProfile"; + + @QField(isEditable = false) + private Integer id; + + @QField(isEditable = false) + private Instant createDate; + + @QField(isEditable = false) + private Instant modifyDate; + + @QField(possibleValueSourceName = SavedBulkLoadProfile.TABLE_NAME, label = "Bulk Load Profile") + private Integer savedBulkLoadProfileId; + + @QField(label = "User") + private String userId; + + @QField(possibleValueSourceName = ShareScopePossibleValueMetaDataProducer.NAME) + private String scope; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile() + { + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public SharedSavedBulkLoadProfile(QRecord qRecord) throws QException + { + populateFromQRecord(qRecord); + } + + + + + /******************************************************************************* + ** Getter for id + *******************************************************************************/ + public Integer getId() + { + return (this.id); + } + + + + /******************************************************************************* + ** Setter for id + *******************************************************************************/ + public void setId(Integer id) + { + this.id = id; + } + + + + /******************************************************************************* + ** Fluent setter for id + *******************************************************************************/ + public SharedSavedBulkLoadProfile withId(Integer id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + ** Getter for createDate + *******************************************************************************/ + public Instant getCreateDate() + { + return (this.createDate); + } + + + + /******************************************************************************* + ** Setter for createDate + *******************************************************************************/ + public void setCreateDate(Instant createDate) + { + this.createDate = createDate; + } + + + + /******************************************************************************* + ** Fluent setter for createDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withCreateDate(Instant createDate) + { + this.createDate = createDate; + return (this); + } + + + + /******************************************************************************* + ** Getter for modifyDate + *******************************************************************************/ + public Instant getModifyDate() + { + return (this.modifyDate); + } + + + + /******************************************************************************* + ** Setter for modifyDate + *******************************************************************************/ + public void setModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + } + + + + /******************************************************************************* + ** Fluent setter for modifyDate + *******************************************************************************/ + public SharedSavedBulkLoadProfile withModifyDate(Instant modifyDate) + { + this.modifyDate = modifyDate; + return (this); + } + + + + + /******************************************************************************* + ** Getter for userId + *******************************************************************************/ + public String getUserId() + { + return (this.userId); + } + + + + /******************************************************************************* + ** Setter for userId + *******************************************************************************/ + public void setUserId(String userId) + { + this.userId = userId; + } + + + + /******************************************************************************* + ** Fluent setter for userId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withUserId(String userId) + { + this.userId = userId; + return (this); + } + + + + /******************************************************************************* + ** Getter for scope + *******************************************************************************/ + public String getScope() + { + return (this.scope); + } + + + + /******************************************************************************* + ** Setter for scope + *******************************************************************************/ + public void setScope(String scope) + { + this.scope = scope; + } + + + + /******************************************************************************* + ** Fluent setter for scope + *******************************************************************************/ + public SharedSavedBulkLoadProfile withScope(String scope) + { + this.scope = scope; + return (this); + } + + + + /******************************************************************************* + ** Getter for savedBulkLoadProfileId + *******************************************************************************/ + public Integer getSavedBulkLoadProfileId() + { + return (this.savedBulkLoadProfileId); + } + + + + /******************************************************************************* + ** Setter for savedBulkLoadProfileId + *******************************************************************************/ + public void setSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + } + + + + /******************************************************************************* + ** Fluent setter for savedBulkLoadProfileId + *******************************************************************************/ + public SharedSavedBulkLoadProfile withSavedBulkLoadProfileId(Integer savedBulkLoadProfileId) + { + this.savedBulkLoadProfileId = savedBulkLoadProfileId; + return (this); + } + + +} From 7ba205a26267a81e6e48d9604b7e6c54ddf8e53b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:16:59 -0600 Subject: [PATCH 03/84] CE-1955 Initial checkin --- .../insert/BulkInsertReceiveFileStep.java | 81 +++ .../insert/BulkInsertReceiveMappingStep.java | 59 ++ .../bulk/insert/BulkInsertStepUtils.java | 58 ++ .../bulk/insert/BulkInsertV2ExtractStep.java | 126 +++++ .../bulk/insert/BulkLoadFileRow.java | 167 ++++++ .../AbstractIteratorBasedFileToRows.java | 125 +++++ .../insert/filehandling/CsvFileToRows.java | 112 ++++ .../filehandling/FileToRowsInterface.java | 74 +++ .../insert/filehandling/XlsxFileToRows.java | 102 ++++ .../insert/mapping/BulkInsertMapping.java | 503 ++++++++++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 75 +++ .../insert/mapping/RowsToRecordInterface.java | 69 +++ .../bulk/insert/mapping/TallRowsToRecord.java | 316 +++++++++++ .../bulk/insert/mapping/ValueMapper.java | 79 +++ .../bulk/insert/mapping/WideRowsToRecord.java | 333 ++++++++++++ .../filehandling/CsvFileToRowsTest.java | 60 +++ .../insert/filehandling/TestFileToRows.java | 86 +++ .../filehandling/XlsxFileToRowsTest.java | 106 ++++ .../insert/mapping/FlatRowsToRecordTest.java | 150 ++++++ .../insert/mapping/TallRowsToRecordTest.java | 284 ++++++++++ .../bulk/insert/mapping/ValueMapperTest.java | 134 +++++ .../insert/mapping/WideRowsToRecordTest.java | 305 +++++++++++ 22 files changed, 3404 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java new file mode 100644 index 00000000..6ebcf3a8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + + List bodyRows = new ArrayList<>(); + while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) + { + bodyRows.add(fileToRowsInterface.next().toString()); + } + + runBackendStepOutput.addValue("header", headerRow.toString()); + runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); + System.out.println("Done receiving file"); + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java new file mode 100644 index 00000000..0071aa49 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(true); + bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name" + )); + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + // probably need to what, receive the mapping object, store it into state + // what, do we maybe return to a different sub-mapping screen (e.g., values) + // then at some point - cool - proceed to ETL's steps + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java new file mode 100644 index 00000000..e3dac892 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertStepUtils +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException + { + @SuppressWarnings("unchecked") + ArrayList storageInputs = (ArrayList) input.getValue("theFile"); + if(storageInputs == null) + { + throw (new QException("StorageInputs for theFile were not found in process state")); + } + + if(storageInputs.isEmpty()) + { + throw (new QException("StorageInputs for theFile was an empty list")); + } + + StorageInput storageInput = storageInputs.get(0); + return (storageInput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java new file mode 100644 index 00000000..2e49e48a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; + + +/******************************************************************************* + ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. + *******************************************************************************/ +public class BulkInsertV2ExtractStep extends AbstractExtractStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); + + if(page.size() > remainingLimit) + { + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java new file mode 100644 index 00000000..487f606a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + + +/******************************************************************************* + ** A row of values, e.g., from a file, for bulk-load + *******************************************************************************/ +public class BulkLoadFileRow implements Serializable +{ + private Serializable[] values; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadFileRow(Serializable[] values) + { + this.values = values; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public int size() + { + if(values == null) + { + return (0); + } + + return (values.length); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public boolean hasIndex(int i) + { + if(values == null) + { + return (false); + } + + if(i >= values.length || i < 0) + { + return (false); + } + + return (true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValue(int i) + { + if(values == null) + { + throw new IllegalStateException("Row has no values"); + } + + if(i >= values.length || i < 0) + { + throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValueElseNull(int i) + { + if(!hasIndex(i)) + { + return (null); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + if(values == null) + { + return ("null"); + } + + return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(",")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BulkLoadFileRow row = (BulkLoadFileRow) o; + return Objects.deepEquals(values, row.values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int hashCode() + { + return Arrays.hashCode(values); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java new file mode 100644 index 00000000..383dfa21 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -0,0 +1,125 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.util.Iterator; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private Iterator iterator; + + private boolean useLast = false; + private BulkLoadFileRow last; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean hasNext() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + return true; + } + + return iterator.hasNext(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow next() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + useLast = false; + return (this.last); + } + + E e = iterator.next(); + + BulkLoadFileRow row = makeRow(e); + + this.last = row; + return (this.last); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract BulkLoadFileRow makeRow(E e); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void unNext() + { + useLast = true; + } + + + + /******************************************************************************* + ** Getter for iterator + *******************************************************************************/ + public Iterator getIterator() + { + return (this.iterator); + } + + + + /******************************************************************************* + ** Setter for iterator + *******************************************************************************/ + public void setIterator(Iterator iterator) + { + this.iterator = iterator; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java new file mode 100644 index 00000000..f39f4d45 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class CsvFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private CSVParser csvParser; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static CsvFileToRows forString(String csv) throws QException + { + CsvFileToRows csvFileToRows = new CsvFileToRows(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes()); + csvFileToRows.init(byteArrayInputStream); + + return (csvFileToRows); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT + .withIgnoreSurroundingSpaces() + ); + setIterator(csvParser.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening CSV Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(CSVRecord csvRecord) + { + Serializable[] values = new Serializable[csvRecord.size()]; + int i = 0; + for(String s : csvRecord) + { + values[i++] = s; + } + + return (new BulkLoadFileRow(values)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(csvParser != null) + { + csvParser.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java new file mode 100644 index 00000000..d2d6c78a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.util.Iterator; +import java.util.Locale; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface FileToRowsInterface extends AutoCloseable, Iterator +{ + + /*************************************************************************** + ** + ***************************************************************************/ + static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException + { + FileToRowsInterface rs; + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) + { + rs = new CsvFileToRows(); + } + else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx")) + { + rs = new XlsxFileToRows(); + } + else + { + throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx")); + } + + rs.init(inputStream); + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + void init(InputStream inputStream) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + void unNext(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java new file mode 100644 index 00000000..21c9d928 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -0,0 +1,102 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.stream.Stream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.Sheet; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private ReadableWorkbook workbook; + private Stream rows; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + workbook = new ReadableWorkbook(inputStream); + Sheet sheet = workbook.getFirstSheet(); + + rows = sheet.openStream(); + setIterator(rows.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening XLSX Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow) + { + Serializable[] values = new Serializable[readerRow.getCellCount()]; + + for(int i = 0; i < readerRow.getCellCount(); i++) + { + values[i] = readerRow.getCell(i).getText(); + } + + return new BulkLoadFileRow(values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(workbook != null) + { + workbook.close(); + } + + if(rows != null) + { + rows.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java new file mode 100644 index 00000000..d95d48e4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -0,0 +1,503 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertMapping implements Serializable +{ + private String tableName; + private Boolean hasHeaderRow; + + private Layout layout; + + ///////////////////////////////////////////////////////////////////// + // keys in here are: // + // fieldName (for the main table) // + // association.fieldName (for an associated child table) // + // association.association.fieldName (for grandchild associations) // + ///////////////////////////////////////////////////////////////////// + private Map fieldNameToHeaderNameMap = new HashMap<>(); + private Map fieldNameToIndexMap = new HashMap<>(); + private Map fieldNameToDefaultValueMap = new HashMap<>(); + private Map> fieldNameToValueMapping = new HashMap<>(); + + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private List mappedAssociations = new ArrayList<>(); + + private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum Layout + { + FLAT(FlatRowsToRecord::new), + TALL(TallRowsToRecord::new), + WIDE(WideRowsToRecord::new); + + + /*************************************************************************** + ** + ***************************************************************************/ + private final Supplier supplier; + + + + /*************************************************************************** + ** + ***************************************************************************/ + Layout(Supplier supplier) + { + this.supplier = supplier; + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public RowsToRecordInterface newRowsToRecordInterface() + { + return (supplier.get()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + if(hasHeaderRow && fieldNameToHeaderNameMap != null) + { + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + } + else if(fieldNameToIndexMap != null) + { + return (fieldNameToIndexMap); + } + + throw (new QException("Mapping was not properly configured.")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map> getFieldNameToValueMappingForTable(String associatedTableName) + { + Map> rs = new HashMap<>(); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet()) + { + if(shouldProcessFieldForTable(entry.getKey(), associatedTableName)) + { + String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey(); + rs.put(key, entry.getValue()); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain) + { + return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p -> + { + List fieldNameParts = new ArrayList<>(); + List associationParts = new ArrayList<>(); + + if(StringUtils.hasContent(fieldNameWithAssociationPrefix)) + { + fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\."))); + } + + if(StringUtils.hasContent(associationChain)) + { + associationParts.addAll(Arrays.asList(associationChain.split("\\."))); + } + + if(!fieldNameParts.isEmpty()) + { + fieldNameParts.remove(fieldNameParts.size() - 1); + } + + return (fieldNameParts.equals(associationParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + { + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(field.getName(), headerIndex); + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkInsertMapping withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMap + *******************************************************************************/ + public Map getFieldNameToHeaderNameMap() + { + return (this.fieldNameToHeaderNameMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public void setFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToIndexMap + *******************************************************************************/ + public Map getFieldNameToIndexMap() + { + return (this.fieldNameToIndexMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToIndexMap + *******************************************************************************/ + public void setFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToIndexMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for mappedAssociations + *******************************************************************************/ + public List getMappedAssociations() + { + return (this.mappedAssociations); + } + + + + /******************************************************************************* + ** Setter for mappedAssociations + *******************************************************************************/ + public void setMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + } + + + + /******************************************************************************* + ** Fluent setter for mappedAssociations + *******************************************************************************/ + public BulkInsertMapping withMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToDefaultValueMap + *******************************************************************************/ + public Map getFieldNameToDefaultValueMap() + { + if(this.fieldNameToDefaultValueMap == null) + { + this.fieldNameToDefaultValueMap = new HashMap<>(); + } + + return (this.fieldNameToDefaultValueMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public void setFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToValueMapping + *******************************************************************************/ + public Map> getFieldNameToValueMapping() + { + return (this.fieldNameToValueMapping); + } + + + + /******************************************************************************* + ** Setter for fieldNameToValueMapping + *******************************************************************************/ + public void setFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToValueMapping + *******************************************************************************/ + public BulkInsertMapping withFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public Layout getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(Layout layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkInsertMapping withLayout(Layout layout) + { + this.layout = layout; + return (this); + } + + + + /******************************************************************************* + ** Getter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public Map> getTallLayoutGroupByIndexMap() + { + return (this.tallLayoutGroupByIndexMap); + } + + + + /******************************************************************************* + ** Setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public void setTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public BulkInsertMapping withTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java new file mode 100644 index 00000000..ac98adf4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FlatRowsToRecord implements RowsToRecordInterface +{ + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java new file mode 100644 index 00000000..91481980 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RowsToRecordInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + { + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + Serializable value = null; + if(index != null && row != null) + { + value = row.getValueElseNull(index); + } + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + { + value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + } + + if(value != null) + { + record.setValue(fieldName, value); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java new file mode 100644 index 00000000..1dbccd92 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -0,0 +1,316 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TallRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + + String associationNameChain = ""; + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + recordGroupByValues = null; + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // i wrote this condition in here: && rs.size() < limit // + // but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... // + ///////////////////////////////////////////////////////////////////////////////////////////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + QRecord record = new QRecord(); + + Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); + + ////////////////////////////////////////////////////// + // get all rows for the main table from the 0th row // + ////////////////////////////////////////////////////// + BulkLoadFileRow row = rows.get(0); + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + List rs = new ArrayList<>(); + + QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName()); + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + for(BulkLoadFileRow row : rows) + { + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + } + + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + + ////////////////////////////////////////////////// + // use the current row to start the next record // + ////////////////////////////////////////////////// + rowsForCurrentRecord.add(row); + recordGroupByValues = rowGroupByValues; + } + } + + /////////// + // final // + /////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getGroupByValues(BulkLoadFileRow row, List indexes) + { + List rowGroupByValues = new ArrayList<>(); + boolean haveAnyGroupByValues = false; + for(Integer index : indexes) + { + Serializable value = row.getValueElseNull(index); + rowGroupByValues.add(value); + + if(value != null && !"".equals(value)) + { + haveAnyGroupByValues = true; + } + } + + if(!haveAnyGroupByValues) + { + return (null); + } + + return (rowGroupByValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java new file mode 100644 index 00000000..09e074d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ValueMapper +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping) + { + valueMapping(records, mapping, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry> entry : mappingForTable.entrySet()) + { + String fieldName = entry.getKey(); + Map map = entry.getValue(); + String value = record.getValueString(fieldName); + if(value != null && map.containsKey(value)) + { + record.setValue(fieldName, map.get(value)); + } + } + + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java new file mode 100644 index 00000000..40e37071 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java @@ -0,0 +1,333 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException + { + for(String associationName : mapping.getMappedAssociations()) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + // List associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record); + List associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + { + List rs = new ArrayList<>(); + + Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + { + if(entry.getKey().startsWith(associationName + ".")) + { + String fieldName = entry.getKey().substring(associationName.length() + 1); + + ////////////////////////////////////////////////////////////////////////// + // make sure the name here is for this table - not a sub-table under it // + ////////////////////////////////////////////////////////////////////////// + if(!fieldName.contains(".")) + { + fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + } + } + } + + ///////////////////////////////////////////////////////////////////// + // loop over the length of the record, building associated records // + ///////////////////////////////////////////////////////////////////// + QRecord associatedRecord = new QRecord(); + Set processedFieldNames = new HashSet<>(); + boolean gotAnyValues = false; + int subStartIndex = -1; + + for(int i = startIndex; i < endIndex; i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + { + if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + { + /////////////////////////////////////////////// + // ok - this is a value for this association // + /////////////////////////////////////////////// + if(subStartIndex == -1) + { + subStartIndex = i; + } + + String fieldName = entry.getKey(); + if(processedFieldNames.contains(fieldName)) + { + ///////////////////////////////////////////////// + // this means we're starting a new sub-record! // + ///////////////////////////////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + rs.add(associatedRecord); + } + + associatedRecord = new QRecord(); + processedFieldNames = new HashSet<>(); + gotAnyValues = false; + subStartIndex = i + 1; + } + + processedFieldNames.add(fieldName); + + Serializable value = row.getValueElseNull(i); + if(value != null && !"".equals(value)) + { + gotAnyValues = true; + } + + setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + } + } + } + + //////////////////////// + // handle final value // + //////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + rs.add(associatedRecord); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + } + } + } + + /*************************************************************************** + ** + ***************************************************************************/ + // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException + // { + // List rs = new ArrayList<>(); + // String associationNameChainForRecursiveCalls = associationName; + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); + // } + // } + + // Map> indexes = new HashMap<>(); + // for(int i = 0; i < headerRow.size(); i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); + // } + // } + // } + + // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); + + // ////////////////////////////////////////////////////// + // // figure out how many sub-rows we'll be processing // + // ////////////////////////////////////////////////////// + // for(int i = 0; i < maxIndex; i++) + // { + // QRecord associatedRecord = new QRecord(); + // boolean gotAnyValues = false; + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // String fieldName = entry.getKey(); + // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) + // { + // Integer index = indexes.get(fieldName).get(i); + // Serializable value = row.getValueElseNull(index); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); + // } + // } + + // if(gotAnyValues) + // { + // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); + // rs.add(associatedRecord); + // } + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java new file mode 100644 index 00000000..84bdce2f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for CsvFileToRows + *******************************************************************************/ +class CsvFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] csvBytes = """ + one,two,three + 1,2,3,4 + """.getBytes(); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }), headerRow); + assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }), bodyRow); + assertFalse(fileToRowsInterface.hasNext()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java new file mode 100644 index 00000000..e1142e89 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class TestFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private final List rows; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TestFileToRows(List rows) + { + this.rows = rows; + setIterator(this.rows.iterator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + /////////// + // noop! // + /////////// + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + /////////// + // noop! // + /////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(Serializable[] values) + { + return (new BulkLoadFileRow(values)); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java new file mode 100644 index 00000000..8a381ad1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.time.LocalDate; +import java.time.Month; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for XlsxFileToRows + *******************************************************************************/ +class XlsxFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] byteArray = writeExcelBytes(); + + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow); + assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow); + + /////////////////////////////////////////////////////////////////////////////////////// + // make sure there's at least a limit (less than 20) to how many more rows there are // + /////////////////////////////////////////////////////////////////////////////////////// + int otherRowCount = 0; + while(fileToRowsInterface.hasNext() && otherRowCount < 20) + { + fileToRowsInterface.next(); + otherRowCount++; + } + assertFalse(fileToRowsInterface.hasNext()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static byte[] writeExcelBytes() throws QException + { + ReportFormat format = ReportFormat.XLSX; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport()); + GenerateReportActionTest.insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + + byte[] byteArray = baos.toByteArray(); + return byteArray; + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java new file mode 100644 index 00000000..e5474032 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FlatRowsToRecord + *******************************************************************************/ +class FlatRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToHeaderNameMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" }, + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "cost", "cost" + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { 1, "Homer", "Simpson", true }, + new Serializable[] { 2, "Marge", "Simpson", false }, + new Serializable[] { 3, "Bart", "Simpson", "A" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1 } + )); + + BulkLoadFileRow header = null; + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java new file mode 100644 index 00000000..dfd872f5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -0,0 +1,284 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for TallRowsToRecord + *******************************************************************************/ +class TallRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLines() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsic() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart + 1, , , BEER, 500, Coupon Code, 10QOff + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException + { + Integer DEFAULT_STORE_ID = 101; + Integer DEFAULT_LINE_NO = 102; + String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, , , DONUT, , Coupon Code, 10QOff, Size, Large + 1, , , BEER, 500, , , Flavor, Hops + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5), + "orderLine.extrinsics", List.of(7) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testShouldProcessAssociation() + { + TallRowsToRecord tallRowsToRecord = new TallRowsToRecord(); + assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz")); + + assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java new file mode 100644 index 00000000..9e4108e9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for ValueMapper + *******************************************************************************/ +class ValueMapperTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( + "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), + "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), + "lineItem.sku", Map.of("ABC", "Alphabet"), + "lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "extrinsics.key", Map.of("1", "one", "2", "two") + )); + + QRecord inputRecord = new QRecord() + .withValue("storeId", "QQQMart") + .withValue("shipToName", "HoJu") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "ABC") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "foo") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "bar") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", 1) + .withValue("value", "foo") + ); + JSONObject beforeJson = recordToJson(inputRecord); + + QRecord expectedRecord = new QRecord() + .withValue("storeId", 1) + .withValue("shipToName", "Homer") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "Alphabet") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "bar") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "baz") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "one") + .withValue("value", "foo") + ); + JSONObject expectedJson = recordToJson(expectedRecord); + + ValueMapper.valueMapping(List.of(inputRecord), mapping); + JSONObject actualJson = recordToJson(inputRecord); + + System.out.println("Before"); + System.out.println(beforeJson.toString(3)); + System.out.println("Actual"); + System.out.println(actualJson.toString(3)); + System.out.println("Expected"); + System.out.println(expectedJson.toString(3)); + + assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static JSONObject recordToJson(QRecord record) + { + JSONObject jsonObject = new JSONObject(); + for(Map.Entry valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet()) + { + jsonObject.put(valueEntry.getKey(), valueEntry.getValue()); + } + for(Map.Entry> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + JSONArray jsonArray = new JSONArray(); + for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue())) + { + jsonArray.put(recordToJson(associationRecord)); + } + jsonObject.put(associationEntry.getKey(), jsonArray); + } + return (jsonObject); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java new file mode 100644 index 00000000..990ad1ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java @@ -0,0 +1,305 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderAndLines(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testOrderLinesAndOrderExtrinsic(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2, Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException + { + Integer DEFAULT_STORE_ID = 42; + Integer DEFAULT_LINE_NO = 47; + String DEFAULT_LINE_EXTRA_VALUE = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file From d8a0a6c68d00ee27308db29e5a1b661d64fdf915 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:43:17 -0600 Subject: [PATCH 04/84] CE-1955 Move prime-test-database into mainline, to be loaded when javalin starts --- qqq-sample-project/pom.xml | 1 - .../sampleapp/SampleJavalinServer.java | 2 + .../metadata/SampleMetaDataProvider.java | 184 +++++++++++++++++- .../resources/prime-test-database.sql | 21 ++ 4 files changed, 206 insertions(+), 2 deletions(-) rename qqq-sample-project/src/{test => main}/resources/prime-test-database.sql (82%) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index d7e8e2bf..2ff7aee7 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -74,7 +74,6 @@ com.h2database h2 2.2.220 - test diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index 7ec4a911..37f3e3d1 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -64,6 +64,8 @@ public class SampleJavalinServer { qInstance = SampleMetaDataProvider.defineInstance(); + SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); + QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); javalinService = Javalin.create(config -> { diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index ec2aa618..37526837 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -22,7 +22,10 @@ package com.kingsrook.sampleapp.metadata; +import java.io.InputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,7 +40,10 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; import com.kingsrook.qqq.backend.core.model.metadata.branding.QBrandingMetaData; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -48,8 +54,13 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QuickSightChartMe import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; @@ -58,6 +69,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMet import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; @@ -70,9 +82,13 @@ import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinali import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3TableBackendDetails; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.sampleapp.dashboard.widgets.PersonsByCreateDateBarChart; import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; +import org.apache.commons.io.IOUtils; /******************************************************************************* @@ -80,7 +96,7 @@ import com.kingsrook.sampleapp.processes.clonepeople.ClonePeopleTransformStep; *******************************************************************************/ public class SampleMetaDataProvider { - public static boolean USE_MYSQL = true; + public static boolean USE_MYSQL = false; public static final String RDBMS_BACKEND_NAME = "rdbms"; public static final String FILESYSTEM_BACKEND_NAME = "filesystem"; @@ -97,6 +113,7 @@ public class SampleMetaDataProvider public static final String PROCESS_NAME_SLEEP_INTERACTIVE = "sleepInteractive"; public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_PET = "pet"; public static final String TABLE_NAME_CARRIER = "carrier"; public static final String TABLE_NAME_CITY = "city"; @@ -106,6 +123,9 @@ public class SampleMetaDataProvider public static final String SCREEN_0 = "screen0"; public static final String SCREEN_1 = "screen1"; + public static final String BACKEND_NAME_UPLOAD_ARCHIVE = "uploadArchive"; + public static final String UPLOAD_FILE_ARCHIVE_TABLE_NAME = "uploadFileArchive"; + /******************************************************************************* @@ -120,6 +140,10 @@ public class SampleMetaDataProvider qInstance.addBackend(defineFilesystemBackend()); qInstance.addTable(defineTableCarrier()); qInstance.addTable(defineTablePerson()); + qInstance.addPossibleValueSource(QPossibleValueSource.newForTable(TABLE_NAME_PERSON)); + qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum(PetSpecies.NAME, PetSpecies.values())); + qInstance.addTable(defineTablePet()); + qInstance.addJoin(defineTablePersonJoinPet()); qInstance.addTable(defineTableCityFile()); qInstance.addProcess(defineProcessGreetPeople()); qInstance.addProcess(defineProcessGreetPeopleInteractive()); @@ -128,6 +152,9 @@ public class SampleMetaDataProvider qInstance.addProcess(defineProcessScreenThenSleep()); qInstance.addProcess(defineProcessSimpleThrow()); + qInstance.add(defineUploadArchiveBackend()); + qInstance.add(defineTableUploadFileArchive()); + MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, SampleMetaDataProvider.class.getPackageName()); defineWidgets(qInstance); @@ -139,6 +166,26 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + public static void primeTestDatabase(String sqlFileName) throws Exception + { + try(Connection connection = ConnectionManager.getConnection(SampleMetaDataProvider.defineRdbmsBackend())) + { + InputStream primeTestDatabaseSqlStream = SampleMetaDataProvider.class.getResourceAsStream("/" + sqlFileName); + List lines = IOUtils.readLines(primeTestDatabaseSqlStream, StandardCharsets.UTF_8); + lines = lines.stream().filter(line -> !line.startsWith("-- ")).toList(); + String joinedSQL = String.join("\n", lines); + for(String sql : joinedSQL.split(";")) + { + QueryManager.executeUpdate(connection, sql); + } + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -192,6 +239,7 @@ public class SampleMetaDataProvider .withIcon(new QIcon().withName("emoji_people")) .withChild(qInstance.getProcess(PROCESS_NAME_GREET).withIcon(new QIcon().withName("emoji_people"))) .withChild(qInstance.getTable(TABLE_NAME_PERSON).withIcon(new QIcon().withName("person"))) + .withChild(qInstance.getTable(TABLE_NAME_PET).withIcon(new QIcon().withName("pets"))) .withChild(qInstance.getTable(TABLE_NAME_CITY).withIcon(new QIcon().withName("location_city"))) .withChild(qInstance.getProcess(PROCESS_NAME_GREET_INTERACTIVE).withIcon(new QIcon().withName("waving_hand"))) .withWidgets(List.of(PersonsByCreateDateBarChart.class.getSimpleName(), QuickSightChartRenderer.class.getSimpleName())) @@ -340,11 +388,62 @@ public class SampleMetaDataProvider QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + qTableMetaData.withAssociation(new Association() + .withAssociatedTableName(TABLE_NAME_PET) + .withName("pets") + .withJoinName(QJoinMetaData.makeInferredJoinName(TABLE_NAME_PERSON, TABLE_NAME_PET))); + return (qTableMetaData); } + /******************************************************************************* + ** + *******************************************************************************/ + public static QTableMetaData defineTablePet() + { + QTableMetaData qTableMetaData = new QTableMetaData() + .withName(TABLE_NAME_PET) + .withLabel("Pet") + .withBackendName(RDBMS_BACKEND_NAME) + .withPrimaryKeyField("id") + .withRecordLabelFormat("%s %s") + .withRecordLabelFields("name") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withBackendName("create_date").withIsEditable(false)) + .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date").withIsEditable(false)) + .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name").withIsRequired(true)) + .withField(new QFieldMetaData("personId", QFieldType.INTEGER).withBackendName("person_id").withIsRequired(true).withPossibleValueSourceName(TABLE_NAME_PERSON)) + .withField(new QFieldMetaData("speciesId", QFieldType.INTEGER).withBackendName("species_id").withIsRequired(true).withPossibleValueSourceName(PetSpecies.NAME)) + .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) + + .withSection(new QFieldSection("identity", "Identity", new QIcon("badge"), Tier.T1, List.of("id", "name"))) + .withSection(new QFieldSection("basicInfo", "Basic Info", new QIcon("dataset"), Tier.T2, List.of("personId", "speciesId", "birthDate"))) + .withSection(new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); + + QInstanceEnricher.setInferredFieldBackendNames(qTableMetaData); + + return (qTableMetaData); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static QJoinMetaData defineTablePersonJoinPet() + { + return new QJoinMetaData() + .withLeftTable(TABLE_NAME_PERSON) + .withRightTable(TABLE_NAME_PET) + .withInferredName() + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "personId")); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -524,6 +623,40 @@ public class SampleMetaDataProvider + /******************************************************************************* + ** + *******************************************************************************/ + public static QBackendMetaData defineUploadArchiveBackend() + { + return new FilesystemBackendMetaData() + .withName(BACKEND_NAME_UPLOAD_ARCHIVE) + .withBasePath("/tmp/" + BACKEND_NAME_UPLOAD_ARCHIVE); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static QTableMetaData defineTableUploadFileArchive() + { + return (new QTableMetaData() + .withName(UPLOAD_FILE_ARCHIVE_TABLE_NAME) + .withBackendName(BACKEND_NAME_UPLOAD_ARCHIVE) + .withPrimaryKeyField("fileName") + // .withSupplementalMetaData(new ApiTableMetaDataContainer()) // empty container means no apis for this table + .withField(new QFieldMetaData("fileName", QFieldType.STRING)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)) + .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE)) + .withBackendDetails(new S3TableBackendDetails() + .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") + .withBasePath("upload-file-archive"))); + } + + + /******************************************************************************* ** Testing backend step - just sleeps however long you ask it to (or, throws if ** you don't provide a number of seconds to sleep). @@ -625,4 +758,53 @@ public class SampleMetaDataProvider } } + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum PetSpecies implements PossibleValueEnum + { + DOG(1, "Dog"), + CAT(1, "Cat"); + + private final Integer id; + private final String label; + + public static final String NAME = "petSpecies"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + PetSpecies(int id, String label) + { + this.id = id; + this.label = label; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public Integer getPossibleValueId() + { + return (id); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (label); + } + } + } diff --git a/qqq-sample-project/src/test/resources/prime-test-database.sql b/qqq-sample-project/src/main/resources/prime-test-database.sql similarity index 82% rename from qqq-sample-project/src/test/resources/prime-test-database.sql rename to qqq-sample-project/src/main/resources/prime-test-database.sql index 10185156..1f5e6bc9 100644 --- a/qqq-sample-project/src/test/resources/prime-test-database.sql +++ b/qqq-sample-project/src/main/resources/prime-test-database.sql @@ -42,6 +42,27 @@ INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, a INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (4, 'Tyler', 'Samples', NULL, 'tsamples@mmltholdings.com', 1, 950000, 75); INSERT INTO person (id, first_name, last_name, birth_date, email, is_employed, annual_salary, days_worked) VALUES (5, 'Garret', 'Richardson', '1981-01-01', 'grichardson@mmltholdings.com', 0, 1500000, 1); +DROP TABLE IF EXISTS pet; +CREATE TABLE pet +( + id INT AUTO_INCREMENT primary key , + create_date TIMESTAMP DEFAULT now(), + modify_date TIMESTAMP DEFAULT now(), + + name VARCHAR(80) NOT NULL, + species_id INTEGER NOT NULL, + person_id INTEGER NOT NULL, + birth_date DATE +); + +INSERT INTO pet (id, name, species_id, person_id) VALUES (1, 'Charlie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (2, 'Coco', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (3, 'Louie', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (4, 'Barkley', 1, 1); +INSERT INTO pet (id, name, species_id, person_id) VALUES (5, 'Toby', 1, 2); +INSERT INTO pet (id, name, species_id, person_id) VALUES (6, 'Mae', 2, 3); + + DROP TABLE IF EXISTS carrier; CREATE TABLE carrier ( From e809c773f9ae0c7759a5473afa101c11997d29d5 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:44:00 -0600 Subject: [PATCH 05/84] CE-1955 Switch to handle uploaded files via StorageAction into the uploadFileArchive table --- .../javalin/QJavalinProcessHandler.java | 72 +++++++++---------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 48dfef38..7576c64f 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.io.PipedOutputStream; import java.io.Serializable; import java.time.LocalDate; @@ -49,24 +50,22 @@ import com.kingsrook.qqq.backend.core.actions.processes.CancelProcessAction; import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback; import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; -import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; -import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -78,9 +77,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMeta import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ExceptionUtils; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -490,7 +486,7 @@ public class QJavalinProcessHandler ** todo - make query params have a "field-" type of prefix?? ** *******************************************************************************/ - private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException + private static void populateRunProcessRequestWithValuesFromContext(Context context, RunProcessInput runProcessInput) throws IOException, QException { ////////////////////////// // process query string // @@ -521,20 +517,41 @@ public class QJavalinProcessHandler //////////////////////////// // process uploaded files // //////////////////////////// - for(UploadedFile uploadedFile : context.uploadedFiles()) + for(Map.Entry> entry : context.uploadedFileMap().entrySet()) { - try(InputStream content = uploadedFile.content()) + String name = entry.getKey(); + List uploadedFiles = entry.getValue(); + ArrayList storageInputs = new ArrayList<>(); + runProcessInput.addValue(name, storageInputs); + + String storageTableName = QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName(); + if(!StringUtils.hasContent(storageTableName)) { - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes(content.readAllBytes()); - qUploadedFile.setFilename(uploadedFile.filename()); + throw (new QException("UploadFileArchiveTableName was not specified in javalinMetaData. Cannot accept file uploads.")); + } - UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(key, qUploadedFile); - LOG.info("Stored uploaded file in TempFileStateProvider under key: " + key); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key); + for(UploadedFile uploadedFile : uploadedFiles) + { + String reference = QValueFormatter.formatDate(LocalDate.now()) + + File.separator + runProcessInput.getProcessName() + + File.separator + UUID.randomUUID() + "-" + uploadedFile.filename(); - archiveUploadedFile(runProcessInput, qUploadedFile); + StorageInput storageInput = new StorageInput(storageTableName).withReference(reference); + storageInputs.add(storageInput); + + try + ( + InputStream content = uploadedFile.content(); + OutputStream outputStream = new StorageAction().createOutputStream(storageInput); + ) + { + content.transferTo(outputStream); + LOG.info("Streamed uploaded file", logPair("storageTable", storageTableName), logPair("reference", reference), logPair("processName", runProcessInput.getProcessName()), logPair("uploadFileName", uploadedFile.filename())); + } + catch(QException e) + { + throw (new QException("Error creating output stream in table [" + storageTableName + "] for storage action", e)); + } } } @@ -565,27 +582,6 @@ public class QJavalinProcessHandler - /******************************************************************************* - ** - *******************************************************************************/ - private static void archiveUploadedFile(RunProcessInput runProcessInput, QUploadedFile qUploadedFile) - { - String fileName = QValueFormatter.formatDate(LocalDate.now()) - + File.separator + runProcessInput.getProcessName() - + File.separator + qUploadedFile.getFilename(); - - InsertInput insertInput = new InsertInput(); - insertInput.setTableName(QJavalinImplementation.javalinMetaData.getUploadedFileArchiveTableName()); - insertInput.setRecords(List.of(new QRecord() - .withValue("fileName", fileName) - .withValue("contents", qUploadedFile.getBytes()) - )); - - new InsertAction().executeAsync(insertInput); - } - - - /******************************************************************************* ** *******************************************************************************/ From 5f081fce44098e6c1a407facb7b446af4ffca1c8 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 09:48:02 -0600 Subject: [PATCH 06/84] CE-1955 Checkstyle! --- .../insert/mapping/TallRowsToRecordTest.java | 24 +++++++++---------- .../insert/mapping/WideRowsToRecordTest.java | 22 ++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index dfd872f5..c623ddda 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -167,9 +167,9 @@ class TallRowsToRecordTest extends BaseTest @Test void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException { - Integer DEFAULT_STORE_ID = 101; - Integer DEFAULT_LINE_NO = 102; - String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file"; + Integer defaultStoreId = 101; + Integer defaultLineNo = 102; + String defaultOrderLineExtraSource = "file"; CsvFileToRows fileToRows = CsvFileToRows.forString(""" orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value @@ -197,9 +197,9 @@ class TallRowsToRecordTest extends BaseTest "orderLine.extrinsics.value", "Line Extrinsic Value" )) .withFieldNameToDefaultValueMap(Map.of( - "storeId", DEFAULT_STORE_ID, - "orderLine.lineNumber", DEFAULT_LINE_NO, - "orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource )) .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) .withTallLayoutGroupByIndexMap(Map.of( @@ -219,17 +219,17 @@ class TallRowsToRecordTest extends BaseTest QRecord order = records.get(0); assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); lineItem = order.getAssociatedRecords().get("orderLine").get(1); assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); @@ -238,16 +238,16 @@ class TallRowsToRecordTest extends BaseTest order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); lineItem = order.getAssociatedRecords().get("orderLine").get(0); assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java index 990ad1ef..51a6c5f7 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java @@ -222,9 +222,9 @@ class WideRowsToRecordTest extends BaseTest ***************************************************************************/ private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException { - Integer DEFAULT_STORE_ID = 42; - Integer DEFAULT_LINE_NO = 47; - String DEFAULT_LINE_EXTRA_VALUE = "bar"; + Integer defaultStoreId = 42; + Integer defaultLineNo = 47; + String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); @@ -245,9 +245,9 @@ class WideRowsToRecordTest extends BaseTest .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) .withFieldNameToDefaultValueMap(Map.of( - "storeId", DEFAULT_STORE_ID, - "orderLine.lineNumber", DEFAULT_LINE_NO, - "orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue )) .withTableName(TestUtils.TABLE_NAME_ORDER) .withLayout(BulkInsertMapping.Layout.WIDE) @@ -259,10 +259,10 @@ class WideRowsToRecordTest extends BaseTest QRecord order = records.get(0); assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -276,15 +276,15 @@ class WideRowsToRecordTest extends BaseTest lineItem = order.getAssociatedRecords().get("orderLine").get(2); assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); lineItem = order.getAssociatedRecords().get("orderLine").get(0); From da2be57a17eabf5e4e66066eb88b006e090e1521 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 10:00:19 -0600 Subject: [PATCH 07/84] CE-1955 Add fastexcel-reader (and a pinned version of commons-io, for compatibility) --- qqq-backend-core/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index c9a3c8f5..0944e1e9 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -102,6 +102,11 @@ fastexcel 0.12.15 + + org.dhatim + fastexcel-reader + 0.18.4 + org.apache.poi poi @@ -112,6 +117,14 @@ poi-ooxml 5.2.5 + + + + commons-io + commons-io + 2.16.0 + + com.auth0 auth0 From 4b590b5653a97e908aea9c77bec2a8096fdb765e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 12 Nov 2024 10:02:44 -0600 Subject: [PATCH 08/84] CE-1955 make public stuff used by another test now --- .../core/actions/reporting/GenerateReportActionTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 1cf8404a..984a9aa2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -88,7 +88,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; *******************************************************************************/ public class GenerateReportActionTest extends BaseTest { - private static final String REPORT_NAME = "personReport1"; + public static final String REPORT_NAME = "personReport1"; @@ -651,7 +651,7 @@ public class GenerateReportActionTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private static QReportMetaData defineTableOnlyReport() + public static QReportMetaData defineTableOnlyReport() { QReportMetaData report = new QReportMetaData() .withName(REPORT_NAME) From 39b322336f55a23d585af59ecc42929509c562af Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 16:06:38 -0600 Subject: [PATCH 09/84] CE-1955 Add transaction to validateSecurityFields --- .../qqq/backend/core/actions/tables/DeleteAction.java | 2 +- .../qqq/backend/core/actions/tables/InsertAction.java | 2 +- .../qqq/backend/core/actions/tables/UpdateAction.java | 4 ++-- .../helpers/ValidateRecordSecurityLockHelper.java | 10 ++++++---- .../implementations/memory/MemoryRecordStore.java | 2 +- .../scripts/StoreScriptRevisionProcessStep.java | 2 +- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java index f964b62e..0ded3c93 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/DeleteAction.java @@ -320,7 +320,7 @@ public class DeleteAction QTableMetaData table = deleteInput.getTable(); List primaryKeysNotFound = validateRecordsExistAndCanBeAccessed(deleteInput, oldRecordList.get()); - ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE); + ValidateRecordSecurityLockHelper.validateSecurityFields(table, oldRecordList.get(), ValidateRecordSecurityLockHelper.Action.DELETE, deleteInput.getTransaction()); /////////////////////////////////////////////////////////////////////////// // after all validations, run the pre-delete customizer, if there is one // diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 75b17a22..1c799b34 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -258,7 +258,7 @@ public class InsertAction extends AbstractQActionFunction records, Action action) throws QException + public static void validateSecurityFields(QTableMetaData table, List records, Action action, QBackendTransaction transaction) throws QException { MultiRecordSecurityLock locksToCheck = getRecordSecurityLocks(table, action); if(locksToCheck == null || CollectionUtils.nullSafeIsEmpty(locksToCheck.getLocks())) @@ -101,7 +102,7 @@ public class ValidateRecordSecurityLockHelper // actually check lock values // //////////////////////////////// Map errorRecords = new HashMap<>(); - evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, locksToCheck, errorRecords, new ArrayList<>(), madeUpPrimaryKeys, transaction); ///////////////////////////////// // propagate errors to records // @@ -141,7 +142,7 @@ public class ValidateRecordSecurityLockHelper ** BUT - WRITE locks - in their case, we read the record no matter what, and in ** here we need to verify we have a key that allows us to WRITE the record. *******************************************************************************/ - private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys) throws QException + private static void evaluateRecordLocks(QTableMetaData table, List records, Action action, RecordSecurityLock recordSecurityLock, Map errorRecords, List treePosition, Map madeUpPrimaryKeys, QBackendTransaction transaction) throws QException { if(recordSecurityLock instanceof MultiRecordSecurityLock multiRecordSecurityLock) { @@ -152,7 +153,7 @@ public class ValidateRecordSecurityLockHelper for(RecordSecurityLock childLock : CollectionUtils.nonNullList(multiRecordSecurityLock.getLocks())) { treePosition.add(i); - evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys); + evaluateRecordLocks(table, records, action, childLock, errorRecords, treePosition, madeUpPrimaryKeys, transaction); treePosition.remove(treePosition.size() - 1); i++; } @@ -225,6 +226,7 @@ public class ValidateRecordSecurityLockHelper // query will be like (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) OR (fkey1=? and fkey2=?) // //////////////////////////////////////////////////////////////////////////////////////////////// QueryInput queryInput = new QueryInput(); + queryInput.setTransaction(transaction); queryInput.setTableName(leftMostJoin.getLeftTable()); QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR); queryInput.setFilter(filter); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java index 7591becb..b53d69a2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/memory/MemoryRecordStore.java @@ -193,7 +193,7 @@ public class MemoryRecordStore if(recordMatches) { qRecord.setErrors(new ArrayList<>()); - ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT); + ValidateRecordSecurityLockHelper.validateSecurityFields(input.getTable(), List.of(qRecord), ValidateRecordSecurityLockHelper.Action.SELECT, null); if(CollectionUtils.nullSafeHasContents(qRecord.getErrors())) { ////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java index 96a2e43f..b042360c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/scripts/StoreScriptRevisionProcessStep.java @@ -101,7 +101,7 @@ public class StoreScriptRevisionProcessStep implements BackendStep /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // in case the app added a security field to the scripts table, make sure the user is allowed to edit the script // /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE); + ValidateRecordSecurityLockHelper.validateSecurityFields(QContext.getQInstance().getTable(Script.TABLE_NAME), List.of(script), ValidateRecordSecurityLockHelper.Action.UPDATE, transaction); if(CollectionUtils.nullSafeHasContents(script.getErrors())) { throw (new QPermissionDeniedException(script.getErrors().get(0).getMessage())); From 6aafc3d55392f019d66188407c249efbdde96322 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 20:15:47 -0600 Subject: [PATCH 10/84] CE-1955 Mark Serializable --- .../core/model/metadata/frontend/QFrontendFieldMetaData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java index 54b8aba6..6f424b31 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/frontend/QFrontendFieldMetaData.java @@ -42,7 +42,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; * *******************************************************************************/ @JsonInclude(Include.NON_NULL) -public class QFrontendFieldMetaData +public class QFrontendFieldMetaData implements Serializable { private String name; private String label; From 062240a0a5ce3f5f9451aed94a63b2ea9bf49e76 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 18 Nov 2024 20:16:09 -0600 Subject: [PATCH 11/84] CE-1955 Add BULK_LOAD_* values --- .../backend/core/model/metadata/processes/QComponentType.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java index d4bd7ff7..c15eba2a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QComponentType.java @@ -29,6 +29,9 @@ public enum QComponentType { HELP_TEXT, BULK_EDIT_FORM, + BULK_LOAD_FILE_MAPPING_FORM, + BULK_LOAD_VALUE_MAPPING_FORM, + BULK_LOAD_PROFILE_FORM, VALIDATION_REVIEW_SCREEN, EDIT_FORM, VIEW_FORM, From c09198eed599dc5b05b5f976fbad628e40d397d7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:37:05 -0600 Subject: [PATCH 12/84] CE-1955 Initial checkin --- .../SavedBulkLoadProfileMetaDataProvider.java | 10 + .../DeleteSavedBulkLoadProfileProcess.java | 88 +++++++++ .../QuerySavedBulkLoadProfileProcess.java | 129 ++++++++++++ .../StoreSavedBulkLoadProfileProcess.java | 171 ++++++++++++++++ .../SavedBulkLoadProfileProcessTests.java | 186 ++++++++++++++++++ 5 files changed, 584 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index d8d376e1..23826835 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -40,6 +40,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.DeleteSavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.QuerySavedBulkLoadProfileProcess; +import com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles.StoreSavedBulkLoadProfileProcess; /******************************************************************************* @@ -68,6 +71,13 @@ public class SavedBulkLoadProfileMetaDataProvider { instance.addPossibleValueSource(new ShareScopePossibleValueMetaDataProducer().produce(new QInstance())); } + + //////////////////////////////////// + // processes for working with 'em // + //////////////////////////////////// + instance.add(StoreSavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(QuerySavedBulkLoadProfileProcess.getProcessMetaData()); + instance.add(DeleteSavedBulkLoadProfileProcess.getProcessMetaData()); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..e909751e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/DeleteSavedBulkLoadProfileProcess.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; + + +/******************************************************************************* + ** Process used by the delete bulkLoadProfile dialog + *******************************************************************************/ +public class DeleteSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(DeleteSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("deleteSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(DeleteSavedBulkLoadProfileProcess.class)) + .withName("delete") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + DeleteInput input = new DeleteInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKeys(List.of(savedBulkLoadProfileId)); + new DeleteAction().execute(input); + } + catch(Exception e) + { + LOG.warn("Error deleting saved bulkLoadProfile", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..c1bdaa41 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/QuerySavedBulkLoadProfileProcess.java @@ -0,0 +1,129 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialogs + *******************************************************************************/ +public class QuerySavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(QuerySavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("querySavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(QuerySavedBulkLoadProfileProcess.class)) + .withName("query") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("id"); + + try + { + if(savedBulkLoadProfileId != null) + { + GetInput input = new GetInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setPrimaryKey(savedBulkLoadProfileId); + + GetOutput output = new GetAction().execute(input); + if(output.getRecord() == null) + { + throw (new QNotFoundException("The requested bulkLoadProfile was not found.")); + } + + runBackendStepOutput.addRecord(output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfile", output.getRecord()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) List.of(output.getRecord())); + } + else + { + String tableName = runBackendStepInput.getValueString("tableName"); + + QueryInput input = new QueryInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName)) + .withOrderBy(new QFilterOrderBy("label"))); + + QueryOutput output = new QueryAction().execute(input); + runBackendStepOutput.setRecords(output.getRecords()); + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) output.getRecords()); + } + } + catch(QNotFoundException qnfe) + { + LOG.info("BulkLoadProfile not found", logPair("savedBulkLoadProfileId", savedBulkLoadProfileId)); + throw (qnfe); + } + catch(Exception e) + { + LOG.warn("Error querying for saved bulkLoadProfiles", e); + throw (e); + } + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java new file mode 100644 index 00000000..0d8f33d4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/StoreSavedBulkLoadProfileProcess.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.ActionHelper; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Process used by the saved bulkLoadProfile dialog + *******************************************************************************/ +public class StoreSavedBulkLoadProfileProcess implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(StoreSavedBulkLoadProfileProcess.class); + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static QProcessMetaData getProcessMetaData() + { + return (new QProcessMetaData() + .withName("storeSavedBulkLoadProfile") + .withStepList(List.of( + new QBackendStepMetaData() + .withCode(new QCodeReference(StoreSavedBulkLoadProfileProcess.class)) + .withName("store") + ))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + ActionHelper.validateSession(runBackendStepInput); + + try + { + String userId = QContext.getQSession().getUser().getIdReference(); + String tableName = runBackendStepInput.getValueString("tableName"); + String label = runBackendStepInput.getValueString("label"); + + String mappingJson = processMappingJson(runBackendStepInput.getValueString("mappingJson")); + + QRecord qRecord = new QRecord() + .withValue("id", runBackendStepInput.getValueInteger("id")) + .withValue("mappingJson", mappingJson) + .withValue("label", label) + .withValue("tableName", tableName) + .withValue("userId", userId); + + List savedBulkLoadProfileList; + if(qRecord.getValueInteger("id") == null) + { + checkForDuplicates(userId, tableName, label, null); + + InsertInput input = new InsertInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + InsertOutput output = new InsertAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + else + { + checkForDuplicates(userId, tableName, label, qRecord.getValueInteger("id")); + + UpdateInput input = new UpdateInput(); + input.setTableName(SavedBulkLoadProfile.TABLE_NAME); + input.setRecords(List.of(qRecord)); + + UpdateOutput output = new UpdateAction().execute(input); + savedBulkLoadProfileList = output.getRecords(); + } + + runBackendStepOutput.addValue("savedBulkLoadProfileList", (Serializable) savedBulkLoadProfileList); + } + catch(Exception e) + { + LOG.warn("Error storing saved bulkLoadProfile", e); + throw (e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String processMappingJson(String mappingJson) + { + return mappingJson; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static void checkForDuplicates(String userId, String tableName, String label, Integer id) throws QException + { + QueryInput queryInput = new QueryInput(); + queryInput.setTableName(SavedBulkLoadProfile.TABLE_NAME); + queryInput.setFilter(new QQueryFilter( + new QFilterCriteria("userId", QCriteriaOperator.EQUALS, userId), + new QFilterCriteria("tableName", QCriteriaOperator.EQUALS, tableName), + new QFilterCriteria("label", QCriteriaOperator.EQUALS, label))); + + if(id != null) + { + queryInput.getFilter().addCriteria(new QFilterCriteria("id", QCriteriaOperator.NOT_EQUALS, id)); + } + + QueryOutput queryOutput = new QueryAction().execute(queryInput); + if(CollectionUtils.nullSafeHasContents(queryOutput.getRecords())) + { + throw (new QUserFacingException("You already have a saved Bulk Load Profile on this table with this name.")); + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java new file mode 100644 index 00000000..a1692327 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/savedbulkloadprofiles/SavedBulkLoadProfileProcessTests.java @@ -0,0 +1,186 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.savedbulkloadprofiles; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit tests for all saved-bulk-load-profile processes + *******************************************************************************/ +class SavedBulkLoadProfileProcessTests extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + QInstance qInstance = QContext.getQInstance(); + new SavedBulkLoadProfileMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + String tableName = TestUtils.TABLE_NAME_PERSON_MEMORY; + + { + //////////////////////////////////////////// + // query - should be no profiles to start // + //////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + + Integer savedBulkLoadProfileId; + { + ///////////////////////// + // store a new profile // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + savedBulkLoadProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + assertNotNull(savedBulkLoadProfileId); + + ////////////////////////////////////////////////////////////////// + // try to store it again - should throw a "duplicate" exception // + ////////////////////////////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ////////////////////////////////////// + // query - should find our profiles // + ////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + { + //////////////////////// + // update our Profile // + //////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + assertEquals(1, savedBulkLoadProfileList.size()); + assertEquals(1, savedBulkLoadProfileList.get(0).getValueInteger("id")); + assertEquals("My Updated Profile", savedBulkLoadProfileList.get(0).getValueString("label")); + } + + Integer anotherSavedProfileId; + { + ///////////////////////////////////////////////////////////////////////////////////////////// + // store a second one w/ different name (will be used below in update-dupe-check use-case) // + ///////////////////////////////////////////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("label", "My Second Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + List savedBulkLoadProfileList = (List) runProcessOutput.getValues().get("savedBulkLoadProfileList"); + anotherSavedProfileId = savedBulkLoadProfileList.get(0).getValueInteger("id"); + } + + { + ///////////////////////////////////////////////// + // try to rename the second to match the first // + ///////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(StoreSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessInput.addValue("label", "My Updated Profile"); + runProcessInput.addValue("tableName", tableName); + runProcessInput.addValue("mappingJson", JsonUtils.toJson(new BulkLoadProfile())); + + ////////////////////////////////////////// + // should throw a "duplicate" exception // + ////////////////////////////////////////// + assertThatThrownBy(() -> new RunProcessAction().execute(runProcessInput)) + .isInstanceOf(QUserFacingException.class) + .hasMessageContaining("already have a saved Bulk Load Profile"); + } + + { + ///////////////////////// + // delete our profiles // + ///////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(DeleteSavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("id", savedBulkLoadProfileId); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + + runProcessInput.addValue("id", anotherSavedProfileId); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + } + + { + ///////////////////////////////////////// + // query - should be no profiles again // + ///////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(QuerySavedBulkLoadProfileProcess.getProcessMetaData().getName()); + runProcessInput.addValue("tableName", tableName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(0, ((List) runProcessOutput.getValues().get("savedBulkLoadProfileList")).size()); + } + } + +} \ No newline at end of file From b684f2409b27e383627f432ac48fff714980134e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:37:36 -0600 Subject: [PATCH 13/84] CE-1955 Avoid type-based exceptions checking security key values --- .../backend/core/model/session/QSession.java | 39 ++++++++++++++++--- .../core/model/session/QSessionTest.java | 14 ++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java index cf050a18..284942ba 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/session/QSession.java @@ -48,7 +48,7 @@ public class QSession implements Serializable, Cloneable private QUser user; private String uuid; - private Set permissions; + private Set permissions; private Map> securityKeyValues; private Map backendVariants; @@ -360,12 +360,38 @@ public class QSession implements Serializable, Cloneable return (false); } - List values = securityKeyValues.get(keyName); - Serializable valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + List values = securityKeyValues.get(keyName); + + Serializable valueAsType; + try + { + valueAsType = ValueUtils.getValueAsFieldType(fieldType, value); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means the value isn't in the session. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (false); + } + for(Serializable keyValue : values) { - Serializable keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); - if(keyValueAsType.equals(valueAsType)) + Serializable keyValueAsType = null; + try + { + keyValueAsType = ValueUtils.getValueAsFieldType(fieldType, keyValue); + } + catch(Exception e) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // an exception in getValueAsFieldType would indicate, e.g., a non-number string trying to come back as integer. // + // so - assume that any such mismatch means this key isn't a match. + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + } + + if(valueAsType.equals(keyValueAsType)) { return (true); } @@ -561,6 +587,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Getter for valuesForFrontend *******************************************************************************/ @@ -591,6 +618,7 @@ public class QSession implements Serializable, Cloneable } + /******************************************************************************* ** Fluent setter for a single valuesForFrontend *******************************************************************************/ @@ -604,5 +632,4 @@ public class QSession implements Serializable, Cloneable return (this); } - } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java index cd676342..dd085268 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/session/QSessionTest.java @@ -76,7 +76,7 @@ class QSessionTest extends BaseTest void testMixedValueTypes() { QSession session = new QSession().withSecurityKeyValues(Map.of( - "storeId", List.of("100", "200", 300) + "storeId", List.of("100", "200", 300, "four-hundred") )); for(int i : List.of(100, 200, 300)) @@ -86,6 +86,18 @@ class QSessionTest extends BaseTest assertTrue(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should contain: " + i); assertTrue(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should contain: " + i); } + + //////////////////////////////////////////////////////////////////////////// + // next two blocks - used to throw exceptions - now, gracefully be false. // + //////////////////////////////////////////////////////////////////////////// + int i = 400; + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", i, QFieldType.STRING), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", String.valueOf(i), QFieldType.STRING), "Should not contain: " + i); + + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.INTEGER), "Should not contain: " + i); + assertFalse(session.hasSecurityKeyValue("storeId", "one-hundred", QFieldType.STRING), "Should not contain: " + i); } From d8ac14a756ed8c0c5b93a994243efbd2c807faf1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:44:43 -0600 Subject: [PATCH 14/84] CE-1955 Checkpoint on bulk-load backend --- .../BulkInsertPrepareFileMappingStep.java | 240 +++++++++++++++ .../BulkInsertPrepareValueMappingStep.java | 250 ++++++++++++++++ .../BulkInsertReceiveFileMappingStep.java | 200 +++++++++++++ .../insert/BulkInsertReceiveFileStep.java | 81 ------ .../BulkInsertReceiveValueMappingStep.java | 105 +++++++ .../bulk/insert/BulkInsertStepUtils.java | 96 ++++++ .../bulk/insert/BulkInsertTransformStep.java | 68 ++++- .../bulk/insert/BulkInsertV2ExtractStep.java | 1 + .../AbstractIteratorBasedFileToRows.java | 2 +- .../insert/filehandling/CsvFileToRows.java | 2 +- .../filehandling/FileToRowsInterface.java | 2 +- .../insert/filehandling/XlsxFileToRows.java | 2 +- .../insert/mapping/BulkInsertMapping.java | 67 ++++- .../mapping/BulkInsertWideLayoutMapping.java | 216 ++++++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 6 +- .../insert/mapping/RowsToRecordInterface.java | 34 ++- .../bulk/insert/mapping/TallRowsToRecord.java | 27 +- .../bulk/insert/mapping/ValueMapper.java | 72 ++++- .../WideRowsToRecordWithExplicitMapping.java | 260 +++++++++++++++++ ...=> WideRowsToRecordWithSpreadMapping.java} | 78 +---- .../insert/{ => model}/BulkLoadFileRow.java | 2 +- .../bulk/insert/model/BulkLoadProfile.java | 133 +++++++++ .../insert/model/BulkLoadProfileField.java | 195 +++++++++++++ .../insert/model/BulkLoadTableStructure.java | 275 ++++++++++++++++++ .../BulkInsertPrepareFileMappingStepTest.java | 69 +++++ ...ulkInsertPrepareValueMappingStepTest.java} | 42 ++- .../filehandling/CsvFileToRowsTest.java | 2 +- .../insert/filehandling/TestFileToRows.java | 2 +- .../filehandling/XlsxFileToRowsTest.java | 2 +- .../insert/mapping/FlatRowsToRecordTest.java | 2 +- .../insert/mapping/TallRowsToRecordTest.java | 101 ++++++- .../bulk/insert/mapping/ValueMapperTest.java | 7 +- ...deRowsToRecordWithExplicitMappingTest.java | 269 +++++++++++++++++ ...ideRowsToRecordWithSpreadMappingTest.java} | 10 +- .../qqq/backend/core/utils/TestUtils.java | 4 +- 35 files changed, 2682 insertions(+), 242 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{WideRowsToRecord.java => WideRowsToRecordWithSpreadMapping.java} (75%) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{ => model}/BulkLoadFileRow.java (99%) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java rename qqq-backend-core/src/{main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java => test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java} (50%) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{WideRowsToRecordTest.java => WideRowsToRecordWithSpreadMappingTest.java} (97%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java new file mode 100644 index 00000000..f7ee32ce --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -0,0 +1,240 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.File; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareFileMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); + buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFileDetailsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + File file = new File(storageInput.getReference()); + runBackendStepOutput.addValue("fileBaseName", file.getName()); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + ///////////////////////////////////////////////// + // read the 1st row, and assume it is a header // + ///////////////////////////////////////////////// + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + ArrayList headerValues = new ArrayList<>(); + ArrayList headerLetters = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + headerValues.add(ValueUtils.getValueAsString(headerRow.getValue(i))); + headerLetters.add(toHeaderLetter(i)); + } + runBackendStepOutput.addValue("headerValues", headerValues); + runBackendStepOutput.addValue("headerLetters", headerLetters); + + /////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under preview-rows limit, read rows // + /////////////////////////////////////////////////////////////////////////////////////////// + int previewRows = 0; + int previewRowsLimit = 5; + ArrayList> bodyValues = new ArrayList<>(); + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.add(new ArrayList<>()); + } + + while(fileToRowsInterface.hasNext() && previewRows < previewRowsLimit) + { + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + previewRows++; + + for(int i = 0; i < headerRow.size(); i++) + { + bodyValues.get(i).add(ValueUtils.getValueAsString(bodyRow.getValueElseNull(i))); + } + } + runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } + catch(Exception e) + { + throw (new QException("Error reading bulk load file", e)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static String toHeaderLetter(int i) + { + StringBuilder rs = new StringBuilder(); + + do + { + rs.insert(0, (char) ('A' + (i % 26))); + i = (i / 26) - 1; + } + while(i >= 0); + + return (rs.toString()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) + { + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); + runBackendStepOutput.addValue("tableStructure", tableStructure); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> f.getLabel())); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java new file mode 100644 index 00000000..85397442 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -0,0 +1,250 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.google.gson.reflect.TypeToken; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertPrepareValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertPrepareValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + ///////////////////////////////////////////////////////////// + // prep the frontend for what field we're going to map now // + ///////////////////////////////////////////////////////////// + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + if(valueMappingFieldIndex == null) + { + valueMappingFieldIndex = 0; + } + else + { + valueMappingFieldIndex++; + } + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // if there are no more fields (values) to map, then proceed to the standard streamed-ETL preview // + //////////////////////////////////////////////////////////////////////////////////////////////////// + if(valueMappingFieldIndex >= fieldNamesToDoValueMapping.size()) + { + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + return; + } + + runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex); + + String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName); + + runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field())); + runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName); + runBackendStepInput.addValue("valueMappingFieldTableName", tableAndField.table().getName()); + + //////////////////////////////////////////////////// + // get all the values from the file in this field // + // todo - should do all mapping fields at once? // + //////////////////////////////////////////////////// + ArrayList fileValues = getValuesForField(tableAndField.table(), tableAndField.field(), fullFieldName, runBackendStepInput); + runBackendStepOutput.addValue("fileValues", fileValues); + + /////////////////////////////////////////////// + // clear these in case not getting set below // + /////////////////////////////////////////////// + runBackendStepOutput.addValue("valueMapping", new HashMap<>()); + runBackendStepOutput.addValue("mappedValueLabels", new HashMap<>()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + HashMap valueMapping = null; + if(bulkInsertMapping.getFieldNameToValueMapping() != null && bulkInsertMapping.getFieldNameToValueMapping().containsKey(fullFieldName)) + { + valueMapping = CollectionUtils.useOrWrap(bulkInsertMapping.getFieldNameToValueMapping().get(fullFieldName), new TypeToken<>() {}); + runBackendStepOutput.addValue("valueMapping", valueMapping); + + if(StringUtils.hasContent(tableAndField.field().getPossibleValueSourceName())) + { + HashMap possibleValueLabels = loadPossibleValues(tableAndField.field(), valueMapping); + runBackendStepOutput.addValue("mappedValueLabels", possibleValueLabels); + } + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert prepare value mapping", e); + throw new QException("Unhandled error in bulk insert prepare value mapping step", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static TableAndField getTableAndField(String tableName, String fullFieldName) throws QException + { + List parts = new ArrayList<>(List.of(fullFieldName.split("\\."))); + String fieldBaseName = parts.remove(parts.size() - 1); + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + for(String associationName : parts) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + table = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + + TableAndField result = new TableAndField(table, table.getField(fieldBaseName)); + return result; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public record TableAndField(QTableMetaData table, QFieldMetaData field) {} + + + + /*************************************************************************** + ** + ***************************************************************************/ + private HashMap loadPossibleValues(QFieldMetaData field, Map valueMapping) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setPossibleValueSourceName(field.getPossibleValueSourceName()); + input.setIdList(new ArrayList<>(new HashSet<>(valueMapping.values()))); // go through a set to strip dupes + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + + HashMap rs = new HashMap<>(); + for(QPossibleValue result : output.getResults()) + { + Serializable id = (Serializable) result.getId(); + rs.put(id, result.getLabel()); + } + return rs; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private ArrayList getValuesForField(QTableMetaData table, QFieldMetaData field, String fullFieldName, RunBackendStepInput runBackendStepInput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + + String associationNameChain = null; + if(fullFieldName.contains(".")) + { + associationNameChain = fullFieldName.substring(0, fullFieldName.lastIndexOf('.')); + } + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Set values = new LinkedHashSet<>(); + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow); + int index = fieldIndexes.get(field.getName()); + + while(fileToRowsInterface.hasNext()) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + Serializable value = row.getValueElseNull(index); + if(value != null) + { + values.add(ValueUtils.getValueAsString(value)); + } + + if(values.size() > 100) + { + throw (new QUserFacingException("Too many unique values were found for mapping for field: " + field.getName())); + } + } + + return (new ArrayList<>(values)); + } + catch(Exception e) + { + throw (new QException("Error getting values from file", e)); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java new file mode 100644 index 00000000..8f72bb82 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -0,0 +1,200 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveFileMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now build the mapping object that the backend wants - based on the bulkLoadProfile from the frontend // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(bulkLoadProfile.getHasHeaderRow()); + bulkInsertMapping.setLayout(BulkInsertMapping.Layout.valueOf(bulkLoadProfile.getLayout())); + + ////////////////////////////////////////////////////////////////////////////////////////////// + // handle field to name or index mappings (depending on if there's a header row being used) // + ////////////////////////////////////////////////////////////////////////////////////////////// + if(BooleanUtils.isTrue(bulkInsertMapping.getHasHeaderRow())) + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + Map fieldNameToHeaderNameMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToHeaderNameMap(fieldNameToHeaderNameMap); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + } + } + } + else + { + Map fieldNameToIndexMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToIndexMap(fieldNameToIndexMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getColumnIndex() != null) + { + fieldNameToIndexMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getColumnIndex()); + } + } + } + + ///////////////////////////////////// + // do fields w/ default values now // + ///////////////////////////////////// + HashMap fieldNameToDefaultValueMap = new HashMap<>(); + bulkInsertMapping.setFieldNameToDefaultValueMap(fieldNameToDefaultValueMap); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getDefaultValue() != null) + { + fieldNameToDefaultValueMap.put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getDefaultValue()); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////// + // frontend at this point will have sent just told us which field names need value mapping // + // store those - and let them drive the value-mapping screens that we'll go through next // + // todo - uh, what if those come from profile, dummy!? + ///////////////////////////////////////////////////////////////////////////////////////////// + ArrayList fieldNamesToDoValueMapping = new ArrayList<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping())) + { + fieldNamesToDoValueMapping.add(bulkLoadProfileField.getFieldName()); + + if(CollectionUtils.nullSafeHasContents(bulkLoadProfileField.getValueMappings())) + { + bulkInsertMapping.getFieldNameToValueMapping().put(bulkLoadProfileField.getFieldName(), bulkLoadProfileField.getValueMappings()); + } + } + } + runBackendStepOutput.addValue("fieldNamesToDoValueMapping", new ArrayList<>(fieldNamesToDoValueMapping)); + + /////////////////////////////////////////////////////////////////////////////////////// + // figure out what associations are being mapped, by looking at the full field names // + /////////////////////////////////////////////////////////////////////////////////////// + Set associationNameSet = new HashSet<>(); + for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) + { + if(bulkLoadProfileField.getFieldName().contains(".")) + { + associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + } + } + bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////// + // at this point we're done populating the bulkInsertMapping object. put it in the process state. // + ///////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + if(CollectionUtils.nullSafeHasContents(fieldNamesToDoValueMapping)) + { + ////////////////////////////////////////////////////////////////////////////////// + // just go to the prepareValueMapping backend step - it'll figure out the rest. // + // it's also where the value-mapping loop of steps points. // + // and, this will actually be the default (e.g., the step after this one). // + ////////////////////////////////////////////////////////////////////////////////// + } + else + { + ////////////////////////////////////////////////////////////////////////////////// + // else - if no values to map - continue with the standard streamed-ETL preview // + ////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepStreamedETLPreview(runBackendStepOutput); + } + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java deleted file mode 100644 index 6ebcf3a8..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; -import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.utils.JsonUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertReceiveFileStep implements BackendStep -{ - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); - - try - ( - InputStream inputStream = new StorageAction().getInputStream(storageInput); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); - ) - { - BulkLoadFileRow headerRow = fileToRowsInterface.next(); - - List bodyRows = new ArrayList<>(); - while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) - { - bodyRows.add(fileToRowsInterface.next().toString()); - } - - runBackendStepOutput.addValue("header", headerRow.toString()); - runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); - System.out.println("Done receiving file"); - } - catch(QException qe) - { - throw qe; - } - catch(Exception e) - { - throw new QException("Unhandled error in bulk insert extract step", e); - } - - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java new file mode 100644 index 00000000..f9c17667 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.fasterxml.jackson.core.type.TypeReference; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveValueMappingStep implements BackendStep +{ + private static final QLogger LOG = QLogger.getLogger(BulkInsertReceiveValueMappingStep.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + try + { + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + List fieldNamesToDoValueMapping = (List) runBackendStepInput.getValue("fieldNamesToDoValueMapping"); + Integer valueMappingFieldIndex = runBackendStepInput.getValueInteger("valueMappingFieldIndex"); + + String fieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + + /////////////////////////////////////////////////////////////////// + // read process values - construct a bulkLoadProfile out of them // + /////////////////////////////////////////////////////////////////// + BulkLoadProfile bulkLoadProfile = BulkInsertStepUtils.getBulkLoadProfile(runBackendStepInput); + + ///////////////////////////////////////////////////////////////////////// + // put the list of bulk load profile into the process state - it's the // + // thing that the frontend will be looking at as the saved profile // + ///////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // get the bulkInsertMapping object from the process, creating a fieldNameToValueMapping map within it if needed // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + Map> fieldNameToValueMapping = bulkInsertMapping.getFieldNameToValueMapping(); + if(fieldNameToValueMapping == null) + { + fieldNameToValueMapping = new HashMap<>(); + bulkInsertMapping.setFieldNameToValueMapping(fieldNameToValueMapping); + } + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + //////////////////////////////////////////////// + // put the mapped values into the mapping map // + //////////////////////////////////////////////// + Map mappedValues = JsonUtils.toObject(runBackendStepInput.getValueString("mappedValuesJSON"), new TypeReference<>() {}); + fieldNameToValueMapping.put(fieldName, mappedValues); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // always return to the prepare-mapping step - as it will determine if it's time to break the loop or not. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.setNextStepPrepareValueMapping(runBackendStepOutput); + } + catch(Exception e) + { + LOG.warn("Error in bulk insert receive mapping", e); + throw new QException("Unhandled error in bulk insert receive mapping step", e); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index e3dac892..3e4ffa40 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -22,10 +22,22 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; +import java.util.HashMap; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.json.JSONArray; +import org.json.JSONObject; /******************************************************************************* @@ -55,4 +67,88 @@ public class BulkInsertStepUtils return (storageInput); } + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepStreamedETLPreview(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveValueMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void setNextStepPrepareValueMapping(RunBackendStepOutput runBackendStepOutput) + { + runBackendStepOutput.setOverrideLastStepName("receiveFileMapping"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadProfile getBulkLoadProfile(RunBackendStepInput runBackendStepInput) + { + String version = runBackendStepInput.getValueString("version"); + if("v1".equals(version)) + { + String layout = runBackendStepInput.getValueString("layout"); + Boolean hasHeaderRow = runBackendStepInput.getValueBoolean("hasHeaderRow"); + + ArrayList fieldList = new ArrayList<>(); + + JSONArray array = new JSONArray(runBackendStepInput.getValueString("fieldListJSON")); + for(int i = 0; i < array.length(); i++) + { + JSONObject jsonObject = array.getJSONObject(i); + BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); + fieldList.add(bulkLoadProfileField); + bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); + bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); + bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); + + if(BooleanUtils.isTrue(bulkLoadProfileField.getDoValueMapping()) && jsonObject.has("valueMappings")) + { + bulkLoadProfileField.setValueMappings(new HashMap<>()); + JSONObject valueMappingsJsonObject = jsonObject.getJSONObject("valueMappings"); + for(String fileValue : valueMappingsJsonObject.keySet()) + { + bulkLoadProfileField.getValueMappings().put(fileValue, ValueUtils.getValueAsString(valueMappingsJsonObject.get(fileValue))); + } + } + } + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withFieldList(fieldList) + .withHasHeaderRow(hasHeaderRow) + .withLayout(layout); + + return (bulkLoadProfile); + } + else + { + throw (new IllegalArgumentException("Unexpected version for bulk load profile: " + version)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void handleSavedBulkLoadProfileIdValue(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + Integer savedBulkLoadProfileId = runBackendStepInput.getValueInteger("savedBulkLoadProfileId"); + if(savedBulkLoadProfileId != null) + { + QRecord savedBulkLoadProfileRecord = GetAction.execute(SavedBulkLoadProfile.TABLE_NAME, savedBulkLoadProfileId); + runBackendStepOutput.addValue("savedBulkLoadProfileRecord", savedBulkLoadProfileRecord); + } + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f4eff9a3..18c03671 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -111,6 +111,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep // since we're doing a unique key check in this class, we can tell the loadViaInsert step that it (rather, the InsertAction) doesn't need to re-do one. // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue(LoadViaInsertStep.FIELD_SKIP_UNIQUE_KEY_CHECK, true); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////// + BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); } @@ -121,8 +126,43 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int rowsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + int recordsInThisPage = runBackendStepInput.getRecords().size(); + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + + // split the records w/o UK errors into those w/ e + List recordsWithoutAnyErrors = new ArrayList<>(); + List recordsWithSomeErrors = new ArrayList<>(); + for(QRecord record : runBackendStepInput.getRecords()) + { + if(CollectionUtils.nullSafeHasContents(record.getErrors())) + { + recordsWithSomeErrors.add(record); + } + else + { + recordsWithoutAnyErrors.add(record); + } + } + + ////////////////////////////////////////////////////////////////// + // propagate errors that came into this step out to the summary // + ////////////////////////////////////////////////////////////////// + if(!recordsWithSomeErrors.isEmpty()) + { + for(QRecord record : recordsWithSomeErrors) + { + String message = record.getErrors().get(0).getMessage(); + processSummaryWarningsAndErrorsRollup.addError(message, null); + } + } + + if(recordsWithoutAnyErrors.isEmpty()) + { + //////////////////////////////////////////////////////////////////////////////// + // skip th rest of this method if there aren't any records w/o errors in them // + //////////////////////////////////////////////////////////////////////////////// + this.rowsProcessed += recordsInThisPage; + } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -130,7 +170,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep InsertInput insertInput = new InsertInput(); insertInput.setInputSource(QInputSource.USER); insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(runBackendStepInput.getRecords()); + insertInput.setRecords(recordsWithoutAnyErrors); insertInput.setSkipUniqueKeyCheck(true); ////////////////////////////////////////////////////////////////////// @@ -145,7 +185,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, runBackendStepInput.getRecords(), true); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true); runBackendStepInput.setRecords(recordsAfterCustomizer); /////////////////////////////////////////////////////////////////////////////////////// @@ -159,13 +199,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, runBackendStepInput.getRecords(), uniqueKey).keySet()); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet()); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // + // todo - move this up (before the early return?) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { @@ -187,12 +228,12 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(runBackendStepInput, existingKeys, uniqueKeys, table); + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - insertInput.setRecords(recordsWithoutUkErrors); + insertInput.setRecords(recordsWithoutAnyErrors); InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); @@ -222,8 +263,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); - - this.rowsProcessed += rowsInThisPage; + this.rowsProcessed += recordsInThisPage; } @@ -231,7 +271,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep /******************************************************************************* ** *******************************************************************************/ - private List getRecordsWithoutUniqueKeyErrors(RunBackendStepInput runBackendStepInput, Map>> existingKeys, List uniqueKeys, QTableMetaData table) + private List getRecordsWithoutUniqueKeyErrors(List records, Map>> existingKeys, List uniqueKeys, QTableMetaData table) { //////////////////////////////////////////////////// // if there are no UK's, proceed with all records // @@ -239,7 +279,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithoutUkErrors = new ArrayList<>(); if(existingKeys.isEmpty()) { - recordsWithoutUkErrors.addAll(runBackendStepInput.getRecords()); + recordsWithoutUkErrors.addAll(records); } else { @@ -255,7 +295,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep // else, get each records keys and see if it already exists or not // // also, build a set of keys we've seen (within this page (or overall?)) // /////////////////////////////////////////////////////////////////////////// - for(QRecord record : runBackendStepInput.getRecords()) + for(QRecord record : records) { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { @@ -333,8 +373,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary .withMessageSuffix(" inserted, because of duplicate values in a unique key on the fields (" + uniqueKey.getDescription(table) + "), with values" - + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") - + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) + + (ukErrorSummary.areThereMoreSampleValues ? " such as: " : ": ") + + StringUtils.joinWithCommasAndAnd(new ArrayList<>(ukErrorSummary.sampleValues))) .withSingularFutureMessage(" record will not be") .withPluralFutureMessage(" records will not be") diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index 2e49e48a..efacca0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java index 383dfa21..be80c82a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -23,7 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.util.Iterator; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java index f39f4d45..7a58ddef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -28,7 +28,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Serializable; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java index d2d6c78a..9d02ee0f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -27,7 +27,7 @@ import java.util.Iterator; import java.util.Locale; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index 21c9d928..a6e336de 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -27,7 +27,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java index d95d48e4..fef870b1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -32,8 +32,9 @@ import java.util.function.Supplier; import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -62,8 +63,10 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private List mappedAssociations = new ArrayList<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private Map wideLayoutMapping = new HashMap<>(); + + private List mappedAssociations = new ArrayList<>(); private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); @@ -72,11 +75,11 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - public enum Layout + public enum Layout implements PossibleValueEnum { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecord::new); + WIDE(WideRowsToRecordWithExplicitMapping::new); /*************************************************************************** @@ -95,6 +98,7 @@ public class BulkInsertMapping implements Serializable } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +106,28 @@ public class BulkInsertMapping implements Serializable { return (supplier.get()); } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueId() + { + return name(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return StringUtils.ucFirst(name().toLowerCase()); + } } @@ -500,4 +526,35 @@ public class BulkInsertMapping implements Serializable return (this); } + + + /******************************************************************************* + ** Getter for wideLayoutMapping + *******************************************************************************/ + public Map getWideLayoutMapping() + { + return (this.wideLayoutMapping); + } + + + + /******************************************************************************* + ** Setter for wideLayoutMapping + *******************************************************************************/ + public void setWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + } + + + + /******************************************************************************* + ** Fluent setter for wideLayoutMapping + *******************************************************************************/ + public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) + { + this.wideLayoutMapping = wideLayoutMapping; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java new file mode 100644 index 00000000..ad0cce0f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java @@ -0,0 +1,216 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertWideLayoutMapping +{ + private List childRecordMappings; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkInsertWideLayoutMapping(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class ChildRecordMapping + { + Map fieldNameToHeaderNameMaps; + Map associationNameToChildRecordMappingMap; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public Map getFieldNameToHeaderNameMaps() + { + return (this.fieldNameToHeaderNameMaps); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMaps + *******************************************************************************/ + public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) + { + this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public Map getAssociationNameToChildRecordMappingMap() + { + return (this.associationNameToChildRecordMappingMap); + } + + + + /******************************************************************************* + ** Setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + } + + + + /******************************************************************************* + ** Fluent setter for associationNameToChildRecordMappingMap + *******************************************************************************/ + public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) + { + this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Map getFieldIndexes(BulkLoadFileRow headerRow) + { + // todo memoize or otherwise don't recompute + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) + { + String headerName = entry.getValue(); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(entry.getKey(), headerIndex); + } + } + } + + return (rs); + } + } + + + + /******************************************************************************* + ** Getter for childRecordMappings + *******************************************************************************/ + public List getChildRecordMappings() + { + return (this.childRecordMappings); + } + + + + /******************************************************************************* + ** Setter for childRecordMappings + *******************************************************************************/ + public void setChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + } + + + + /******************************************************************************* + ** Fluent setter for childRecordMappings + *******************************************************************************/ + public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) + { + this.childRecordMappings = childRecordMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index ac98adf4..44702312 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -30,8 +30,8 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /******************************************************************************* @@ -62,13 +62,13 @@ public class FlatRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 91481980..25aa6543 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -26,8 +26,10 @@ import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -44,26 +46,50 @@ public interface RowsToRecordInterface /*************************************************************************** - ** + ** returns true if value from row was used, else false. ***************************************************************************/ - default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) { - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + String fieldName = field.getName(); + QFieldType type = field.getType(); + + boolean valueFromRowWasUsed = false; + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; Serializable value = null; if(index != null && row != null) { value = row.getValueElseNull(index); + if(value != null && !"".equals(value)) + { + valueFromRowWasUsed = true; + } } else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) { value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); } + /* note - moving this to ValueMapper... + if(value != null) + { + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + */ + if(value != null) { record.setValue(fieldName, value); } + + return (valueFromRowWasUsed); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 1dbccd92..690f705d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -35,8 +35,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -75,7 +75,12 @@ public class TallRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); - List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -126,13 +131,24 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } + /*************************************************************************** + ** + ***************************************************************************/ + private List groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException + { + Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); + return new ArrayList<>(fieldIndexes.values()); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -148,7 +164,7 @@ public class TallRowsToRecord implements RowsToRecordInterface BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); } ///////////////////////////// @@ -230,7 +246,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) { - throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, associationNameChainForRecursiveCalls); + // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } List rowGroupByValues = getGroupByValues(row, groupByIndexes); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 09e074d3..0f1e8cfc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -25,9 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import java.util.List; import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -35,12 +45,16 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; *******************************************************************************/ public class ValueMapper { + private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); + + + /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping) + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException { - valueMapping(records, mapping, null); + valueMapping(records, mapping, table, null); } @@ -48,7 +62,7 @@ public class ValueMapper /*************************************************************************** ** ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException { if(CollectionUtils.nullSafeIsEmpty(records)) { @@ -58,20 +72,58 @@ public class ValueMapper Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { - for(Map.Entry> entry : mappingForTable.entrySet()) + for(Map.Entry valueEntry : record.getValues().entrySet()) { - String fieldName = entry.getKey(); - Map map = entry.getValue(); - String value = record.getValueString(fieldName); - if(value != null && map.containsKey(value)) + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) { - record.setValue(fieldName, map.get(value)); + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null) + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + } + } + + record.setValue(field.getName(), value); } + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) { - valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java new file mode 100644 index 00000000..adc67ec7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java @@ -0,0 +1,260 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException + { + for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) + { + String associationName = entry.getKey(); + BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; + + for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) + { + QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); + if(associatedRecord != null) + { + record.withAssociatedRecord(associationName, associatedRecord); + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); + + QRecord associatedRecord = new QRecord(); + boolean usedAnyValuesFromRow = false; + + for(QFieldMetaData field : table.getFields().values()) + { + boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + usedAnyValuesFromRow |= valueFromRowWasUsed; + } + + if(usedAnyValuesFromRow) + { + processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); + return (associatedRecord); + } + else + { + return (null); + } + } + + // /*************************************************************************** + // ** + // ***************************************************************************/ + // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + // { + // List rs = new ArrayList<>(); + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationName + ".")) + // { + // String fieldName = entry.getKey().substring(associationName.length() + 1); + + // ////////////////////////////////////////////////////////////////////////// + // // make sure the name here is for this table - not a sub-table under it // + // ////////////////////////////////////////////////////////////////////////// + // if(!fieldName.contains(".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + // } + // } + // } + + // ///////////////////////////////////////////////////////////////////// + // // loop over the length of the record, building associated records // + // ///////////////////////////////////////////////////////////////////// + // QRecord associatedRecord = new QRecord(); + // Set processedFieldNames = new HashSet<>(); + // boolean gotAnyValues = false; + // int subStartIndex = -1; + + // for(int i = startIndex; i < endIndex; i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // /////////////////////////////////////////////// + // // ok - this is a value for this association // + // /////////////////////////////////////////////// + // if(subStartIndex == -1) + // { + // subStartIndex = i; + // } + + // String fieldName = entry.getKey(); + // if(processedFieldNames.contains(fieldName)) + // { + // ///////////////////////////////////////////////// + // // this means we're starting a new sub-record! // + // ///////////////////////////////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + // rs.add(associatedRecord); + // } + + // associatedRecord = new QRecord(); + // processedFieldNames = new HashSet<>(); + // gotAnyValues = false; + // subStartIndex = i + 1; + // } + + // processedFieldNames.add(fieldName); + + // Serializable value = row.getValueElseNull(i); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + // } + // } + // } + + // //////////////////////// + // // handle final value // + // //////////////////////// + // if(gotAnyValues) + // { + // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + // rs.add(associatedRecord); + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java similarity index 75% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 40e37071..f87fa17b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -37,8 +37,8 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -48,7 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; /******************************************************************************* ** *******************************************************************************/ -public class WideRowsToRecord implements RowsToRecordInterface +public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface { private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -77,7 +77,7 @@ public class WideRowsToRecord implements RowsToRecordInterface for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); } processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); @@ -85,7 +85,7 @@ public class WideRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping); + ValueMapper.valueMapping(rs, mapping, table); return (rs); } @@ -199,7 +199,7 @@ public class WideRowsToRecord implements RowsToRecordInterface gotAnyValues = true; } - setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + setValueOrDefault(associatedRecord, table.getField(fieldName), associationName, mapping, row, i); } } } @@ -228,77 +228,11 @@ public class WideRowsToRecord implements RowsToRecordInterface { if(!processedFieldNames.contains(field.getName())) { - setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); } } } - /*************************************************************************** - ** - ***************************************************************************/ - // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException - // { - // List rs = new ArrayList<>(); - // String associationNameChainForRecursiveCalls = associationName; - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); - // } - // } - - // Map> indexes = new HashMap<>(); - // for(int i = 0; i < headerRow.size(); i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); - // } - // } - // } - - // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); - - // ////////////////////////////////////////////////////// - // // figure out how many sub-rows we'll be processing // - // ////////////////////////////////////////////////////// - // for(int i = 0; i < maxIndex; i++) - // { - // QRecord associatedRecord = new QRecord(); - // boolean gotAnyValues = false; - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // String fieldName = entry.getKey(); - // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) - // { - // Integer index = indexes.get(fieldName).get(i); - // Serializable value = row.getValueElseNull(index); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); - // } - // } - - // if(gotAnyValues) - // { - // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); - // rs.add(associatedRecord); - // } - // } - - // return (rs); - // } - /*************************************************************************** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java similarity index 99% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java index 487f606a..23b80a56 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadFileRow.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java new file mode 100644 index 00000000..8bcfb97c --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -0,0 +1,133 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; + + +/*************************************************************************** + * this is the model of a saved bulk load profile - which is what passes back + * and forth with the frontend. + ****************************************************************************/ +public class BulkLoadProfile implements Serializable +{ + private ArrayList fieldList; + private Boolean hasHeaderRow; + private String layout; + + + + /******************************************************************************* + ** Getter for fieldList + *******************************************************************************/ + public ArrayList getFieldList() + { + return (this.fieldList); + } + + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkLoadProfile withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public String getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(String layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkLoadProfile withLayout(String layout) + { + this.layout = layout; + return (this); + } + + + /******************************************************************************* + ** Setter for fieldList + *******************************************************************************/ + public void setFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + } + + + + /******************************************************************************* + ** Fluent setter for fieldList + *******************************************************************************/ + public BulkLoadProfile withFieldList(ArrayList fieldList) + { + this.fieldList = fieldList; + return (this); + } + + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java new file mode 100644 index 00000000..6bedb18b --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.Map; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class BulkLoadProfileField +{ + private String fieldName; + private Integer columnIndex; + private Serializable defaultValue; + private Boolean doValueMapping; + private Map valueMappings; + + + + /******************************************************************************* + ** Getter for fieldName + *******************************************************************************/ + public String getFieldName() + { + return (this.fieldName); + } + + + + /******************************************************************************* + ** Setter for fieldName + *******************************************************************************/ + public void setFieldName(String fieldName) + { + this.fieldName = fieldName; + } + + + + /******************************************************************************* + ** Fluent setter for fieldName + *******************************************************************************/ + public BulkLoadProfileField withFieldName(String fieldName) + { + this.fieldName = fieldName; + return (this); + } + + + + /******************************************************************************* + ** Getter for columnIndex + *******************************************************************************/ + public Integer getColumnIndex() + { + return (this.columnIndex); + } + + + + /******************************************************************************* + ** Setter for columnIndex + *******************************************************************************/ + public void setColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + } + + + + /******************************************************************************* + ** Fluent setter for columnIndex + *******************************************************************************/ + public BulkLoadProfileField withColumnIndex(Integer columnIndex) + { + this.columnIndex = columnIndex; + return (this); + } + + + + /******************************************************************************* + ** Getter for defaultValue + *******************************************************************************/ + public Serializable getDefaultValue() + { + return (this.defaultValue); + } + + + + /******************************************************************************* + ** Setter for defaultValue + *******************************************************************************/ + public void setDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + } + + + + /******************************************************************************* + ** Fluent setter for defaultValue + *******************************************************************************/ + public BulkLoadProfileField withDefaultValue(Serializable defaultValue) + { + this.defaultValue = defaultValue; + return (this); + } + + + + /******************************************************************************* + ** Getter for doValueMapping + *******************************************************************************/ + public Boolean getDoValueMapping() + { + return (this.doValueMapping); + } + + + + /******************************************************************************* + ** Setter for doValueMapping + *******************************************************************************/ + public void setDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for doValueMapping + *******************************************************************************/ + public BulkLoadProfileField withDoValueMapping(Boolean doValueMapping) + { + this.doValueMapping = doValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for valueMappings + *******************************************************************************/ + public Map getValueMappings() + { + return (this.valueMappings); + } + + + + /******************************************************************************* + ** Setter for valueMappings + *******************************************************************************/ + public void setValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + } + + + + /******************************************************************************* + ** Fluent setter for valueMappings + *******************************************************************************/ + public BulkLoadProfileField withValueMappings(Map valueMappings) + { + this.valueMappings = valueMappings; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java new file mode 100644 index 00000000..db55198f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadTableStructure.java @@ -0,0 +1,275 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; + + +import java.io.Serializable; +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadTableStructure implements Serializable +{ + private boolean isMain; + private boolean isMany; + + private String tableName; + private String label; + private String associationPath; // null/empty for main table, then associationName for a child, associationName.associationName for a grandchild + + private ArrayList fields; // mmm, not marked as serializable (at this time) - is okay? + private ArrayList associations; + + + + /******************************************************************************* + ** Getter for isMain + *******************************************************************************/ + public boolean getIsMain() + { + return (this.isMain); + } + + + + /******************************************************************************* + ** Setter for isMain + *******************************************************************************/ + public void setIsMain(boolean isMain) + { + this.isMain = isMain; + } + + + + /******************************************************************************* + ** Fluent setter for isMain + *******************************************************************************/ + public BulkLoadTableStructure withIsMain(boolean isMain) + { + this.isMain = isMain; + return (this); + } + + + + /******************************************************************************* + ** Getter for isMany + *******************************************************************************/ + public boolean getIsMany() + { + return (this.isMany); + } + + + + /******************************************************************************* + ** Setter for isMany + *******************************************************************************/ + public void setIsMany(boolean isMany) + { + this.isMany = isMany; + } + + + + /******************************************************************************* + ** Fluent setter for isMany + *******************************************************************************/ + public BulkLoadTableStructure withIsMany(boolean isMany) + { + this.isMany = isMany; + return (this); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkLoadTableStructure withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for label + *******************************************************************************/ + public String getLabel() + { + return (this.label); + } + + + + /******************************************************************************* + ** Setter for label + *******************************************************************************/ + public void setLabel(String label) + { + this.label = label; + } + + + + /******************************************************************************* + ** Fluent setter for label + *******************************************************************************/ + public BulkLoadTableStructure withLabel(String label) + { + this.label = label; + return (this); + } + + + + /******************************************************************************* + ** Getter for fields + *******************************************************************************/ + public ArrayList getFields() + { + return (this.fields); + } + + + + /******************************************************************************* + ** Setter for fields + *******************************************************************************/ + public void setFields(ArrayList fields) + { + this.fields = fields; + } + + + + /******************************************************************************* + ** Fluent setter for fields + *******************************************************************************/ + public BulkLoadTableStructure withFields(ArrayList fields) + { + this.fields = fields; + return (this); + } + + + + /******************************************************************************* + ** Getter for associationPath + *******************************************************************************/ + public String getAssociationPath() + { + return (this.associationPath); + } + + + + /******************************************************************************* + ** Setter for associationPath + *******************************************************************************/ + public void setAssociationPath(String associationPath) + { + this.associationPath = associationPath; + } + + + + /******************************************************************************* + ** Fluent setter for associationPath + *******************************************************************************/ + public BulkLoadTableStructure withAssociationPath(String associationPath) + { + this.associationPath = associationPath; + return (this); + } + + + + /******************************************************************************* + ** Getter for associations + *******************************************************************************/ + public ArrayList getAssociations() + { + return (this.associations); + } + + + + /******************************************************************************* + ** Setter for associations + *******************************************************************************/ + public void setAssociations(ArrayList associations) + { + this.associations = associations; + } + + + + /******************************************************************************* + ** Fluent setter for associations + *******************************************************************************/ + public BulkLoadTableStructure withAssociations(ArrayList associations) + { + this.associations = associations; + return (this); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public void addAssociation(BulkLoadTableStructure association) + { + if(this.associations == null) + { + this.associations = new ArrayList<>(); + } + this.associations.add(association); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java new file mode 100644 index 00000000..ea6e0e8f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStepTest.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for BulkInsertPrepareMappingStep + *******************************************************************************/ +class BulkInsertPrepareFileMappingStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("PointlessArithmeticExpression") + @Test + void testToHeaderLetter() + { + assertEquals("A", BulkInsertPrepareFileMappingStep.toHeaderLetter(0)); + assertEquals("B", BulkInsertPrepareFileMappingStep.toHeaderLetter(1)); + assertEquals("Z", BulkInsertPrepareFileMappingStep.toHeaderLetter(25)); + + assertEquals("AA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 0)); + assertEquals("AB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 1)); + assertEquals("AZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 + 25)); + + assertEquals("BA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 0)); + assertEquals("BB", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 1)); + assertEquals("BZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 + 25)); + + assertEquals("ZA", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 0)); + assertEquals("ZB", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 1)); + assertEquals("ZZ", BulkInsertPrepareFileMappingStep.toHeaderLetter(26 * 26 + 25)); + + assertEquals("AAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 0)); + assertEquals("AAB", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 1)); + assertEquals("AAC", BulkInsertPrepareFileMappingStep.toHeaderLetter(27 * 26 + 2)); + + assertEquals("ABA", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 0)); + assertEquals("ABB", BulkInsertPrepareFileMappingStep.toHeaderLetter(28 * 26 + 1)); + + assertEquals("BAA", BulkInsertPrepareFileMappingStep.toHeaderLetter(2 * 26 * 26 + 26 + 0)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java similarity index 50% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java index 0071aa49..6953c5bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStepTest.java @@ -22,38 +22,34 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; -import java.util.Map; -import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* - ** + ** Unit test for BulkInsertPrepareValueMappingStep *******************************************************************************/ -public class BulkInsertReceiveMappingStep implements BackendStep +class BulkInsertPrepareValueMappingStepTest extends BaseTest { - /*************************************************************************** + /******************************************************************************* ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + *******************************************************************************/ + @Test + void test() throws QException { - BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); - bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); - bulkInsertMapping.setHasHeaderRow(true); - bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( - "firstName", "firstName", - "lastName", "Last Name" - )); - runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + assertEquals(TestUtils.TABLE_NAME_ORDER, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").table().getName()); + assertEquals("orderNo", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderNo").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").table().getName()); + assertEquals("sku", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.sku").field().getName()); + + assertEquals(TestUtils.TABLE_NAME_LINE_ITEM_EXTRINSIC, BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").table().getName()); + assertEquals("key", BulkInsertPrepareValueMappingStep.getTableAndField(TestUtils.TABLE_NAME_ORDER, "orderLine.extrinsics.key").field().getName()); - // probably need to what, receive the mapping object, store it into state - // what, do we maybe return to a different sub-mapping screen (e.g., values) - // then at some point - cool - proceed to ETL's steps } -} +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java index 84bdce2f..77b60bd8 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -25,7 +25,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.ByteArrayInputStream; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java index e1142e89..a9d07b63 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -26,7 +26,7 @@ import java.io.InputStream; import java.io.Serializable; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; /*************************************************************************** diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index 8a381ad1..c681f685 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index e5474032..e8c561e5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -29,8 +29,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index c623ddda..5c942fa2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -87,14 +87,14 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -145,7 +145,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -155,7 +155,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -168,7 +168,7 @@ class TallRowsToRecordTest extends BaseTest void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException { Integer defaultStoreId = 101; - Integer defaultLineNo = 102; + String defaultLineNo = "102"; String defaultOrderLineExtraSource = "file"; CsvFileToRows fileToRows = CsvFileToRows.forString(""" @@ -221,7 +221,7 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -240,7 +240,92 @@ class TallRowsToRecordTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testAutomaticGroupByAllIndexes() throws QException + { + Integer defaultStoreId = 101; + String defaultLineNo = "102"; + String defaultOrderLineExtraSource = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, Homer, Simpson, DONUT, 12, Coupon Code, 10QOff, Size, Large + 1, Homer, Simpson, BEER, 500, , , Flavor, Hops + 1, Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.source", defaultOrderLineExtraSource + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(defaultOrderLineExtraSource, defaultOrderLineExtraSource), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 9e4108e9..7b868766 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -26,8 +26,11 @@ import java.io.Serializable; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -44,7 +47,7 @@ class ValueMapperTest extends BaseTest ** *******************************************************************************/ @Test - void test() + void test() throws QException { BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), @@ -94,7 +97,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping); + ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java new file mode 100644 index 00000000..39434f78 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java @@ -0,0 +1,269 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordWithExplicitMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """; + + Integer defaultStoreId = 42; + Integer defaultLineNo = 47; + String defaultLineExtraValue = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withWideLayoutMapping(Map.of( + "orderLine", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 1", "quantity", "Quantity 1"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 2", "quantity", "Quantity 2"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) + )))), + new BulkInsertWideLayoutMapping.ChildRecordMapping( + Map.of("sku", "SKU 3", "quantity", "Quantity 3"), + Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) + )))) + )), + "extrinsics", new BulkInsertWideLayoutMapping(List.of( + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) + )) + )) + + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId, + "orderLine.lineNumber", defaultLineNo, + "orderLine.extrinsics.value", defaultLineExtraValue + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java similarity index 97% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 51a6c5f7..45bcc751 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -28,8 +28,8 @@ import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -39,7 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* ** Unit test for WideRowsToRecord *******************************************************************************/ -class WideRowsToRecordTest extends BaseTest +class WideRowsToRecordWithSpreadMappingTest extends BaseTest { /******************************************************************************* @@ -80,7 +80,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -150,7 +150,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( @@ -229,7 +229,7 @@ class WideRowsToRecordTest extends BaseTest CsvFileToRows fileToRows = CsvFileToRows.forString(csv); BulkLoadFileRow header = fileToRows.next(); - WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + WideRowsToRecordWithSpreadMapping rowsToRecord = new WideRowsToRecordWithSpreadMapping(); BulkInsertMapping mapping = new BulkInsertMapping() .withFieldNameToHeaderNameMap(Map.of( diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index ff274dc9..8ca6da8f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -644,6 +644,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() @@ -700,7 +701,8 @@ public class TestUtils .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("lineItemId", QFieldType.INTEGER)) .withField(new QFieldMetaData("key", QFieldType.STRING)) - .withField(new QFieldMetaData("value", QFieldType.STRING)); + .withField(new QFieldMetaData("value", QFieldType.STRING)) + .withField(new QFieldMetaData("source", QFieldType.STRING)); // doesn't really make sense, but useful to have an extra field here in some bulk-load tests } From 22ce5acf4620ef9a905ae9e9596e5e47caa30537 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:45:24 -0600 Subject: [PATCH 15/84] CE-1955 Make filename its own path element in uploadedFile processing --- .../kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index 7576c64f..cb9ee53e 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -534,7 +534,8 @@ public class QJavalinProcessHandler { String reference = QValueFormatter.formatDate(LocalDate.now()) + File.separator + runProcessInput.getProcessName() - + File.separator + UUID.randomUUID() + "-" + uploadedFile.filename(); + + File.separator + UUID.randomUUID() + + File.separator + uploadedFile.filename(); StorageInput storageInput = new StorageInput(storageTableName).withReference(reference); storageInputs.add(storageInput); From 07886214f5f427170015cb0f2e787466db821a84 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 08:53:49 -0600 Subject: [PATCH 16/84] CE-1955 Test fixes --- .../bulk/insert/BulkInsertTransformStep.java | 2 +- .../insert/mapping/FlatRowsToRecordTest.java | 2 +- .../bulk/insert/mapping/ValueMapperTest.java | 8 ++++---- ...ideRowsToRecordWithExplicitMappingTest.java | 18 +++++++++--------- .../WideRowsToRecordWithSpreadMappingTest.java | 14 +++++++------- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 18c03671..6e28c10a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -233,7 +233,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// - insertInput.setRecords(recordsWithoutAnyErrors); + insertInput.setRecords(recordsWithoutUkErrors); InsertAction insertAction = new InsertAction(); insertAction.performValidations(insertInput, true); List validationResultRecords = insertInput.getRecords(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index e8c561e5..435dcf92 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -84,7 +84,7 @@ class FlatRowsToRecordTest extends BaseTest records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); - assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); assertEquals(List.of("Ned"), getValues(records, "firstName")); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 7b868766..7317d963 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -52,15 +52,15 @@ class ValueMapperTest extends BaseTest BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), - "lineItem.sku", Map.of("ABC", "Alphabet"), - "lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "orderLine.sku", Map.of("ABC", "Alphabet"), + "orderLine.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), "extrinsics.key", Map.of("1", "one", "2", "two") )); QRecord inputRecord = new QRecord() .withValue("storeId", "QQQMart") .withValue("shipToName", "HoJu") - .withAssociatedRecord("lineItem", new QRecord() + .withAssociatedRecord("orderLine", new QRecord() .withValue("sku", "ABC") .withAssociatedRecord("extrinsics", new QRecord() .withValue("key", "myKey") @@ -80,7 +80,7 @@ class ValueMapperTest extends BaseTest QRecord expectedRecord = new QRecord() .withValue("storeId", 1) .withValue("shipToName", "Homer") - .withAssociatedRecord("lineItem", new QRecord() + .withAssociatedRecord("orderLine", new QRecord() .withValue("sku", "Alphabet") .withAssociatedRecord("extrinsics", new QRecord() .withValue("key", "myKey") diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java index 39434f78..4f8bd7a1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java @@ -83,13 +83,13 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -139,7 +139,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -147,7 +147,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -166,7 +166,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest """; Integer defaultStoreId = 42; - Integer defaultLineNo = 47; + String defaultLineNo = "47"; String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); @@ -197,8 +197,8 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest new BulkInsertWideLayoutMapping.ChildRecordMapping( Map.of("sku", "SKU 3", "quantity", "Quantity 3"), Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), + new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) )))) )), "extrinsics", new BulkInsertWideLayoutMapping(List.of( @@ -225,7 +225,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -247,7 +247,7 @@ class WideRowsToRecordWithExplicitMappingTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 45bcc751..61fc6d35 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -101,13 +101,13 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); } @@ -173,7 +173,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -181,7 +181,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); } @@ -223,7 +223,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException { Integer defaultStoreId = 42; - Integer defaultLineNo = 47; + String defaultLineNo = "47"; String defaultLineExtraValue = "bar"; CsvFileToRows fileToRows = CsvFileToRows.forString(csv); @@ -261,7 +261,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals("Homer", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); @@ -283,7 +283,7 @@ class WideRowsToRecordWithSpreadMappingTest extends BaseTest assertEquals("Ned", order.getValueString("shipToName")); assertEquals(defaultStoreId, order.getValue("storeId")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); From 2918235f46fedebbc7af5f66c26a3511245bb848 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:25:17 -0600 Subject: [PATCH 17/84] CE-1955 Add version field to the built BulkLoadProfile --- .../bulk/insert/BulkInsertStepUtils.java | 1 + .../bulk/insert/model/BulkLoadProfile.java | 38 +++++++++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index 3e4ffa40..7c7f008b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -125,6 +125,7 @@ public class BulkInsertStepUtils } BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion(version) .withFieldList(fieldList) .withHasHeaderRow(hasHeaderRow) .withLayout(layout); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java index 8bcfb97c..2fe07b3c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfile.java @@ -33,8 +33,10 @@ import java.util.ArrayList; public class BulkLoadProfile implements Serializable { private ArrayList fieldList; - private Boolean hasHeaderRow; - private String layout; + + private Boolean hasHeaderRow; + private String layout; + private String version; @@ -48,7 +50,6 @@ public class BulkLoadProfile implements Serializable - /******************************************************************************* ** Getter for hasHeaderRow *******************************************************************************/ @@ -110,6 +111,7 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* ** Setter for fieldList *******************************************************************************/ @@ -130,4 +132,34 @@ public class BulkLoadProfile implements Serializable } + /******************************************************************************* + ** Getter for version + *******************************************************************************/ + public String getVersion() + { + return (this.version); + } + + + + /******************************************************************************* + ** Setter for version + *******************************************************************************/ + public void setVersion(String version) + { + this.version = version; + } + + + + /******************************************************************************* + ** Fluent setter for version + *******************************************************************************/ + public BulkLoadProfile withVersion(String version) + { + this.version = version; + return (this); + } + + } From 07c04132773e583939a6de369680124276740475 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:25:40 -0600 Subject: [PATCH 18/84] CE-1955 Initial checkin (plus add a memory-storage table to testutils) --- .../bulk/insert/BulkInsertV2Test.java | 236 ++++++++++++++++++ .../qqq/backend/core/utils/TestUtils.java | 18 ++ 2 files changed, 254 insertions(+) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java new file mode 100644 index 00000000..8beaa891 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java @@ -0,0 +1,236 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for full bulk insert process + *******************************************************************************/ +class BulkInsertV2Test extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + @AfterEach + void beforeAndAfterEach() + { + MemoryRecordStore.getInstance().reset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow1() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvRow2() + { + return (""" + "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois", + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static String getPersonCsvHeaderUsingLabels() + { + return (""" + "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws Exception + { + String defaultEmail = "noone@kingsrook.com"; + + /////////////////////////////////////// + // make sure table is empty to start // + /////////////////////////////////////// + assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); + + QInstance qInstance = QContext.getQInstance(); + String processName = "PersonBulkInsertV2"; + new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); + + ///////////////////////////////////////////////////////// + // start the process - expect to go to the upload step // + ///////////////////////////////////////////////////////// + RunProcessInput runProcessInput = new RunProcessInput(); + runProcessInput.setProcessName(processName); + RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); + String processUUID = runProcessOutput.getProcessUUID(); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); + + ////////////////////////////// + // simulate the file upload // + ////////////////////////////// + String storageReference = UUID.randomUUID() + ".csv"; + StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference); + try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput)) + { + outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); + } + catch(IOException e) + { + throw (e); + } + + ////////////////////////// + // continue post-upload // + ////////////////////////// + runProcessInput.setProcessUUID(processUUID); + runProcessInput.setStartAfterStep("upload"); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); + runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); + assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); + + //////////////////////////////////////////////////////////////////////////////// + // all subsequent steps will want these data - so set up a lambda to set them // + //////////////////////////////////////////////////////////////////////////////// + Consumer addProfileToRunProcessInput = (RunProcessInput input) -> + { + input.addValue("version", "v1"); + input.addValue("layout", "FLAT"); + input.addValue("hasHeaderRow", "true"); + input.addValue("fieldListJSON", JsonUtils.toJson(List.of( + new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), + new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), + new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true), + new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) + ))); + }; + + //////////////////////////////// + // continue post file-mapping // + //////////////////////////////// + runProcessInput.setStartAfterStep("fileMapping"); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + Serializable valueMappingField = runProcessOutput.getValue("valueMappingField"); + assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class); + assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); + assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); + assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); + + ///////////////////////////////// + // continue post value-mapping // + ///////////////////////////////// + runProcessInput.setStartAfterStep("valueMapping"); + runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2))); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); + + ///////////////////////////////// + // continue post review screen // + ///////////////////////////////// + runProcessInput.setStartAfterStep("review"); + addProfileToRunProcessInput.accept(runProcessInput); + runProcessOutput = new RunProcessAction().execute(runProcessInput); + assertThat(runProcessOutput.getRecords()).hasSize(2); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); + assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); + assertThat(runProcessOutput.getException()).isEmpty(); + + //////////////////////////////////// + // query for the inserted records // + //////////////////////////////////// + List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); + + assertEquals("John", records.get(0).getValueString("firstName")); + assertEquals("Jane", records.get(1).getValueString("firstName")); + + assertNotNull(records.get(0).getValue("id")); + assertNotNull(records.get(1).getValue("id")); + + assertEquals(2, records.get(0).getValueInteger("homeStateId")); + assertEquals(1, records.get(1).getValueInteger("homeStateId")); + + assertEquals(defaultEmail, records.get(0).getValueString("email")); + assertEquals(defaultEmail, records.get(1).getValueString("email")); + + assertEquals(42, records.get(0).getValueInteger("noOfShoes")); + assertNull(records.get(1).getValue("noOfShoes")); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 8ca6da8f..89adac13 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -142,6 +142,7 @@ public class TestUtils public static final String APP_NAME_MISCELLANEOUS = "miscellaneous"; public static final String TABLE_NAME_TWO_KEYS = "twoKeys"; + public static final String TABLE_NAME_MEMORY_STORAGE = "memoryStorage"; public static final String TABLE_NAME_PERSON = "person"; public static final String TABLE_NAME_SHAPE = "shape"; public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache"; @@ -204,6 +205,7 @@ public class TestUtils qInstance.addTable(defineTablePerson()); qInstance.addTable(defineTableTwoKeys()); + qInstance.addTable(defineTableMemoryStorage()); qInstance.addTable(definePersonFileTable()); qInstance.addTable(definePersonMemoryTable()); qInstance.addTable(definePersonMemoryCacheTable()); @@ -594,6 +596,22 @@ public class TestUtils + /******************************************************************************* + ** Define a table in the memory store that can be used for the StorageAction + *******************************************************************************/ + public static QTableMetaData defineTableMemoryStorage() + { + return new QTableMetaData() + .withName(TABLE_NAME_MEMORY_STORAGE) + .withLabel("Memory Storage") + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("reference") + .withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false)) + .withField(new QFieldMetaData("contents", QFieldType.BLOB)); + } + + + /******************************************************************************* ** Define the 'person' table used in standard tests. *******************************************************************************/ From 9ad9d526346f3d596fb775c6dcf8e72006254fc3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 19 Nov 2024 10:29:13 -0600 Subject: [PATCH 19/84] CE-1955 Add method defineTableBulkInsertV2 (needs to not be v2 i guess) --- .../core/instances/QInstanceEnricher.java | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 0e101379..ee307632 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -54,6 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPer import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; @@ -74,7 +75,13 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -867,6 +874,116 @@ public class QInstanceEnricher + /******************************************************************************* + ** + *******************************************************************************/ + public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName) + { + qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values())); + + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); + + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( + BulkInsertV2ExtractStep.class, + BulkInsertTransformStep.class, + BulkInsertLoadStep.class, + values + ) + .withName(processName) + .withIcon(new QIcon().withName("library_add")) + .withLabel(table.getLabel() + " Bulk Insert") + .withTableName(table.getName()) + .withIsHidden(true) + .withPermissionRules(qInstance.getDefaultPermissionRules().clone() + .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); + + List editableFields = new ArrayList<>(); + for(QFieldSection section : CollectionUtils.nonNullList(table.getSections())) + { + for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames())) + { + try + { + QFieldMetaData field = table.getField(fieldName); + if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB)) + { + editableFields.add(field); + } + } + catch(Exception e) + { + // shrug? + } + } + } + + String fieldsForHelpText = editableFields.stream() + .map(QFieldMetaData::getLabel) + .collect(Collectors.joining(", ")); + + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() + .withName("upload") + .withLabel("Upload File") + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData() + .withType(QComponentType.HELP_TEXT) + .withValue("previewText", "file upload instructions") + .withValue("text", "Upload a CSV or Excel (.xlsx) file with the following columns:\n" + fieldsForHelpText)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); + + QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() + .withName("prepareFileMapping") + .withCode(new QCodeReference(BulkInsertPrepareFileMappingStep.class)); + + QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() + .withName("fileMapping") + .withLabel("File Mapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); + + QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() + .withName("receiveFileMapping") + .withCode(new QCodeReference(BulkInsertReceiveFileMappingStep.class)); + + QBackendStepMetaData prepareValueMappingStep = new QBackendStepMetaData() + .withName("prepareValueMapping") + .withCode(new QCodeReference(BulkInsertPrepareValueMappingStep.class)); + + QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() + .withName("valueMapping") + .withLabel("Value Mapping") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); + + QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() + .withName("receiveValueMapping") + .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); + + int i = 0; + process.addStep(i++, uploadScreen); + + process.addStep(i++, prepareFileMappingStep); + process.addStep(i++, fileMappingScreen); + process.addStep(i++, receiveFileMappingStep); + + process.addStep(i++, prepareValueMappingStep); + process.addStep(i++, valueMappingScreen); + process.addStep(i++, receiveValueMappingStep); + + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW).setRecordListFields(editableFields); + + ////////////////////////////////////////////////////////////////////////////////////////// + // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // + ////////////////////////////////////////////////////////////////////////////////////////// + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) + .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + + qInstance.addProcess(process); + } + + + /******************************************************************************* ** *******************************************************************************/ From 58ae17bbac3b2d58085066f0c31bc0b080462e0c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 10:07:26 -0600 Subject: [PATCH 20/84] CE-1955 - Bulk load checkpoint: - Switch wide format to identify associations via comma-number-indexes... - Add suggested mappings - use header name instead of column index for mappings - add counts of children process summary lines - excel value/type handling --- .../core/instances/QInstanceEnricher.java | 68 +---- .../BulkInsertPrepareFileMappingStep.java | 128 ++------- .../BulkInsertPrepareValueMappingStep.java | 2 +- .../BulkInsertReceiveFileMappingStep.java | 15 +- .../BulkInsertReceiveValueMappingStep.java | 2 +- .../bulk/insert/BulkInsertStepUtils.java | 1 + .../bulk/insert/BulkInsertTransformStep.java | 29 +- .../bulk/insert/BulkInsertV2ExtractStep.java | 2 +- .../insert/filehandling/XlsxFileToRows.java | 41 ++- .../mapping/BulkInsertWideLayoutMapping.java | 216 -------------- .../mapping/BulkLoadMappingSuggester.java | 232 +++++++++++++++ .../BulkLoadTableStructureBuilder.java | 132 +++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 1 + .../insert/mapping/RowsToRecordInterface.java | 58 ++-- .../bulk/insert/mapping/TallRowsToRecord.java | 1 + .../bulk/insert/mapping/ValueMapper.java | 1 + ...licitFieldNameSuffixIndexBasedMapping.java | 196 +++++++++++++ .../WideRowsToRecordWithExplicitMapping.java | 260 ----------------- .../WideRowsToRecordWithSpreadMapping.java | 1 + .../{mapping => model}/BulkInsertMapping.java | 65 ++--- .../insert/model/BulkLoadProfileField.java | 32 +++ ....java => BulkInsertV2FullProcessTest.java} | 15 +- .../mapping/BulkLoadMappingSuggesterTest.java | 209 ++++++++++++++ .../insert/mapping/FlatRowsToRecordTest.java | 1 + .../insert/mapping/TallRowsToRecordTest.java | 1 + .../bulk/insert/mapping/ValueMapperTest.java | 1 + ...tFieldNameSuffixIndexBasedMappingTest.java | 175 ++++++++++++ ...deRowsToRecordWithExplicitMappingTest.java | 269 ------------------ ...WideRowsToRecordWithSpreadMappingTest.java | 1 + 29 files changed, 1167 insertions(+), 988 deletions(-) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{mapping => model}/BulkInsertMapping.java (93%) rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{BulkInsertV2Test.java => BulkInsertV2FullProcessTest.java} (92%) create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index ee307632..9a48204a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -73,7 +73,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; @@ -81,7 +80,6 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -815,75 +813,11 @@ public class QInstanceEnricher /******************************************************************************* ** *******************************************************************************/ - private void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) + public void defineTableBulkInsert(QInstance qInstance, QTableMetaData table, String processName) { Map values = new HashMap<>(); values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - BulkInsertExtractStep.class, - BulkInsertTransformStep.class, - BulkInsertLoadStep.class, - values - ) - .withName(processName) - .withLabel(table.getLabel() + " Bulk Insert") - .withTableName(table.getName()) - .withIsHidden(true) - .withPermissionRules(qInstance.getDefaultPermissionRules().clone() - .withCustomPermissionChecker(new QCodeReference(BulkTableActionProcessPermissionChecker.class))); - - List editableFields = new ArrayList<>(); - for(QFieldSection section : CollectionUtils.nonNullList(table.getSections())) - { - for(String fieldName : CollectionUtils.nonNullList(section.getFieldNames())) - { - try - { - QFieldMetaData field = table.getField(fieldName); - if(field.getIsEditable() && !field.getType().equals(QFieldType.BLOB)) - { - editableFields.add(field); - } - } - catch(Exception e) - { - // shrug? - } - } - } - - String fieldsForHelpText = editableFields.stream() - .map(QFieldMetaData::getLabel) - .collect(Collectors.joining(", ")); - - QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() - .withName("upload") - .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV file with the following columns:\n" + fieldsForHelpText)) - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); - - process.addStep(0, uploadScreen); - process.getFrontendStep("review").setRecordListFields(editableFields); - qInstance.addProcess(process); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public void defineTableBulkInsertV2(QInstance qInstance, QTableMetaData table, String processName) - { - qInstance.addPossibleValueSource(QPossibleValueSource.newForEnum("bulkInsertFileLayout", BulkInsertMapping.Layout.values())); - - Map values = new HashMap<>(); - values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); - QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( BulkInsertV2ExtractStep.class, BulkInsertTransformStep.class, diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index f7ee32ce..f939df3a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -25,27 +25,19 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; import java.io.File; import java.io.InputStream; import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Set; +import java.util.List; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; -import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; -import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadMappingSuggester; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -62,7 +54,26 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { buildFileDetailsForMappingStep(runBackendStepInput, runBackendStepOutput); - buildFieldsForMappingStep(runBackendStepInput, runBackendStepOutput); + + String tableName = runBackendStepInput.getValueString("tableName"); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void buildSuggestedMapping(List headerValues, BulkLoadTableStructure tableStructure, RunBackendStepOutput runBackendStepOutput) + { + BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); + BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); + runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); } @@ -121,6 +132,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep } } runBackendStepOutput.addValue("bodyValuesPreview", bodyValues); + } catch(Exception e) { @@ -147,94 +159,4 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep return (rs.toString()); } - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void buildFieldsForMappingStep(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) - { - String tableName = runBackendStepInput.getValueString("tableName"); - BulkLoadTableStructure tableStructure = buildTableStructure(tableName, null, null); - runBackendStepOutput.addValue("tableStructure", tableStructure); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) - { - QTableMetaData table = QContext.getQInstance().getTable(tableName); - - BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); - tableStructure.setTableName(tableName); - tableStructure.setLabel(table.getLabel()); - - Set associationJoinFieldNamesToExclude = new HashSet<>(); - - if(association == null) - { - tableStructure.setIsMain(true); - tableStructure.setIsMany(false); - tableStructure.setAssociationPath(null); - } - else - { - tableStructure.setIsMain(false); - - QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); - if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) - { - tableStructure.setIsMany(true); - } - - for(JoinOn joinOn : join.getJoinOns()) - { - //////////////////////////////////////////////////////////////////////////////////////////////// - // don't allow the user to map the "join field" from a child up to its parent // - // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // - //////////////////////////////////////////////////////////////////////////////////////////////// - if(join.getLeftTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); - } - else if(join.getRightTable().equals(tableName)) - { - associationJoinFieldNamesToExclude.add(joinOn.getRightField()); - } - } - - if(!StringUtils.hasContent(parentAssociationPath)) - { - tableStructure.setAssociationPath(association.getName()); - } - else - { - tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); - } - } - - ArrayList fields = new ArrayList<>(); - tableStructure.setFields(fields); - for(QFieldMetaData field : table.getFields().values()) - { - if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) - { - fields.add(field); - } - } - - fields.sort(Comparator.comparing(f -> f.getLabel())); - - for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) - { - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); - tableStructure.addAssociation(associatedStructure); - } - - return (tableStructure); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 85397442..19e696e5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -51,7 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleVal import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index 8f72bb82..bc6ae239 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -37,7 +37,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInpu import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; @@ -102,7 +102,12 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep BulkLoadFileRow headerRow = fileToRowsInterface.next(); for(BulkLoadProfileField bulkLoadProfileField : bulkLoadProfile.getFieldList()) { - if(bulkLoadProfileField.getColumnIndex() != null) + if(bulkLoadProfileField.getHeaderName() != null) + { + String headerName = bulkLoadProfileField.getHeaderName(); + fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); + } + else if(bulkLoadProfileField.getColumnIndex() != null) { String headerName = ValueUtils.getValueAsString(headerRow.getValueElseNull(bulkLoadProfileField.getColumnIndex())); fieldNameToHeaderNameMap.put(bulkLoadProfileField.getFieldName(), headerName); @@ -164,7 +169,11 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep { if(bulkLoadProfileField.getFieldName().contains(".")) { - associationNameSet.add(bulkLoadProfileField.getFieldName().substring(0, bulkLoadProfileField.getFieldName().lastIndexOf('.'))); + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + // handle parent.child.grandchild.fieldName,index.index.index if we do sub-indexes for grandchildren... // + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNameBeforeIndex = bulkLoadProfileField.getFieldName().split(",")[0]; + associationNameSet.add(fieldNameBeforeIndex.substring(0, fieldNameBeforeIndex.lastIndexOf('.'))); } } bulkInsertMapping.setMappedAssociations(new ArrayList<>(associationNameSet)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java index f9c17667..e1b5bc9a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveValueMappingStep.java @@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.utils.JsonUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java index 7c7f008b..b0db3ad0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -109,6 +109,7 @@ public class BulkInsertStepUtils BulkLoadProfileField bulkLoadProfileField = new BulkLoadProfileField(); fieldList.add(bulkLoadProfileField); bulkLoadProfileField.setFieldName(jsonObject.optString("fieldName")); + bulkLoadProfileField.setHeaderName(jsonObject.has("headerName") ? jsonObject.getString("headerName") : null); bulkLoadProfileField.setColumnIndex(jsonObject.has("columnIndex") ? jsonObject.getInt("columnIndex") : null); bulkLoadProfileField.setDefaultValue((Serializable) jsonObject.opt("defaultValue")); bulkLoadProfileField.setDoValueMapping(jsonObject.optBoolean("doValueMapping")); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 6e28c10a..09d9ca8d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; @@ -67,7 +68,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); - private Map ukErrorSummaries = new HashMap<>(); + private Map ukErrorSummaries = new HashMap<>(); + private Map associationsToInsertSummaries = new HashMap<>(); private QTableMetaData table; @@ -259,6 +261,13 @@ public class BulkInsertTransformStep extends AbstractTransformStep { okSummary.incrementCountAndAddPrimaryKey(null); outputRecords.add(record); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + String associationName = entry.getKey(); + ProcessSummaryLine associationToInsertLine = associationsToInsertSummaries.computeIfAbsent(associationName, x -> new ProcessSummaryLine(Status.OK)); + associationToInsertLine.incrementCount(CollectionUtils.nonNullList(entry.getValue()).size()); + } } } @@ -366,6 +375,24 @@ public class BulkInsertTransformStep extends AbstractTransformStep okSummary.pickMessage(isForResultScreen); okSummary.addSelfToListIfAnyCount(rs); + for(Map.Entry entry : associationsToInsertSummaries.entrySet()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(entry.getKey())).findFirst(); + if(association.isPresent()) + { + QTableMetaData associationTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + String associationLabel = associationTable.getLabel(); + + ProcessSummaryLine line = entry.getValue(); + line.setSingularFutureMessage(associationLabel + " record will be inserted."); + line.setPluralFutureMessage(associationLabel + " records will be inserted."); + line.setSingularPastMessage(associationLabel + " record was inserted."); + line.setPluralPastMessage(associationLabel + " records were inserted."); + line.pickMessage(isForResultScreen); + line.addSelfToListIfAnyCount(rs); + } + } + for(Map.Entry entry : ukErrorSummaries.entrySet()) { UniqueKey uniqueKey = entry.getKey(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java index efacca0f..2da27e7d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -32,8 +32,8 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index a6e336de..f5cecd9c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -25,9 +25,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.time.LocalDateTime; +import java.util.Optional; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Sheet; @@ -74,7 +79,41 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Optional dateTime = readerRow.getCellAsDate(i); + if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100) + { + yield dateTime.get(); + } + + Optional optionalBigDecimal = readerRow.getCellAsNumber(i); + if(optionalBigDecimal.isPresent()) + { + BigDecimal bigDecimal = optionalBigDecimal.get(); + if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0) + { + yield bigDecimal.intValue(); + } + + yield bigDecimal; + } + + yield (null); + } + case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null); + case STRING, FORMULA -> cell.getText(); + case EMPTY, ERROR -> null; + }; + } } return new BulkLoadFileRow(values); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java deleted file mode 100644 index ad0cce0f..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertWideLayoutMapping.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class BulkInsertWideLayoutMapping -{ - private List childRecordMappings; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public BulkInsertWideLayoutMapping(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static class ChildRecordMapping - { - Map fieldNameToHeaderNameMaps; - Map associationNameToChildRecordMappingMap; - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Constructor - ** - *******************************************************************************/ - public ChildRecordMapping(Map fieldNameToHeaderNameMaps, Map associationNameToChildRecordMappingMap) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Getter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public Map getFieldNameToHeaderNameMaps() - { - return (this.fieldNameToHeaderNameMaps); - } - - - - /******************************************************************************* - ** Setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public void setFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - } - - - - /******************************************************************************* - ** Fluent setter for fieldNameToHeaderNameMaps - *******************************************************************************/ - public ChildRecordMapping withFieldNameToHeaderNameMaps(Map fieldNameToHeaderNameMaps) - { - this.fieldNameToHeaderNameMaps = fieldNameToHeaderNameMaps; - return (this); - } - - - - /******************************************************************************* - ** Getter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public Map getAssociationNameToChildRecordMappingMap() - { - return (this.associationNameToChildRecordMappingMap); - } - - - - /******************************************************************************* - ** Setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public void setAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - } - - - - /******************************************************************************* - ** Fluent setter for associationNameToChildRecordMappingMap - *******************************************************************************/ - public ChildRecordMapping withAssociationNameToChildRecordMappingMap(Map associationNameToChildRecordMappingMap) - { - this.associationNameToChildRecordMappingMap = associationNameToChildRecordMappingMap; - return (this); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - public Map getFieldIndexes(BulkLoadFileRow headerRow) - { - // todo memoize or otherwise don't recompute - Map rs = new HashMap<>(); - - //////////////////////////////////////////////////////// - // for the current file, map header values to indexes // - //////////////////////////////////////////////////////// - Map headerToIndexMap = new HashMap<>(); - for(int i = 0; i < headerRow.size(); i++) - { - String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - headerToIndexMap.put(headerValue, i); - } - - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - // loop over fields - finding what header name they are mapped to - then what index that header is at. // - ///////////////////////////////////////////////////////////////////////////////////////////////////////// - for(Map.Entry entry : fieldNameToHeaderNameMaps.entrySet()) - { - String headerName = entry.getValue(); - if(headerName != null) - { - Integer headerIndex = headerToIndexMap.get(headerName); - if(headerIndex != null) - { - rs.put(entry.getKey(), headerIndex); - } - } - } - - return (rs); - } - } - - - - /******************************************************************************* - ** Getter for childRecordMappings - *******************************************************************************/ - public List getChildRecordMappings() - { - return (this.childRecordMappings); - } - - - - /******************************************************************************* - ** Setter for childRecordMappings - *******************************************************************************/ - public void setChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - } - - - - /******************************************************************************* - ** Fluent setter for childRecordMappings - *******************************************************************************/ - public BulkInsertWideLayoutMapping withChildRecordMappings(List childRecordMappings) - { - this.childRecordMappings = childRecordMappings; - return (this); - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java new file mode 100644 index 00000000..71f0edcc --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggester.java @@ -0,0 +1,232 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; + + +/******************************************************************************* + ** Given a bulk-upload, create a suggested mapping + *******************************************************************************/ +public class BulkLoadMappingSuggester +{ + private Map massagedHeadersWithoutNumbersToIndexMap; + private Map massagedHeadersWithNumbersToIndexMap; + + private String layout = "FLAT"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public BulkLoadProfile suggestBulkLoadMappingProfile(BulkLoadTableStructure tableStructure, List headerRow) + { + massagedHeadersWithoutNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), true); + + if(!massagedHeadersWithoutNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithoutNumbersToIndexMap.put(headerValue, i); + } + } + + massagedHeadersWithNumbersToIndexMap = new LinkedHashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = massageHeader(headerRow.get(i), false); + + if(!massagedHeadersWithNumbersToIndexMap.containsKey(headerValue)) + { + massagedHeadersWithNumbersToIndexMap.put(headerValue, i); + } + } + + ArrayList fieldList = new ArrayList<>(); + processTable(tableStructure, fieldList, headerRow); + + ///////////////////////////////////////////////// + // sort the fields to match the column indexes // + ///////////////////////////////////////////////// + fieldList.sort(Comparator.comparing(blpf -> blpf.getColumnIndex())); + + BulkLoadProfile bulkLoadProfile = new BulkLoadProfile() + .withVersion("v1") + .withLayout(layout) + .withHasHeaderRow(true) + .withFieldList(fieldList); + + return (bulkLoadProfile); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processTable(BulkLoadTableStructure tableStructure, ArrayList fieldList, List headerRow) + { + Map rs = new HashMap<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + String fieldName = massageHeader(field.getName(), false); + String fieldLabel = massageHeader(field.getLabel(), false); + String tablePlusFieldLabel = massageHeader(QContext.getQInstance().getTable(tableStructure.getTableName()).getLabel() + ": " + field.getLabel(), false); + String fullFieldName = (StringUtils.hasContent(tableStructure.getAssociationPath()) ? (tableStructure.getAssociationPath() + ".") : "") + field.getName(); + + //////////////////////////////////////////////////////////////////////////////////// + // consider, if this is a many-table, if there are many matches, for wide mode... // + //////////////////////////////////////////////////////////////////////////////////// + if(tableStructure.getIsMany()) + { + List matchingIndexes = new ArrayList<>(); + + for(Map.Entry entry : massagedHeadersWithNumbersToIndexMap.entrySet()) + { + String header = entry.getKey(); + if(header.matches(fieldName + "\\d*$") || header.matches(fieldLabel + "\\d*$")) + { + matchingIndexes.add(entry.getValue()); + } + } + + if(CollectionUtils.nullSafeHasContents(matchingIndexes)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////// + // if we found more than 1 match - consider this a likely wide file, and build fields as wide-fields // + // else, if only 1, allow us to go down into the TALL block below // + // note - should we do a merger at the end, in case we found some wide, some tall? // + /////////////////////////////////////////////////////////////////////////////////////////////////////// + if(matchingIndexes.size() > 1) + { + layout = "WIDE"; + + int i = 0; + for(Integer index : matchingIndexes) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName + "," + i) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + i++; + } + + continue; + } + } + } + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // else - look for matches, first w/ headers with numbers, then headers w/o numbers checking labels and names // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Integer index = null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for each of these potential identities of the field: // + // 1) its label, massaged // + // 2) its name, massaged // + // 3) its label, massaged, with numbers stripped away // + // 4) its name, massaged, with numbers stripped away // + // check if that identity is in the massagedHeadersWithNumbersToIndexMap, or the massagedHeadersWithoutNumbersToIndexMap. // + // this is currently successful in the both versions of the address 1 / address 2 <=> address / address 2 use-case // + // that is, BulkLoadMappingSuggesterTest.testChallengingAddress1And2 // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String fieldIdentity : ListBuilder.of(fieldLabel, fieldName, tablePlusFieldLabel, massageHeader(fieldLabel, true), massageHeader(fieldName, true))) + { + if(massagedHeadersWithNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithNumbersToIndexMap.get(fieldIdentity); + } + else if(massagedHeadersWithoutNumbersToIndexMap.containsKey(fieldIdentity)) + { + index = massagedHeadersWithoutNumbersToIndexMap.get(fieldIdentity); + } + + if(index != null) + { + break; + } + } + + if(index != null) + { + fieldList.add(new BulkLoadProfileField() + .withFieldName(fullFieldName) + .withHeaderName(headerRow.get(index)) + .withColumnIndex(index) + ); + + if(tableStructure.getIsMany() && layout.equals("FLAT")) + { + ////////////////////////////////////////////////////////////////////////////////////////// + // the first time we find an is-many child, if we were still marked as flat, go to tall // + ////////////////////////////////////////////////////////////////////////////////////////// + layout = "TALL"; + } + } + } + + //////////////////////////////////////////// + // recursively process child associations // + //////////////////////////////////////////// + for(BulkLoadTableStructure associationTableStructure : CollectionUtils.nonNullList(tableStructure.getAssociations())) + { + processTable(associationTableStructure, fieldList, headerRow); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String massageHeader(String header, boolean stripNumbers) + { + if(header == null) + { + return (null); + } + + String massagedWithNumbers = header.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", ""); + return stripNumbers ? massagedWithNumbers.replaceAll("[0-9]", "") : massagedWithNumbers; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java new file mode 100644 index 00000000..0ac81f00 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ObjectUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** utility to build BulkLoadTableStructure objects for a QQQ Table. + *******************************************************************************/ +public class BulkLoadTableStructureBuilder +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static BulkLoadTableStructure buildTableStructure(String tableName) + { + return (buildTableStructure(tableName, null, null)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static BulkLoadTableStructure buildTableStructure(String tableName, Association association, String parentAssociationPath) + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + + BulkLoadTableStructure tableStructure = new BulkLoadTableStructure(); + tableStructure.setTableName(tableName); + tableStructure.setLabel(table.getLabel()); + + Set associationJoinFieldNamesToExclude = new HashSet<>(); + + if(association == null) + { + tableStructure.setIsMain(true); + tableStructure.setIsMany(false); + tableStructure.setAssociationPath(null); + } + else + { + tableStructure.setIsMain(false); + + QJoinMetaData join = QContext.getQInstance().getJoin(association.getJoinName()); + if(join.getType().equals(JoinType.ONE_TO_MANY) || join.getType().equals(JoinType.MANY_TO_ONE)) + { + tableStructure.setIsMany(true); + } + + for(JoinOn joinOn : join.getJoinOns()) + { + //////////////////////////////////////////////////////////////////////////////////////////////// + // don't allow the user to map the "join field" from a child up to its parent // + // (e.g., you can't map lineItem.orderId -- that'll happen automatically via the association) // + //////////////////////////////////////////////////////////////////////////////////////////////// + if(join.getLeftTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getLeftField()); + } + else if(join.getRightTable().equals(tableName)) + { + associationJoinFieldNamesToExclude.add(joinOn.getRightField()); + } + } + + if(!StringUtils.hasContent(parentAssociationPath)) + { + tableStructure.setAssociationPath(association.getName()); + } + else + { + tableStructure.setAssociationPath(parentAssociationPath + "." + association.getName()); + } + } + + ArrayList fields = new ArrayList<>(); + tableStructure.setFields(fields); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getIsEditable() && !associationJoinFieldNamesToExclude.contains(field.getName())) + { + fields.add(field); + } + } + + fields.sort(Comparator.comparing(f -> ObjectUtils.requireNonNullElse(f.getLabel(), f.getName(), ""))); + + for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) + { + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); + tableStructure.addAssociation(associatedStructure); + } + + return (tableStructure); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 44702312..d1a5d4c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 25aa6543..740781c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -27,9 +27,10 @@ import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -48,42 +49,49 @@ public interface RowsToRecordInterface /*************************************************************************** ** returns true if value from row was used, else false. ***************************************************************************/ - default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex) { - String fieldName = field.getName(); - QFieldType type = field.getType(); + return setValueOrDefault(record, field, associationNameChain, mapping, row, columnIndex, null); + } - boolean valueFromRowWasUsed = false; - String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + /*************************************************************************** + ** returns true if value from row was used, else false. + ***************************************************************************/ + default boolean setValueOrDefault(QRecord record, QFieldMetaData field, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer columnIndex, List wideAssociationIndexes) + { + boolean valueFromRowWasUsed = false; - Serializable value = null; - if(index != null && row != null) + ///////////////////////////////////////////////////////////////////////////////////////////////// + // build full field-name -- possibly associations, then field name, then possibly index-suffix // + ///////////////////////////////////////////////////////////////////////////////////////////////// + String fieldName = field.getName(); + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) { - value = row.getValueElseNull(index); + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + String fullFieldName = fieldNameWithAssociationPrefix + wideAssociationSuffix; + + ////////////////////////////////////////////// + // ok - look in the row - then the defaults // + ////////////////////////////////////////////// + Serializable value = null; + if(columnIndex != null && row != null) + { + value = row.getValueElseNull(columnIndex); if(value != null && !"".equals(value)) { valueFromRowWasUsed = true; } } - else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fullFieldName)) { - value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); } - /* note - moving this to ValueMapper... - if(value != null) - { - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); - } - } - */ - if(value != null) { record.setValue(fieldName, value); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 690f705d..4587336c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -36,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index 0f1e8cfc..fdf1b6bd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java new file mode 100644 index 00000000..9c85647e --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -0,0 +1,196 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** use a flatter mapping object, where field names look like: + ** associationChain.fieldName,index.subIndex + *******************************************************************************/ +public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping, table); + + return (rs); + } + + + + /*************************************************************************** + ** may return null, if there were no values in the row for this (sub-wide) record. + ***************************************************************************/ + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + { + ////////////////////////////////////////////////////// + // start by building the record with its own fields // + ////////////////////////////////////////////////////// + QRecord record = new QRecord(); + boolean hadAnyValuesInRow = false; + for(QFieldMetaData field : table.getFields().values()) + { + hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + } + + if(!hadAnyValuesInRow) + { + return (null); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, row, headerRow); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException + { + List rs = new ArrayList<>(); + + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + for(int i = 0; true; i++) + { + // todo - doesn't support grand-children + List wideAssociationIndexes = List.of(i); + Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + if(fieldIndexes.isEmpty()) + { + break; + } + + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + if(record != null) + { + rs.add(record); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java deleted file mode 100644 index adc67ec7..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMapping.java +++ /dev/null @@ -1,260 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.Pair; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class WideRowsToRecordWithExplicitMapping implements RowsToRecordInterface -{ - private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); - - - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException - { - QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); - if(table == null) - { - throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); - } - - List rs = new ArrayList<>(); - - Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); - - while(fileToRowsInterface.hasNext() && rs.size() < limit) - { - BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = new QRecord(); - - for(QFieldMetaData field : table.getFields().values()) - { - setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); - } - - processAssociations(mapping.getWideLayoutMapping(), "", headerRow, mapping, table, row, record); - - rs.add(record); - } - - ValueMapper.valueMapping(rs, mapping, table); - - return (rs); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void processAssociations(Map mappingMap, String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record) throws QException - { - for(Map.Entry entry : CollectionUtils.nonNullMap(mappingMap).entrySet()) - { - String associationName = entry.getKey(); - BulkInsertWideLayoutMapping bulkInsertWideLayoutMapping = entry.getValue(); - - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isEmpty()) - { - throw (new QException("Couldn't find association: " + associationName + " under table: " + table.getName())); - } - - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - - String subChain = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName: associationName; - - for(BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping : bulkInsertWideLayoutMapping.getChildRecordMappings()) - { - QRecord associatedRecord = processAssociation(associatedTable, subChain, childRecordMapping, mapping, row, headerRow); - if(associatedRecord != null) - { - record.withAssociatedRecord(associationName, associatedRecord); - } - } - } - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private QRecord processAssociation(QTableMetaData table, String associationNameChain, BulkInsertWideLayoutMapping.ChildRecordMapping childRecordMapping, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow) throws QException - { - Map fieldIndexes = childRecordMapping.getFieldIndexes(headerRow); - - QRecord associatedRecord = new QRecord(); - boolean usedAnyValuesFromRow = false; - - for(QFieldMetaData field : table.getFields().values()) - { - boolean valueFromRowWasUsed = setValueOrDefault(associatedRecord, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName())); - usedAnyValuesFromRow |= valueFromRowWasUsed; - } - - if(usedAnyValuesFromRow) - { - processAssociations(childRecordMapping.getAssociationNameToChildRecordMappingMap(), associationNameChain, headerRow, mapping, table, row, associatedRecord); - return (associatedRecord); - } - else - { - return (null); - } - } - - // /*************************************************************************** - // ** - // ***************************************************************************/ - // private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException - // { - // List rs = new ArrayList<>(); - - // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); - // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) - // { - // if(entry.getKey().startsWith(associationName + ".")) - // { - // String fieldName = entry.getKey().substring(associationName.length() + 1); - - // ////////////////////////////////////////////////////////////////////////// - // // make sure the name here is for this table - not a sub-table under it // - // ////////////////////////////////////////////////////////////////////////// - // if(!fieldName.contains(".")) - // { - // fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); - // } - // } - // } - - // ///////////////////////////////////////////////////////////////////// - // // loop over the length of the record, building associated records // - // ///////////////////////////////////////////////////////////////////// - // QRecord associatedRecord = new QRecord(); - // Set processedFieldNames = new HashSet<>(); - // boolean gotAnyValues = false; - // int subStartIndex = -1; - - // for(int i = startIndex; i < endIndex; i++) - // { - // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); - - // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) - // { - // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) - // { - // /////////////////////////////////////////////// - // // ok - this is a value for this association // - // /////////////////////////////////////////////// - // if(subStartIndex == -1) - // { - // subStartIndex = i; - // } - - // String fieldName = entry.getKey(); - // if(processedFieldNames.contains(fieldName)) - // { - // ///////////////////////////////////////////////// - // // this means we're starting a new sub-record! // - // ///////////////////////////////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); - // rs.add(associatedRecord); - // } - - // associatedRecord = new QRecord(); - // processedFieldNames = new HashSet<>(); - // gotAnyValues = false; - // subStartIndex = i + 1; - // } - - // processedFieldNames.add(fieldName); - - // Serializable value = row.getValueElseNull(i); - // if(value != null && !"".equals(value)) - // { - // gotAnyValues = true; - // } - - // setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); - // } - // } - // } - - // //////////////////////// - // // handle final value // - // //////////////////////// - // if(gotAnyValues) - // { - // addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); - // processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); - // rs.add(associatedRecord); - // } - - // return (rs); - // } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) - { - for(QFieldMetaData field : table.getFields().values()) - { - if(!processedFieldNames.contains(field.getName())) - { - setValueOrDefault(associatedRecord, field, associationNameChain, mapping, null, null); - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index f87fa17b..7b215d2e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java similarity index 93% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index fef870b1..86c19775 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model; import java.io.Serializable; @@ -34,7 +34,10 @@ import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.FlatRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.TallRowsToRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -63,8 +66,7 @@ public class BulkInsertMapping implements Serializable private Map fieldNameToDefaultValueMap = new HashMap<>(); private Map> fieldNameToValueMapping = new HashMap<>(); - private Map> tallLayoutGroupByIndexMap = new HashMap<>(); - private Map wideLayoutMapping = new HashMap<>(); + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); private List mappedAssociations = new ArrayList<>(); @@ -79,7 +81,7 @@ public class BulkInsertMapping implements Serializable { FLAT(FlatRowsToRecord::new), TALL(TallRowsToRecord::new), - WIDE(WideRowsToRecordWithExplicitMapping::new); + WIDE(WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping::new); /*************************************************************************** @@ -137,10 +139,21 @@ public class BulkInsertMapping implements Serializable ***************************************************************************/ @JsonIgnore public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + return getFieldIndexes(table, associationNameChain, headerRow, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException { if(hasHeaderRow && fieldNameToHeaderNameMap != null) { - return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow, wideAssociationIndexes)); } else if(fieldNameToIndexMap != null) { @@ -208,7 +221,7 @@ public class BulkInsertMapping implements Serializable /*************************************************************************** ** ***************************************************************************/ - private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow, List wideAssociationIndexes) { Map rs = new HashMap<>(); @@ -222,13 +235,19 @@ public class BulkInsertMapping implements Serializable headerToIndexMap.put(headerValue, i); } + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////// // loop over fields - finding what header name they are mapped to - then what index that header is at. // ///////////////////////////////////////////////////////////////////////////////////////////////////////// String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; for(QFieldMetaData field : table.getFields().values()) { - String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); if(headerName != null) { Integer headerIndex = headerToIndexMap.get(headerName); @@ -527,34 +546,4 @@ public class BulkInsertMapping implements Serializable } - - /******************************************************************************* - ** Getter for wideLayoutMapping - *******************************************************************************/ - public Map getWideLayoutMapping() - { - return (this.wideLayoutMapping); - } - - - - /******************************************************************************* - ** Setter for wideLayoutMapping - *******************************************************************************/ - public void setWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - } - - - - /******************************************************************************* - ** Fluent setter for wideLayoutMapping - *******************************************************************************/ - public BulkInsertMapping withWideLayoutMapping(Map wideLayoutMapping) - { - this.wideLayoutMapping = wideLayoutMapping; - return (this); - } - } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java index 6bedb18b..f7ce0f6c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkLoadProfileField.java @@ -33,6 +33,7 @@ public class BulkLoadProfileField { private String fieldName; private Integer columnIndex; + private String headerName; private Serializable defaultValue; private Boolean doValueMapping; private Map valueMappings; @@ -192,4 +193,35 @@ public class BulkLoadProfileField return (this); } + + /******************************************************************************* + ** Getter for headerName + *******************************************************************************/ + public String getHeaderName() + { + return (this.headerName); + } + + + + /******************************************************************************* + ** Setter for headerName + *******************************************************************************/ + public void setHeaderName(String headerName) + { + this.headerName = headerName; + } + + + + /******************************************************************************* + ** Fluent setter for headerName + *******************************************************************************/ + public BulkLoadProfileField withHeaderName(String headerName) + { + this.headerName = headerName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java similarity index 92% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java index 8beaa891..7edbaafa 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2Test.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.JsonUtils; @@ -58,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertV2Test extends BaseTest +class BulkInsertV2FullProcessTest extends BaseTest { /******************************************************************************* @@ -124,7 +125,7 @@ class BulkInsertV2Test extends BaseTest QInstance qInstance = QContext.getQInstance(); String processName = "PersonBulkInsertV2"; - new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); + new QInstanceEnricher(qInstance).defineTableBulkInsert(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName); ///////////////////////////////////////////////////////// // start the process - expect to go to the upload step // @@ -159,6 +160,16 @@ class BulkInsertV2Test extends BaseTest runProcessOutput = new RunProcessAction().execute(runProcessInput); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters")); + + ////////////////////////////////////////////////////// + // assert about the suggested mapping that was done // + ////////////////////////////////////////////////////// + Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); + assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); + assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); //////////////////////////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java new file mode 100644 index 00000000..a54e08ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadMappingSuggesterTest.java @@ -0,0 +1,209 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.List; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for BulkLoadMappingSuggester + *******************************************************************************/ +class BulkLoadMappingSuggesterTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleFlat() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_PERSON_MEMORY); + List headerRow = List.of("Id", "First Name", "lastname", "email", "homestate"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("FLAT", bulkLoadProfile.getLayout()); + assertNull(getFieldByName(bulkLoadProfile, "id")); + assertEquals(1, getFieldByName(bulkLoadProfile, "firstName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "lastName").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "email").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "homeStateId").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleTall() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "shipto name", "sku", "quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testTallWithTableNamesOnAssociations() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("Order No", "Ship To Name", "Order Line: SKU", "Order Line: Quantity"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("TALL", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity").getColumnIndex()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testChallengingAddress1And2() + { + try + { + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address 1", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + + { + QTableMetaData table = QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER); + table.addField(new QFieldMetaData("address1", QFieldType.STRING)); + table.addField(new QFieldMetaData("address2", QFieldType.STRING)); + + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "address", "address 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "address1").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "address2").getColumnIndex()); + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + finally + { + reInitInstanceInContext(TestUtils.defineInstance()); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimpleWide() + { + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(TestUtils.TABLE_NAME_ORDER); + List headerRow = List.of("orderNo", "ship to name", "sku", "quantity1", "sku 2", "quantity 2"); + + BulkLoadProfile bulkLoadProfile = new BulkLoadMappingSuggester().suggestBulkLoadMappingProfile(tableStructure, headerRow); + assertEquals("v1", bulkLoadProfile.getVersion()); + assertEquals("WIDE", bulkLoadProfile.getLayout()); + assertEquals(0, getFieldByName(bulkLoadProfile, "orderNo").getColumnIndex()); + assertEquals(1, getFieldByName(bulkLoadProfile, "shipToName").getColumnIndex()); + assertEquals(2, getFieldByName(bulkLoadProfile, "orderLine.sku,0").getColumnIndex()); + assertEquals(3, getFieldByName(bulkLoadProfile, "orderLine.quantity,0").getColumnIndex()); + assertEquals(4, getFieldByName(bulkLoadProfile, "orderLine.sku,1").getColumnIndex()); + assertEquals(5, getFieldByName(bulkLoadProfile, "orderLine.quantity,1").getColumnIndex()); + + ///////////////////////////////////////////////////////////////// + // assert that the order of fields matches the file's ordering // + ///////////////////////////////////////////////////////////////// + assertEquals(List.of("orderNo", "shipToName", "orderLine.sku,0", "orderLine.quantity,0", "orderLine.sku,1", "orderLine.quantity,1"), + bulkLoadProfile.getFieldList().stream().map(f -> f.getFieldName()).toList()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private BulkLoadProfileField getFieldByName(BulkLoadProfile bulkLoadProfile, String fieldName) + { + return (bulkLoadProfile.getFieldList().stream() + .filter(f -> f.getFieldName().equals(fieldName)) + .findFirst().orElse(null)); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 435dcf92..7124e946 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 5c942fa2..1a0ef2ab 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java index 7317d963..4348cb2b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.json.JSONArray; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java new file mode 100644 index 00000000..6e59c527 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping + *******************************************************************************/ +class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1", + "orderLine.sku,1", "SKU 2", + "orderLine.quantity,1", "Quantity 2", + "orderLine.sku,2", "SKU 3", + "orderLine.quantity,2", "Quantity 3" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(MapBuilder.of(() -> new HashMap()) + .with("orderNo", "orderNo") + .with("shipToName", "Ship To") + + .with("orderLine.sku,0", "SKU 1") + .with("orderLine.quantity,0", "Quantity 1") + .with("orderLine.sku,1", "SKU 2") + .with("orderLine.quantity,1", "Quantity 2") + .with("orderLine.sku,2", "SKU 3") + .with("orderLine.quantity,2", "Quantity 3") + + .with("extrinsics.key,0", "Extrinsic Key 1") + .with("extrinsics.value,0", "Extrinsic Value 1") + .with("extrinsics.key,1", "Extrinsic Key 2") + .with("extrinsics.value,1", "Extrinsic Value 2") + .build() + ) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.lineNumber,0", "1", + "orderLine.lineNumber,1", "2", + "orderLine.lineNumber,2", "3" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2", "3"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("1", "2"), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java deleted file mode 100644 index 4f8bd7a1..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitMappingTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; - - -/******************************************************************************* - ** Unit test for WideRowsToRecord - *******************************************************************************/ -class WideRowsToRecordWithExplicitMappingTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderAndLinesWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )) - )) - .withMappedAssociations(List.of("orderLine")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff - 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 - """; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 1", "quantity", "Quantity 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 2", "quantity", "Quantity 2")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("sku", "SKU 3", "quantity", "Quantity 3")) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - .withMappedAssociations(List.of("orderLine", "extrinsics")) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException - { - String csv = """ - orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1.1, Line Extrinsic Value 1.1, Line Extrinsic Key 1.2, Line Extrinsic Value 1.2, SKU 2, Quantity 2, Line Extrinsic Key 2.1, Line Extrinsic Value 2.1, SKU 3, Quantity 3, Line Extrinsic Key 3.1, Line Extrinsic Value 3.1, Line Extrinsic Key 3.2 - 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, - 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 - """; - - Integer defaultStoreId = 42; - String defaultLineNo = "47"; - String defaultLineExtraValue = "bar"; - - CsvFileToRows fileToRows = CsvFileToRows.forString(csv); - BulkLoadFileRow header = fileToRows.next(); - - WideRowsToRecordWithExplicitMapping rowsToRecord = new WideRowsToRecordWithExplicitMapping(); - - BulkInsertMapping mapping = new BulkInsertMapping() - .withFieldNameToHeaderNameMap(Map.of( - "orderNo", "orderNo", - "shipToName", "Ship To" - )) - .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) - .withWideLayoutMapping(Map.of( - "orderLine", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 1", "quantity", "Quantity 1"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.1", "value", "Line Extrinsic Value 1.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 1.2", "value", "Line Extrinsic Value 1.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 2", "quantity", "Quantity 2"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.1", "value", "Line Extrinsic Value 2.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 2.2", "value", "Line Extrinsic Value 2.2")) - )))), - new BulkInsertWideLayoutMapping.ChildRecordMapping( - Map.of("sku", "SKU 3", "quantity", "Quantity 3"), - Map.of("extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.1", "value", "Line Extrinsic Value 3.1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Line Extrinsic Key 3.2", "value", "Line Extrinsic Value 3.2")) - )))) - )), - "extrinsics", new BulkInsertWideLayoutMapping(List.of( - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 1", "value", "Extrinsic Value 1")), - new BulkInsertWideLayoutMapping.ChildRecordMapping(Map.of("key", "Extrinsic Key 2", "value", "Extrinsic Value 2")) - )) - )) - - .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) - .withFieldNameToDefaultValueMap(Map.of( - "storeId", defaultStoreId, - "orderLine.lineNumber", defaultLineNo, - "orderLine.extrinsics.value", defaultLineExtraValue - )) - .withTableName(TestUtils.TABLE_NAME_ORDER) - .withLayout(BulkInsertMapping.Layout.WIDE) - .withHasHeaderRow(true); - - List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); - assertEquals(2, records.size()); - - QRecord order = records.get(0); - assertEquals(1, order.getValueInteger("orderNo")); - assertEquals("Homer", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); - - QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(1); - assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - lineItem = order.getAssociatedRecords().get("orderLine").get(2); - assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("Brown", defaultLineExtraValue), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - - order = records.get(1); - assertEquals(2, order.getValueInteger("orderNo")); - assertEquals("Ned", order.getValueString("shipToName")); - assertEquals(defaultStoreId, order.getValue("storeId")); - assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); - assertEquals(List.of(defaultLineNo, defaultLineNo), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); - assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); - - lineItem = order.getAssociatedRecords().get("orderLine").get(0); - assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); - assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private List getValues(List records, String fieldName) - { - return (records.stream().map(r -> r.getValue(fieldName)).toList()); - } - -} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java index 61fc6d35..8bd49beb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMappingTest.java @@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; From bdbb2d2d002171aba5438cd2499e6b3a1fc322a4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 10:09:05 -0600 Subject: [PATCH 21/84] CE-1955 - Bulk load checkpoint - setting uploadFileArchiveTable in javalin metadata --- .../java/com/kingsrook/sampleapp/SampleJavalinServer.java | 4 ++++ .../kingsrook/sampleapp/metadata/SampleMetaDataProvider.java | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index 37f3e3d1..ad8166b0 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -25,6 +25,7 @@ package com.kingsrook.sampleapp; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; +import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; import com.kingsrook.sampleapp.metadata.SampleMetaDataProvider; import io.javalin.Javalin; import io.javalin.plugin.bundled.CorsPluginConfig; @@ -67,6 +68,9 @@ public class SampleJavalinServer SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); + qJavalinImplementation.setJavalinMetaData(new QJavalinMetaData() + .withUploadedFileArchiveTableName(SampleMetaDataProvider.UPLOAD_FILE_ARCHIVE_TABLE_NAME)); + javalinService = Javalin.create(config -> { // todo - not all? diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 37526837..77b05601 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -376,7 +376,7 @@ public class SampleMetaDataProvider .withField(new QFieldMetaData("firstName", QFieldType.STRING).withBackendName("first_name").withIsRequired(true)) .withField(new QFieldMetaData("lastName", QFieldType.STRING).withBackendName("last_name").withIsRequired(true)) .withField(new QFieldMetaData("birthDate", QFieldType.DATE).withBackendName("birth_date")) - .withField(new QFieldMetaData("email", QFieldType.STRING)) + .withField(new QFieldMetaData("email", QFieldType.STRING).withIsRequired(true)) .withField(new QFieldMetaData("isEmployed", QFieldType.BOOLEAN).withBackendName("is_employed")) .withField(new QFieldMetaData("annualSalary", QFieldType.DECIMAL).withBackendName("annual_salary").withDisplayFormat(DisplayFormat.CURRENCY)) .withField(new QFieldMetaData("daysWorked", QFieldType.INTEGER).withBackendName("days_worked").withDisplayFormat(DisplayFormat.COMMAS)) @@ -769,7 +769,7 @@ public class SampleMetaDataProvider CAT(1, "Cat"); private final Integer id; - private final String label; + private final String label; public static final String NAME = "petSpecies"; From 3c06e0e589a8f2e68d63eb5339c5db2eb03116c3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:10:01 -0600 Subject: [PATCH 22/84] CE-1955 - Test fixes --- .../actions/reporting/GenerateReportActionTest.java | 6 ++++-- .../bulk/insert/BulkInsertV2FullProcessTest.java | 8 ++++++-- .../bulk/insert/filehandling/XlsxFileToRowsTest.java | 12 ++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java index 984a9aa2..0f84cc78 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/reporting/GenerateReportActionTest.java @@ -643,7 +643,7 @@ public class GenerateReportActionTest extends BaseTest Iterator> iterator = list.iterator(); Map row = iterator.next(); assertEquals(5, list.size()); - assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name"); + assertThat(row).containsOnlyKeys("Id", "First Name", "Last Name", "Birth Date"); } @@ -674,7 +674,9 @@ public class GenerateReportActionTest extends BaseTest .withColumns(List.of( new QReportField().withName("id"), new QReportField().withName("firstName"), - new QReportField().withName("lastName"))))); + new QReportField().withName("lastName"), + new QReportField().withName("birthDate") + )))); return report; } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java index 7edbaafa..b721d89d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java @@ -167,8 +167,12 @@ class BulkInsertV2FullProcessTest extends BaseTest Serializable bulkLoadProfile = runProcessOutput.getValue("bulkLoadProfile"); assertThat(bulkLoadProfile).isInstanceOf(BulkLoadProfile.class); assertThat(((BulkLoadProfile) bulkLoadProfile).getFieldList()).hasSizeGreaterThan(5); - assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); - assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("firstName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getFieldName()); + assertEquals(3, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(0).getColumnIndex()); + assertEquals("lastName", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getFieldName()); + assertEquals(4, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(1).getColumnIndex()); + assertEquals("birthDate", ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getFieldName()); + assertEquals(5, ((BulkLoadProfile) bulkLoadProfile).getFieldList().get(2).getColumnIndex()); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index c681f685..d8c89180 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -24,7 +24,10 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.fil import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.Serializable; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.Month; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -54,7 +57,7 @@ class XlsxFileToRowsTest extends BaseTest ** *******************************************************************************/ @Test - void test() throws QException + void test() throws QException, IOException { byte[] byteArray = writeExcelBytes(); @@ -63,8 +66,8 @@ class XlsxFileToRowsTest extends BaseTest BulkLoadFileRow headerRow = fileToRowsInterface.next(); BulkLoadFileRow bodyRow = fileToRowsInterface.next(); - assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow); - assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow); + assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name", "Birth Date"}), headerRow); + assertEquals(new BulkLoadFileRow(new Serializable[] {1, "Darin", "Jonson", LocalDateTime.of(1980, Month.JANUARY, 31, 0, 0)}), bodyRow); /////////////////////////////////////////////////////////////////////////////////////// // make sure there's at least a limit (less than 20) to how many more rows there are // @@ -83,7 +86,7 @@ class XlsxFileToRowsTest extends BaseTest /*************************************************************************** ** ***************************************************************************/ - private static byte[] writeExcelBytes() throws QException + private static byte[] writeExcelBytes() throws QException, IOException { ReportFormat format = ReportFormat.XLSX; ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -100,6 +103,7 @@ class XlsxFileToRowsTest extends BaseTest new GenerateReportAction().execute(reportInput); byte[] byteArray = baos.toByteArray(); + // FileUtils.writeByteArrayToFile(new File("/tmp/xlsx.xlsx"), byteArray); return byteArray; } From c883749ba9e25bbdf33dce85fd4068852792aec3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:15:13 -0600 Subject: [PATCH 23/84] CE-1955 - Remove bulk-insert v1 test; rename bulkInsertV2 test --- ...st.java => BulkInsertFullProcessTest.java} | 2 +- .../bulk/insert/BulkInsertTest.java | 157 ------------------ 2 files changed, 1 insertion(+), 158 deletions(-) rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/{BulkInsertV2FullProcessTest.java => BulkInsertFullProcessTest.java} (99%) delete mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java similarity index 99% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index b721d89d..7037662a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2FullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -59,7 +59,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; /******************************************************************************* ** Unit test for full bulk insert process *******************************************************************************/ -class BulkInsertV2FullProcessTest extends BaseTest +class BulkInsertFullProcessTest extends BaseTest { /******************************************************************************* diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java deleted file mode 100644 index 496ad0be..00000000 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; -import com.kingsrook.qqq.backend.core.state.StateType; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; -import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - - -/******************************************************************************* - ** Unit test for full bulk insert process - *******************************************************************************/ -class BulkInsertTest extends BaseTest -{ - - /******************************************************************************* - ** - *******************************************************************************/ - @BeforeEach - @AfterEach - void beforeAndAfterEach() - { - MemoryRecordStore.getInstance().reset(); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow1() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvRow2() - { - return (""" - "0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public static String getPersonCsvHeaderUsingLabels() - { - return (""" - "Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email" - """); - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - @Test - void test() throws QException - { - /////////////////////////////////////// - // make sure table is empty to start // - /////////////////////////////////////// - assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty(); - - //////////////////////////////////////////////////////////////// - // create an uploaded file, similar to how an http server may // - //////////////////////////////////////////////////////////////// - QUploadedFile qUploadedFile = new QUploadedFile(); - qUploadedFile.setBytes((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes()); - qUploadedFile.setFilename("test.csv"); - UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE); - TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile); - - RunProcessInput runProcessInput = new RunProcessInput(); - runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert"); - RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); - String processUUID = runProcessOutput.getProcessUUID(); - - runProcessInput.setProcessUUID(processUUID); - runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - - runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class); - - runProcessInput.setStartAfterStep("review"); - runProcessOutput = new RunProcessAction().execute(runProcessInput); - assertThat(runProcessOutput.getRecords()).hasSize(2); - assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result"); - assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class); - assertThat(runProcessOutput.getException()).isEmpty(); - - //////////////////////////////////// - // query for the inserted records // - //////////////////////////////////// - List records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY); - assertEquals("John", records.get(0).getValueString("firstName")); - assertEquals("Jane", records.get(1).getValueString("firstName")); - assertNotNull(records.get(0).getValue("id")); - assertNotNull(records.get(1).getValue("id")); - } - -} \ No newline at end of file From 1c2638a5c48de401419e68315040f34d7cb0fc51 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 25 Nov 2024 11:27:44 -0600 Subject: [PATCH 24/84] CE-1955 - Boosting test-coverage during bulk-load rollout --- .../core/instances/QInstanceEnricher.java | 4 +- .../bulk/insert/BulkInsertExtractStep.java | 134 ++++++++++-------- .../bulk/insert/BulkInsertV2ExtractStep.java | 127 ----------------- .../streamedwithfrontend/NoopLoadStep.java | 3 + .../insert/BulkInsertFullProcessTest.java | 3 +- .../NoopLoadStepTest.java | 50 +++++++ .../core/utils/aggregates/AggregatesTest.java | 51 +++++++ 7 files changed, 186 insertions(+), 186 deletions(-) delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 9a48204a..4e8d0237 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -73,13 +73,13 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.Bulk import com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete.BulkDeleteTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEditTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertTransformStep; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertV2ExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; @@ -819,7 +819,7 @@ public class QInstanceEnricher values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table.getName()); QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData( - BulkInsertV2ExtractStep.class, + BulkInsertExtractStep.class, BulkInsertTransformStep.class, BulkInsertLoadStep.class, values diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java index c29d1648..694ae332 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertExtractStep.java @@ -22,84 +22,106 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.InputStream; import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; -import com.kingsrook.qqq.backend.core.context.QContext; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; -import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.shared.mapping.QKeyBasedFieldMapping; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; -import com.kingsrook.qqq.backend.core.state.AbstractStateKey; -import com.kingsrook.qqq.backend.core.state.TempFileStateProvider; /******************************************************************************* ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. *******************************************************************************/ public class BulkInsertExtractStep extends AbstractExtractStep { + + /*************************************************************************** + ** + ***************************************************************************/ @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - AbstractStateKey stateKey = (AbstractStateKey) runBackendStepInput.getValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME); - Optional optionalUploadedFile = TempFileStateProvider.getInstance().get(QUploadedFile.class, stateKey); - if(optionalUploadedFile.isEmpty()) + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) { - throw (new QException("Could not find uploaded file")); - } + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - byte[] bytes = optionalUploadedFile.get().getBytes(); - String fileName = optionalUploadedFile.get().getFilename(); + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; - ///////////////////////////////////////////////////// - // let the user specify field labels instead names // - ///////////////////////////////////////////////////// - QTableMetaData table = runBackendStepInput.getTable(); - String tableName = runBackendStepInput.getTableName(); - QKeyBasedFieldMapping mapping = new QKeyBasedFieldMapping(); - for(Map.Entry entry : table.getFields().entrySet()) - { - mapping.addMapping(entry.getKey(), entry.getValue().getLabel()); - } + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); - ////////////////////////////////////////////////////////////////////////// - // get the non-editable fields - they'll be blanked out in a customizer // - ////////////////////////////////////////////////////////////////////////// - List nonEditableFields = table.getFields().values().stream() - .filter(f -> !f.getIsEditable()) - .toList(); - - if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) - { - new CsvToQRecordAdapter().buildRecordsFromCsv(new CsvToQRecordAdapter.InputWrapper() - .withRecordPipe(getRecordPipe()) - .withLimit(getLimit()) - .withCsv(new String(bytes)) - .withDoCorrectValueTypes(true) - .withTable(QContext.getQInstance().getTable(tableName)) - .withMapping(mapping) - .withRecordCustomizer((record) -> + if(page.size() > remainingLimit) { - //////////////////////////////////////////// - // remove values from non-editable fields // - //////////////////////////////////////////// - for(QFieldMetaData nonEditableField : nonEditableFields) - { - record.setValue(nonEditableField.getName(), null); - } - })); + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } } - else + catch(QException qe) { - throw (new QUserFacingException("Unsupported file type.")); + throw qe; } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java deleted file mode 100644 index 2da27e7d..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; - - -import java.io.InputStream; -import java.util.List; -import java.util.Objects; -import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; -import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; -import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; -import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; - - -/******************************************************************************* - ** Extract step for generic table bulk-insert ETL process - ** - ** This step does a little bit of transforming, actually - taking rows from - ** an uploaded file, and potentially merging them (for child-table use-cases) - ** and applying the "Mapping" - to put fully built records into the pipe for the - ** Transform step. - *******************************************************************************/ -public class BulkInsertV2ExtractStep extends AbstractExtractStep -{ - - /*************************************************************************** - ** - ***************************************************************************/ - @Override - public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException - { - int rowsAdded = 0; - int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); - - StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); - BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); - RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); - - try - ( - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // - /////////////////////////////////////////////////////////////////////////////////////////////////////////// - InputStream inputStream = new StorageAction().getInputStream(storageInput); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); - ) - { - /////////////////////////////////////////////////////////// - // read the header row (if this file & mapping uses one) // - /////////////////////////////////////////////////////////// - BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - - //////////////////////////////////////////////////////////////////////////////////////////////////////// - // while there are more rows in the file - and we're under the limit - get more records form the file // - //////////////////////////////////////////////////////////////////////////////////////////////////////// - while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) - { - int remainingLimit = originalLimit - rowsAdded; - - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // - //////////////////////////////////////////////////////////////////////////////////////////////////////////// - int pageLimit = Math.min(remainingLimit, getMaxPageSize()); - List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); - - if(page.size() > remainingLimit) - { - ///////////////////////////////////////////////////////////// - // in case we got back more than we asked for, sub-list it // - ///////////////////////////////////////////////////////////// - page = page.subList(0, remainingLimit); - } - - ///////////////////////////////////////////// - // send this page of records into the pipe // - ///////////////////////////////////////////// - getRecordPipe().addRecords(page); - rowsAdded += page.size(); - } - } - catch(QException qe) - { - throw qe; - } - catch(Exception e) - { - throw new QException("Unhandled error in bulk insert extract step", e); - } - - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private int getMaxPageSize() - { - return (1000); - } -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java index 0fcd5abf..f675f128 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStep.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; @@ -33,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp *******************************************************************************/ public class NoopLoadStep extends AbstractLoadStep { + private static final QLogger LOG = QLogger.getLogger(NoopLoadStep.class); /******************************************************************************* @@ -45,6 +47,7 @@ public class NoopLoadStep extends AbstractLoadStep /////////// // noop. // /////////// + LOG.trace("noop"); } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index 7037662a..b819760e 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -188,7 +188,7 @@ class BulkInsertFullProcessTest extends BaseTest new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3), new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4), new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail), - new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true), + new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true).withValueMappings(Map.of("Illinois", 1)), new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8) ))); }; @@ -204,6 +204,7 @@ class BulkInsertFullProcessTest extends BaseTest assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName()); assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues")); assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping")); + assertEquals(Map.of(1, "IL"), runProcessOutput.getValue("mappedValueLabels")); assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex")); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java new file mode 100644 index 00000000..59a25481 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/NoopLoadStepTest.java @@ -0,0 +1,50 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** Unit test for NoopLoadStep + *******************************************************************************/ +class NoopLoadStepTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + ////////////////////////////////////// + // sorry, just here for coverage... // + ////////////////////////////////////// + new NoopLoadStep().runOnePage(new RunBackendStepInput(), new RunBackendStepOutput()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java index 4dff56c6..daddaaf2 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/aggregates/AggregatesTest.java @@ -91,6 +91,57 @@ class AggregatesTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testLong() + { + LongAggregates aggregates = new LongAggregates(); + + assertEquals(0, aggregates.getCount()); + assertNull(aggregates.getMin()); + assertNull(aggregates.getMax()); + assertNull(aggregates.getSum()); + assertNull(aggregates.getAverage()); + + aggregates.add(5L); + assertEquals(1, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(5, aggregates.getMax()); + assertEquals(5, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(10L); + assertEquals(2, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(10, aggregates.getMax()); + assertEquals(15, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("7.5"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(15L); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + aggregates.add(null); + assertEquals(3, aggregates.getCount()); + assertEquals(5, aggregates.getMin()); + assertEquals(15, aggregates.getMax()); + assertEquals(30, aggregates.getSum()); + assertThat(aggregates.getAverage()).isCloseTo(new BigDecimal("10"), Offset.offset(BigDecimal.ZERO)); + + assertEquals(new BigDecimal("750"), aggregates.getProduct()); + assertEquals(new BigDecimal("25.0000"), aggregates.getVariance()); + assertEquals(new BigDecimal("5.0000"), aggregates.getStandardDeviation()); + assertThat(aggregates.getVarP()).isCloseTo(new BigDecimal("16.6667"), Offset.offset(new BigDecimal(".0001"))); + assertThat(aggregates.getStdDevP()).isCloseTo(new BigDecimal("4.0824"), Offset.offset(new BigDecimal(".0001"))); + } + + + /******************************************************************************* ** *******************************************************************************/ From 17fc976877bdc1350c1c8425b5b962a8e532b21f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 11:46:24 -0600 Subject: [PATCH 25/84] CE-1955 - Add rowNo to BulkLoadFileRow, set by FileToRowsInterface objects --- .../AbstractIteratorBasedFileToRows.java | 14 +++++++ .../insert/filehandling/CsvFileToRows.java | 2 +- .../filehandling/FileToRowsInterface.java | 6 +++ .../insert/filehandling/XlsxFileToRows.java | 2 +- .../bulk/insert/model/BulkLoadFileRow.java | 42 +++++++++++++++++-- .../filehandling/CsvFileToRowsTest.java | 4 +- .../insert/filehandling/TestFileToRows.java | 2 +- .../filehandling/XlsxFileToRowsTest.java | 4 +- 8 files changed, 65 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java index be80c82a..88437f1b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -36,6 +36,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn private boolean useLast = false; private BulkLoadFileRow last; + int rowNo = 0; /*************************************************************************** @@ -65,6 +66,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn @Override public BulkLoadFileRow next() { + rowNo++; if(iterator == null) { throw new IllegalStateException("Object was not init'ed"); @@ -99,6 +101,7 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn @Override public void unNext() { + rowNo--; useLast = true; } @@ -122,4 +125,15 @@ public abstract class AbstractIteratorBasedFileToRows implements FileToRowsIn this.iterator = iterator; } + + + /******************************************************************************* + ** Getter for rowNo + ** + *******************************************************************************/ + @Override + public int getRowNo() + { + return rowNo; + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java index 7a58ddef..332c8722 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -92,7 +92,7 @@ public class CsvFileToRows extends AbstractIteratorBasedFileToRows im values[i++] = s; } - return (new BulkLoadFileRow(values)); + return (new BulkLoadFileRow(values, getRowNo())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java index 9d02ee0f..9ab6cce3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -66,6 +66,12 @@ public interface FileToRowsInterface extends AutoCloseable, Iterator Date: Wed, 27 Nov 2024 12:13:15 -0600 Subject: [PATCH 26/84] CE-1955 - Put rows & rowNos in backend details during bulk-load. assert about those. also add tests (and fixes to mapping) for no-header use-cases --- .../insert/mapping/BulkLoadRecordUtils.java | 112 +++++++++++ .../bulk/insert/mapping/FlatRowsToRecord.java | 1 + .../bulk/insert/mapping/TallRowsToRecord.java | 25 ++- ...licitFieldNameSuffixIndexBasedMapping.java | 4 +- .../bulk/insert/model/BulkInsertMapping.java | 34 +++- .../insert/mapping/FlatRowsToRecordTest.java | 65 +++++++ .../insert/mapping/TallRowsToRecordTest.java | 174 ++++++++++++++++++ ...tFieldNameSuffixIndexBasedMappingTest.java | 60 ++++++ 8 files changed, 464 insertions(+), 11 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java new file mode 100644 index 00000000..0c700fe0 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadRecordUtils.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; + + +/******************************************************************************* + ** Utility methods for working with records in a bulk-load. + ** + ** Originally added for working with backendDetails around the source rows. + *******************************************************************************/ +public class BulkLoadRecordUtils +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, BulkLoadFileRow fileRow) + { + return (addBackendDetailsAboutFileRows(record, new ArrayList<>(List.of(fileRow)))); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public static QRecord addBackendDetailsAboutFileRows(QRecord record, ArrayList fileRows) + { + if(CollectionUtils.nullSafeHasContents(fileRows)) + { + Integer firstRowNo = fileRows.get(0).getRowNo(); + Integer lastRowNo = fileRows.get(fileRows.size() - 1).getRowNo(); + + if(Objects.equals(firstRowNo, lastRowNo)) + { + record.addBackendDetail("rowNos", "Row " + firstRowNo); + } + else + { + record.addBackendDetail("rowNos", "Rows " + firstRowNo + "-" + lastRowNo); + } + } + else + { + record.addBackendDetail("rowNos", "Rows ?"); + } + + record.addBackendDetail("fileRows", fileRows); + return (record); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static String getRowNosString(QRecord record) + { + return (record.getBackendDetailString("rowNos")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @SuppressWarnings("unchecked") + public static ArrayList getFileRows(QRecord record) + { + return (ArrayList) record.getBackendDetail("fileRows"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static List getFileRowNos(QRecord record) + { + return (getFileRows(record).stream().map(row -> row.getRowNo()).toList()); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index d1a5d4c0..243496ef 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -60,6 +60,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface { BulkLoadFileRow row = fileToRowsInterface.next(); QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 4587336c..8e0443d3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -67,8 +68,8 @@ public class TallRowsToRecord implements RowsToRecordInterface List rs = new ArrayList<>(); - List rowsForCurrentRecord = new ArrayList<>(); - List recordGroupByValues = null; + ArrayList rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; String associationNameChain = ""; @@ -82,6 +83,9 @@ public class TallRowsToRecord implements RowsToRecordInterface groupByIndexes = groupByAllIndexesFromTable(mapping, table, headerRow, null); } + //////////////////////// + // this is suspect... // + //////////////////////// List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { @@ -108,18 +112,19 @@ public class TallRowsToRecord implements RowsToRecordInterface ////////////////////////////////////////////////////////////// // not first, and not a match, so we can finish this record // ////////////////////////////////////////////////////////////// - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); //////////////////////////////////////// // reset these record-specific values // //////////////////////////////////////// rowsForCurrentRecord = new ArrayList<>(); recordGroupByValues = null; - - ////////////////////////////////////////////////////////////////////////////////////////////////////// - // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // - ////////////////////////////////////////////////////////////////////////////////////////////////////// - fileToRowsInterface.unNext(); } } @@ -129,7 +134,8 @@ public class TallRowsToRecord implements RowsToRecordInterface ///////////////////////////////////////////////////////////////////////////////////////////////// if(!rowsForCurrentRecord.isEmpty()) { - rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + QRecord record = makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord); + rs.add(record); } ValueMapper.valueMapping(rs, mapping, table); @@ -156,6 +162,7 @@ public class TallRowsToRecord implements RowsToRecordInterface private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException { QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken>() {})); Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index 9c85647e..e79dd2c2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -90,7 +90,9 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem ////////////////////////////////////////////////////// // start by building the record with its own fields // ////////////////////////////////////////////////////// - QRecord record = new QRecord(); + QRecord record = new QRecord(); + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index 86c19775..d0164b3f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -157,7 +157,7 @@ public class BulkInsertMapping implements Serializable } else if(fieldNameToIndexMap != null) { - return (fieldNameToIndexMap); + return (getFieldIndexesForNoHeaderUseCase(table, associationNameChain, wideAssociationIndexes)); } throw (new QException("Mapping was not properly configured.")); @@ -165,6 +165,38 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + private Map getFieldIndexesForNoHeaderUseCase(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Integer index = fieldNameToIndexMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(index != null) + { + rs.put(field.getName(), index); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 7124e946..fd1f70c4 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -76,21 +76,86 @@ class FlatRowsToRecordTest extends BaseTest .withHasHeaderRow(true); List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(1, records.size()); assertEquals(List.of("Homer"), getValues(records, "firstName")); assertEquals(List.of("Simpson"), getValues(records, "lastName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 3", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 4", records.get(1).getBackendDetail("rowNos")); records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); assertEquals(List.of("Ned"), getValues(records, "firstName")); assertEquals(List.of(2), getValues(records, "noOfShoes")); assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToColumnIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + // 0, 1, 2, 3, 4 + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2, + "cost", 4 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, null, mapping, 1); + assertEquals(1, records.size()); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 1", records.get(0).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, 2); + assertEquals(2, records.size()); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(null, new BigDecimal("99.95")), getValues(records, "cost")); + assertEquals(1, ((List) records.get(0).getBackendDetail("fileRows")).size()); + assertEquals("Row 2", records.get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", records.get(1).getBackendDetail("rowNos")); + + records = rowsToRecord.nextPage(fileToRows, null, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + assertEquals("Row 4", records.get(0).getBackendDetail("rowNos")); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 1a0ef2ab..126f500b 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -89,6 +89,11 @@ class TallRowsToRecordTest extends BaseTest assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 2-4", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 4", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); @@ -96,6 +101,68 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4 + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = null; + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku", 3, + "orderLine.quantity", 4 + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 1-3", order.getBackendDetail("rowNos")); + assertEquals(3, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Rows 4-5", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } @@ -338,6 +405,113 @@ class TallRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSingleLine() throws QException + { + Integer defaultStoreId = 101; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", defaultStoreId + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(1, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(defaultStoreId, order.getValue("storeId")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPagination() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + 2, Ned, Flanders, BIBLE, 7 + 2, Ned, Flanders, LAWNMOWER, 1 + 3, Bart, Simpson, SKATEBOARD,1 + 3, Bart, Simpson, SLINGSHOT, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 3-4", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(1, records.size()); + order = records.get(0); + assertEquals(3, order.getValueInteger("orderNo")); + assertEquals("Bart", order.getValueString("shipToName")); + assertEquals(List.of("SKATEBOARD", "SLINGSHOT"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals("Rows 5-6", order.getBackendDetail("rowNos")); + assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 6e59c527..38488683 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -86,12 +86,72 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); order = records.get(1); assertEquals(2, order.getValueInteger("orderNo")); assertEquals("Ned", order.getValueString("shipToName")); assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutHeader() throws QException + { + // 0, 1, 2, 3, 4, 5, 6, 7, 8 + String csv = """ + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = null; + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "orderNo", 0, + "shipToName", 1, + "orderLine.sku,0", 3, + "orderLine.quantity,0", 4, + "orderLine.sku,1", 5, + "orderLine.quantity,1", 6, + "orderLine.sku,2", 7, + "orderLine.quantity,2", 8 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(1).getBackendDetail("rowNos")); + assertEquals("Row 1", order.getAssociatedRecords().get("orderLine").get(2).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(7, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); } From c88fd5b7d41389c293a45aef749f125dae974ab2 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 12:35:54 -0600 Subject: [PATCH 27/84] CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors --- .../model/metadata/fields/QFieldType.java | 11 + .../bulk/insert/BulkInsertTransformStep.java | 169 +++++++++++++- .../mapping/BulkLoadValueTypeError.java | 76 +++++++ .../bulk/insert/mapping/ValueMapper.java | 6 +- ...ProcessSummaryWarningsAndErrorsRollup.java | 9 + .../processes/ProcessSummaryAssert.java | 208 ++++++++++++++++++ .../ProcessSummaryLineInterfaceAssert.java | 189 ++++++++++++++++ .../insert/BulkInsertTransformStepTest.java | 190 +++++++++++++--- .../qqq/backend/core/utils/TestUtils.java | 3 +- 9 files changed, 823 insertions(+), 38 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java index b0e8f8e7..a8313116 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java @@ -27,6 +27,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -100,6 +101,16 @@ public enum QFieldType + /*************************************************************************** + ** + ***************************************************************************/ + public String getMixedCaseLabel() + { + return StringUtils.allCapsToMixedCase(name()); + } + + + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 09d9ca8d..f2d87767 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; @@ -47,16 +48,25 @@ import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutp import com.kingsrook.qqq.backend.core.model.actions.processes.Status; import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.processes.implementations.general.ProcessSummaryWarningsAndErrorsRollup; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; /******************************************************************************* @@ -66,7 +76,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { private ProcessSummaryLine okSummary = new ProcessSummaryLine(Status.OK); - private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted"); + private ProcessSummaryWarningsAndErrorsRollup processSummaryWarningsAndErrorsRollup = ProcessSummaryWarningsAndErrorsRollup.build("inserted") + .withDoReplaceSingletonCountLinesWithSuffixOnly(false); + + private ListingHash errorToExampleRowValueMap = new ListingHash<>(); + private ListingHash errorToExampleRowsMap = new ListingHash<>(); private Map ukErrorSummaries = new HashMap<>(); private Map associationsToInsertSummaries = new HashMap<>(); @@ -77,6 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; + private final int EXAMPLE_ROW_LIMIT = 10; /******************************************************************************* @@ -118,6 +133,44 @@ public class BulkInsertTransformStep extends AbstractTransformStep // make sure that if a saved profile was selected on a review screen, that the result screen knows about it. // /////////////////////////////////////////////////////////////////////////////////////////////////////////////// BulkInsertStepUtils.handleSavedBulkLoadProfileIdValue(runBackendStepInput, runBackendStepOutput); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // set up the validationReview widget to render preview records using the table layout, and including the associations // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); + + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + if(bulkInsertMapping != null) + { + ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); + ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); + ArrayList previewRecordAssociationNames = new ArrayList<>(); + + for(String mappedAssociation : bulkInsertMapping.getMappedAssociations()) + { + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst(); + if(association.isPresent()) + { + for(QFieldSection section : table.getSections()) + { + QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(section.getWidgetName()); + if(widget != null && WidgetType.CHILD_RECORD_LIST.getType().equals(widget.getType())) + { + Serializable widgetJoinName = widget.getDefaultValues().get("joinName"); + if(Objects.equals(widgetJoinName, association.get().getJoinName())) + { + previewRecordAssociatedTableNames.add(association.get().getAssociatedTableName()); + previewRecordAssociatedWidgetNames.add(widget.getName()); + previewRecordAssociationNames.add(association.get().getName()); + } + } + } + } + } + runBackendStepOutput.addValue("previewRecordAssociatedTableNames", previewRecordAssociatedTableNames); + runBackendStepOutput.addValue("previewRecordAssociatedWidgetNames", previewRecordAssociatedWidgetNames); + runBackendStepOutput.addValue("previewRecordAssociationNames", previewRecordAssociationNames); + } } @@ -131,7 +184,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep int recordsInThisPage = runBackendStepInput.getRecords().size(); QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); - // split the records w/o UK errors into those w/ e + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // split the records into 2 lists: those w/ errors (e.g., from the bulk-load mapping), and those that are okay // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// List recordsWithoutAnyErrors = new ArrayList<>(); List recordsWithSomeErrors = new ArrayList<>(); for(QRecord record : runBackendStepInput.getRecords()) @@ -153,16 +208,26 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QRecord record : recordsWithSomeErrors) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + if(error instanceof BulkLoadValueTypeError blvte) + { + processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(blvte, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + } + } } } if(recordsWithoutAnyErrors.isEmpty()) { - //////////////////////////////////////////////////////////////////////////////// - // skip th rest of this method if there aren't any records w/o errors in them // - //////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////// + // skip the rest of this method if there aren't any records w/o errors in them // + ///////////////////////////////////////////////////////////////////////////////// this.rowsProcessed += recordsInThisPage; } @@ -248,8 +313,11 @@ public class BulkInsertTransformStep extends AbstractTransformStep { if(CollectionUtils.nullSafeHasContents(record.getErrors())) { - String message = record.getErrors().get(0).getMessage(); - processSummaryWarningsAndErrorsRollup.addError(message, null); + for(QErrorMessage error : record.getErrors()) + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) { @@ -277,6 +345,37 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowValues.size() < EXAMPLE_ROW_LIMIT) + { + rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addToErrorToExampleRowMap(String message, QRecord record) + { + List rowNos = errorToExampleRowsMap.computeIfAbsent(message, k -> new ArrayList<>()); + + if(rowNos.size() < EXAMPLE_ROW_LIMIT) + { + rowNos.add(BulkLoadRecordUtils.getRowNosString(record)); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,9 +510,61 @@ public class BulkInsertTransformStep extends AbstractTransformStep ukErrorSummary.addSelfToListIfAnyCount(rs); } + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // for process summary lines that exist in the error-to-example-row-value map, add those example values to the lines. // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(Map.Entry entry : processSummaryWarningsAndErrorsRollup.getErrorSummaries().entrySet()) + { + String message = entry.getKey(); + if(errorToExampleRowValueMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); + line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); + } + else if(errorToExampleRowsMap.containsKey(message)) + { + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); + line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); + } + } + processSummaryWarningsAndErrorsRollup.addToList(rs); return (rs); } + + + /*************************************************************************** + ** + ***************************************************************************/ + private record RowValue(String row, String value) + { + + /*************************************************************************** + ** + ***************************************************************************/ + public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + { + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + return row + " [" + value + "]"; + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java new file mode 100644 index 00000000..a7e6f371 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -0,0 +1,76 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadValueTypeError extends BadInputStatusMessage +{ + private final String fieldLabel; + private final String fieldName; + private final Serializable value; + private final QFieldType type; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); + this.fieldName = fieldName; + this.value = value; + this.type = type; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java index fdf1b6bd..44a37f6a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -34,7 +34,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -70,6 +69,9 @@ public class ValueMapper return; } + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); for(QRecord record : records) { @@ -102,7 +104,7 @@ public class ValueMapper } catch(Exception e) { - record.addError(new BadInputStatusMessage("Value [" + value + "] for field [" + field.getLabel() + "] could not be converted to type [" + type + "]")); + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java index 606a026b..6296f31c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/general/ProcessSummaryWarningsAndErrorsRollup.java @@ -477,4 +477,13 @@ public class ProcessSummaryWarningsAndErrorsRollup } + + /******************************************************************************* + ** Getter for errorSummaries + ** + *******************************************************************************/ + public Map getErrorSummaries() + { + return errorSummaries; + } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java new file mode 100644 index 00000000..a0bf6a36 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryAssert.java @@ -0,0 +1,208 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummary - that is - a list of ProcessSummaryLineInterface's + *******************************************************************************/ +public class ProcessSummaryAssert extends AbstractAssert> +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryAssert(List actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunProcessOutput runProcessOutput) + { + List processResults = (List) runProcessOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runProcessOutput.getValue("validationSummary"); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @SuppressWarnings("unchecked") + public static ProcessSummaryAssert assertThat(RunBackendStepOutput runBackendStepOutput) + { + List processResults = (List) runBackendStepOutput.getValue("processResults"); + if(processResults == null) + { + processResults = (List) runBackendStepOutput.getValue("validationSummary"); + } + + if(processResults == null) + { + fail("Could not find process results in backend step output."); + } + + return (new ProcessSummaryAssert(processResults, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryAssert assertThat(List actual) + { + return (new ProcessSummaryAssert(actual, ProcessSummaryAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasSize(int expectedSize) + { + Assertions.assertThat(actual).hasSize(expectedSize); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageMatching(String regExp) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().matches(regExp)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message matching [" + regExp + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithMessageContaining(String substr) + { + List foundMessages = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(processSummaryLineInterface.getMessage() == null) + { + processSummaryLineInterface.prepareForFrontend(false); + } + + if(processSummaryLineInterface.getMessage() != null && processSummaryLineInterface.getMessage().contains(substr)) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundMessages.add(processSummaryLineInterface.getMessage()); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with message containing [" + substr + "].\nFound messages were:\n" + StringUtils.join("\n", foundMessages)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasLineWithStatus(Status status) + { + List foundStatuses = new ArrayList<>(); + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + return (new ProcessSummaryLineInterfaceAssert(processSummaryLineInterface, ProcessSummaryLineInterfaceAssert.class)); + } + else + { + foundStatuses.add(String.valueOf(processSummaryLineInterface.getStatus())); + } + } + + failWithMessage("Failed to find a ProcessSummaryLine with status [" + status + "].\nFound statuses were:\n" + StringUtils.join("\n", foundStatuses)); + return (null); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryAssert hasNoLineWithStatus(Status status) + { + for(ProcessSummaryLineInterface processSummaryLineInterface : actual) + { + if(status.equals(processSummaryLineInterface.getStatus())) + { + failWithMessage("Found a ProcessSummaryLine with status [" + status + "], which was not supposed to happen."); + return (null); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java new file mode 100644 index 00000000..861e21a7 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -0,0 +1,189 @@ + +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.actions.processes; + + +import org.assertj.core.api.AbstractAssert; +import org.assertj.core.api.Assertions; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** AssertJ assert class for ProcessSummaryLine. + *******************************************************************************/ +public class ProcessSummaryLineInterfaceAssert extends AbstractAssert +{ + + /******************************************************************************* + ** + *******************************************************************************/ + protected ProcessSummaryLineInterfaceAssert(ProcessSummaryLineInterface actual, Class selfType) + { + super(actual, selfType); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static ProcessSummaryLineInterfaceAssert assertThat(ProcessSummaryLineInterface actual) + { + return (new ProcessSummaryLineInterfaceAssert(actual, ProcessSummaryLineInterfaceAssert.class)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasCount(Integer count) + { + if(actual instanceof ProcessSummaryLine psl) + { + assertEquals(count, psl.getCount(), "Expected count in process summary line"); + } + else + { + failWithMessage("ProcessSummaryLineInterface is not of concrete type ProcessSummaryLine (is: " + actual.getClass().getSimpleName() + ")"); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasStatus(Status status) + { + assertEquals(status, actual.getStatus(), "Expected status in process summary line"); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).matches(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).contains(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageMatching(String regExp) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotMatch(regExp); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveMessageContaining(String substring) + { + if(actual.getMessage() == null) + { + actual.prepareForFrontend(false); + } + + Assertions.assertThat(actual.getMessage()).doesNotContain(substring); + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert hasAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + Assertions.assertThat(psl.getBulletsOfText()) + .isNotNull() + .anyMatch(s -> s.contains(substring)); + } + else + { + Assertions.fail("Process Summary Line was not the expected type."); + } + + return (this); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public ProcessSummaryLineInterfaceAssert doesNotHaveAnyBulletsOfTextContaining(String substring) + { + if(actual instanceof ProcessSummaryLine psl) + { + if(psl.getBulletsOfText() != null) + { + Assertions.assertThat(psl.getBulletsOfText()) + .noneMatch(s -> s.contains(substring)); + } + } + + return (this); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index b87b6799..5a8ae6d3 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -22,10 +22,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryAssert; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -38,8 +42,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,9 +95,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -102,13 +110,13 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); @@ -171,9 +179,9 @@ class BulkInsertTransformStepTest extends BaseTest // insert some records that will cause some UK violations // //////////////////////////////////////////////////////////// TestUtils.insertRecords(table, List.of( - newQRecord("uuid-A", "SKU-1", 1), - newQRecord("uuid-B", "SKU-2", 1), - newQRecord("uuid-C", "SKU-2", 2) + newUkTestQRecord("uuid-A", "SKU-1", 1), + newUkTestQRecord("uuid-B", "SKU-2", 1), + newUkTestQRecord("uuid-C", "SKU-2", 2) )); /////////////////////////////////////////// @@ -186,20 +194,20 @@ class BulkInsertTransformStepTest extends BaseTest input.setTableName(TABLE_NAME); input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); input.setRecords(List.of( - newQRecord("uuid-1", "SKU-A", 1), // OK. - newQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set - newQRecord("uuid-2", "SKU-C", 1), // OK. - newQRecord("uuid-3", "SKU-C", 2), // OK. - newQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set - newQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records - newQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records + newUkTestQRecord("uuid-1", "SKU-A", 1), // OK. + newUkTestQRecord("uuid-1", "SKU-B", 1), // violate uuid UK in this set + newUkTestQRecord("uuid-2", "SKU-C", 1), // OK. + newUkTestQRecord("uuid-3", "SKU-C", 2), // OK. + newUkTestQRecord("uuid-4", "SKU-C", 1), // violate sku/storeId UK in this set + newUkTestQRecord("uuid-A", "SKU-X", 1), // violate uuid UK from pre-existing records + newUkTestQRecord("uuid-D", "SKU-2", 1) // violate sku/storeId UK from pre-existing records )); bulkInsertTransformStep.preRun(input, output); bulkInsertTransformStep.runOnePage(input, output); - /////////////////////////////////////////////////////// - // assert that all records pass. - /////////////////////////////////////////////////////// + /////////////////////////////////// + // assert that all records pass. // + /////////////////////////////////// assertEquals(7, output.getRecords().size()); } @@ -211,8 +219,8 @@ class BulkInsertTransformStepTest extends BaseTest private boolean recordEquals(QRecord record, String uuid, String sku, Integer storeId) { return (record.getValue("uuid").equals(uuid) - && record.getValue("sku").equals(sku) - && record.getValue("storeId").equals(storeId)); + && record.getValue("sku").equals(sku) + && record.getValue("storeId").equals(storeId)); } @@ -220,7 +228,7 @@ class BulkInsertTransformStepTest extends BaseTest /******************************************************************************* ** *******************************************************************************/ - private QRecord newQRecord(String uuid, String sku, int storeId) + private QRecord newUkTestQRecord(String uuid, String sku, int storeId) { return new QRecord() .withValue("uuid", uuid) @@ -229,4 +237,134 @@ class BulkInsertTransformStepTest extends BaseTest .withValue("name", "Some Item"); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValueMappingTypeErrors() throws QException + { + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 1)) + .withError(new BulkLoadValueTypeError("storeId", "A", QFieldType.INTEGER, "Store")) + .withError(new BulkLoadValueTypeError("orderDate", "47", QFieldType.DATE, "Order Date")), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 2)) + .withError(new BulkLoadValueTypeError("storeId", "BCD", QFieldType.INTEGER, "Store")) + )); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // add 102 records with an error in the total field - which is more than the number of examples that should be given // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i)) + .withError(new BulkLoadValueTypeError("total", "three-fifty-" + i, QFieldType.DECIMAL, "Total"))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Store] to type [Integer]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [A]") + .hasAnyBulletsOfTextContaining("Row 2 [BCD]") + .hasStatus(Status.ERROR) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Order Date] to type [Date]") + .hasMessageContaining("Values:") + .doesNotHaveMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 1 [47]") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Cannot convert value for field [Total] to type [Decimal]") + .hasMessageContaining("Example Values:") + .hasAnyBulletsOfTextContaining("Row 3 [three-fifty-0]") + .hasAnyBulletsOfTextContaining("Row 4 [three-fifty-1]") + .hasAnyBulletsOfTextContaining("Row 5 [three-fifty-2]") + .doesNotHaveAnyBulletsOfTextContaining("three-fifty-101") + .hasStatus(Status.ERROR) + .hasCount(102); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testRollupOfValidationErrors() throws QException + { + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + Serializable[] emptyValues = new Serializable[0]; + + String tooLong = ".".repeat(201); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", tooLong), new BulkLoadFileRow(emptyValues, 1)), + BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord().withValue("shipToName", "OK").withValue("storeId", 1), new BulkLoadFileRow(emptyValues, 2)) + )); + + ///////////////////////////////////////////////////////////////////// + // add 102 records with no security key - which should be an error // + ///////////////////////////////////////////////////////////////////// + for(int i = 0; i < 102; i++) + { + input.getRecords().add(BulkLoadRecordUtils.addBackendDetailsAboutFileRows(new QRecord(), new BulkLoadFileRow(emptyValues, 3 + i))); + } + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("value for Ship To Name is too long") + .hasMessageContaining("Records:") + .doesNotHaveMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("without a value in the field: Store Id") + .hasMessageContaining("Example Records:") + .hasAnyBulletsOfTextContaining("Row 1") + .hasAnyBulletsOfTextContaining("Row 3") + .hasAnyBulletsOfTextContaining("Row 4") + .doesNotHaveAnyBulletsOfTextContaining("Row 101") + .hasStatus(Status.ERROR) + .hasCount(103); // the 102, plus row 1. + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + } \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java index 89adac13..54502074 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java @@ -70,6 +70,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; @@ -662,7 +663,7 @@ public class TestUtils .withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withIsEditable(false)) .withField(new QFieldMetaData("orderNo", QFieldType.STRING)) - .withField(new QFieldMetaData("shipToName", QFieldType.STRING)) + .withField(new QFieldMetaData("shipToName", QFieldType.STRING).withMaxLength(200).withBehavior(ValueTooLongBehavior.ERROR)) .withField(new QFieldMetaData("orderDate", QFieldType.DATE)) .withField(new QFieldMetaData("storeId", QFieldType.INTEGER)) .withField(new QFieldMetaData("total", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY).withFieldSecurityLock(new FieldSecurityLock() From 9213b8987b9213a63910b95789d5df564ee62072 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 12:36:35 -0600 Subject: [PATCH 28/84] CE-1955 - Summarize with some examples (including rows nos) for value mapping and other validation errors --- .../actions/processes/ProcessSummaryLine.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java index 98697f80..d4974a07 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLine.java @@ -53,6 +53,7 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface ////////////////////////////////////////////////////////////////////////// private ArrayList primaryKeys; + private ArrayList bulletsOfText; /******************************************************************************* @@ -497,4 +498,35 @@ public class ProcessSummaryLine implements ProcessSummaryLineInterface return (this); } + + /******************************************************************************* + ** Getter for bulletsOfText + *******************************************************************************/ + public ArrayList getBulletsOfText() + { + return (this.bulletsOfText); + } + + + + /******************************************************************************* + ** Setter for bulletsOfText + *******************************************************************************/ + public void setBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + } + + + + /******************************************************************************* + ** Fluent setter for bulletsOfText + *******************************************************************************/ + public ProcessSummaryLine withBulletsOfText(ArrayList bulletsOfText) + { + this.bulletsOfText = bulletsOfText; + return (this); + } + + } From 8c6b4e686398d48c08ad9c305227100fe124e634 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 14:59:57 -0600 Subject: [PATCH 29/84] CE-1955 - Add back to processes --- docs/metaData/Processes.adoc | 65 +++++++- .../actions/processes/RunProcessAction.java | 58 ++++++- .../model/actions/processes/ProcessState.java | 66 ++++++++ .../processes/RunBackendStepInput.java | 11 ++ .../actions/processes/RunProcessInput.java | 32 ++++ .../processes/QFrontendStepMetaData.java | 32 ++++ .../processes/RunProcessActionTest.java | 154 ++++++++++++++++-- .../javalin/QJavalinProcessHandler.java | 44 ++++- .../executors/io/ProcessInitOrStepInput.java | 1 + ...cessInitOrStepOrStatusOutputInterface.java | 2 + 10 files changed, 437 insertions(+), 28 deletions(-) diff --git a/docs/metaData/Processes.adoc b/docs/metaData/Processes.adoc index 7985fb73..789bd6d6 100644 --- a/docs/metaData/Processes.adoc +++ b/docs/metaData/Processes.adoc @@ -38,6 +38,13 @@ See {link-permissionRules} for details. *** 1) by a single call to `.withStepList(List)`, which internally adds each step into the `steps` map. *** 2) by multiple calls to `.addStep(QStepMetaData)`, which adds a step to both the `stepList` and `steps` map. ** If a process also needs optional steps (for a <<_custom_process_flow>>), they should be added by a call to `.addOptionalStep(QStepMetaData)`, which only places them in the `steps` map. +* `stepFlow` - *enum, default LINEAR* - specifies the the flow-control logic between steps. Possible values are: +** `LINEAR` - steps are executed in-order, through the `stepList`. +A backend step _can_ customize the `nextStepName` or re-order the `stepList`, if needed. +In a frontend step, a user may be given the option to go _back_ to a previous step as well. +** `STATE_MACHINE` - steps are executed as a Fine State Machine, starting with the first step in `stepList`, +but then proceeding based on the `nextStepName` specified by the previous step. +Thus allowing much more flexible flows. * `schedule` - *<>* - set up the process to run automatically on the specified schedule. See below for details. * `minInputRecords` - *Integer* - #not used...# @@ -67,6 +74,11 @@ For processes with a user-interface, they must define one or more "screens" in t * `formFields` - *List of String* - list of field names used by the screen as form-inputs. * `viewFields` - *List of String* - list of field names used by the screen as visible outputs. * `recordListFields` - *List of String* - list of field names used by the screen in a record listing. +* `format` - *Optional String* - directive for a frontend to use specialized formatting for the display of the process. +** Consult frontend documentation for supported values and their meanings. +* `backStepName` - *Optional String* - For processes using `LINEAR` flow, if this value is given, +then the frontend should offer a control that the user can take (e.g., a button) to move back to an +earlier step in the process. ==== QFrontendComponentMetaData @@ -90,10 +102,13 @@ Expects a process value named `html`. Expects process values named `downloadFileName` and `serverFilePath`. ** `GOOGLE_DRIVE_SELECT_FOLDER` - Special form that presents a UI from Google Drive, where the user can select a folder (e.g., as a target for uploading files in a subsequent backend step). ** `BULK_EDIT_FORM` - For use by the standard QQQ Bulk Edit process. +** `BULK_LOAD_FILE_MAPPING_FORM`, `BULK_LOAD_VALUE_MAPPING_FORM`, or `BULK_LOAD_PROFILE_FORM` - For use by the standard QQQ Bulk Load process. ** `VALIDATION_REVIEW_SCREEN` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays a component prompting the user to run full validation or to skip it, or, if full validation has been ran, then showing the results of that validation. ** `PROCESS_SUMMARY_RESULTS` - For use by the QQQ Streamed ETL With Frontend process family of processes. Displays the summary results of running the process. +** `WIDGET` - Render a QQQ Widget. +Requires that `widgetName` be given as a value for the component. ** `RECORD_LIST` - _Deprecated. Showed a grid with a list of records as populated by the process._ * `values` - *Map of String → Serializable* - Key=value pairs, with different expectations based on the component's `type`. @@ -116,6 +131,27 @@ It can be used, however, for example, to cause a `defaultValue` to be applied to It can also be used to cause the process to throw an error, if a field is marked as `isRequired`, but a value is not present. ** `recordListMetaData` - *RecordListMetaData object* - _Not used at this time._ +==== QStateMachineStep + +Processes that use `flow = STATE_MACHINE` should use process steps of type `QStateMachineStep`. + +A common pattern seen in state-machine processes, is that they will present a frontend-step to a user, +then always run a given backend-step in response to that screen which the user submitted. +Inside that backend-step, custom application logic will determine the next state to go to, +which is typically another frontend-step (which would then submit data to its corresponding backend-step, +and continue the FSM). + +To help facilitate this pattern, factory methods exist on `QStateMachineStep`, +for constructing the commonly-expected types of state-machine steps: + +* `frontendThenBackend(name, frontendStep, backendStep)` - for the frontend-then-backend pattern described above. +* `backendOnly(name, backendStep)` - for a state that only has a backend step. +This might be useful as a “reset” step, to run before restarting a state-loop. +* `frontendOnly(name, frontendStep)` - for a state that only has a frontend step, +which would always be followed by another state, which must be specified as the `defaultNextStepName` +on the `QStateMachineStep`. + + ==== BasepullConfiguration A "Basepull" process is a common pattern where an application needs to perform some action on all new (or updated) records from a particular data source. @@ -218,12 +254,10 @@ But for some cases, doing page-level transactions can reduce long-transactions a * `withSchedule(QScheduleMetaData schedule)` - Add a <> to the process. [#_custom_process_flow] -==== Custom Process Flow -As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, a process -will execute each of its steps in-order, as defined in the `stepList` property. -However, a Backend Step can customize this flow #todo - write more clearly here... - -There are generally 2 method to call (in a `BackendStep`) to do a dynamic flow: +==== How to customize a Linear process flow +As referenced in the definition of the <<_QProcessMetaData_Properties,QProcessMetaData Properties>>, by default, +(with `flow = LINEAR`) a process will execute each of its steps in-order, as defined in the `stepList` property. +However, a Backend Step can customize this flow as follows: * `RunBackendStepOutput.setOverrideLastStepName(String stepName)` ** QQQ's `RunProcessAction` keeps track of which step it "last" ran, e.g., to tell it which one to run next. @@ -239,7 +273,7 @@ does need to be found in the new `stepNameList` - otherwise, the framework will for figuring out where to go next. [source,java] -.Example of a defining process that can use a flexible flow: +.Example of a defining process that can use a customized linear flow: ---- // for a case like this, it would be recommended to define all step names in constants: public final static String STEP_START = "start"; @@ -324,4 +358,21 @@ public static class StartStep implements BackendStep } ---- +[#_process_back] +==== How to allow a process to go back + +The simplest option to allow a process to present a "Back" button to users, +thus allowing them to move backward through a process +(e.g., from a review screen back to an earlier input screen), is to set the property `backStepName` +on a `QFrontendStepMetaData`. + +If the step that is executed after the user hits "Back" is a backend step, then within that +step, `runBackendStepInput.getIsStepBack()` will return `true` (but ONLY within that first step after +the user hits "Back"). It may be necessary within individual processes to be aware that the user +has chosen to go back, to reset certain values in the process's state. + +Alternatively, if a frontend step's "Back" behavior needs to be dynamic (e.g., sometimes not available, +or sometimes targeting different steps in the process), then in a backend step that runs before the +frontend step, a call to `runBackendStepOutput.getProcessState().setBackStepName()` can be made, +to customize the value which would otherwise come from the `QFrontendStepMetaData`. diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java index 481201e3..a595b4e9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java @@ -122,6 +122,12 @@ public class RunProcessAction UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS); ProcessState processState = primeProcessState(runProcessInput, stateKey, process); + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + // these should always be clear when we're starting a run - so make sure they haven't leaked from previous // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.clearNextStepName(); + processState.clearBackStepName(); + ///////////////////////////////////////////////////////// // if process is 'basepull' style, keep track of 'now' // ///////////////////////////////////////////////////////// @@ -188,14 +194,35 @@ public class RunProcessAction private void runLinearStepLoop(QProcessMetaData process, ProcessState processState, UUIDAndTypeStateKey stateKey, RunProcessInput runProcessInput, RunProcessOutput runProcessOutput) throws Exception { String lastStepName = runProcessInput.getStartAfterStep(); + String startAtStep = runProcessInput.getStartAtStep(); while(true) { /////////////////////////////////////////////////////////////////////////////////////////////////////// // always refresh the step list - as any step that runs can modify it (in the process state). // // this is why we don't do a loop over the step list - as we'd get ConcurrentModificationExceptions. // + // deal with if we were told, from the input, to start After a step, or start At a step. // /////////////////////////////////////////////////////////////////////////////////////////////////////// - List stepList = getAvailableStepList(processState, process, lastStepName); + List stepList; + if(startAtStep == null) + { + stepList = getAvailableStepList(processState, process, lastStepName, false); + } + else + { + stepList = getAvailableStepList(processState, process, startAtStep, true); + + /////////////////////////////////////////////////////////////////////////////////// + // clear this field - so after we run a step, we'll then loop in last-step mode. // + /////////////////////////////////////////////////////////////////////////////////// + startAtStep = null; + + /////////////////////////////////////////////////////////////////////////////////// + // if we're going to run a backend step now, let it see that this is a step-back // + /////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(true); + } + if(stepList.isEmpty()) { break; @@ -232,7 +259,18 @@ public class RunProcessAction ////////////////////////////////////////////////// throw (new QException("Unsure how to run a step of type: " + step.getClass().getName())); } + + //////////////////////////////////////////////////////////////////////////////////////// + // only let this value be set for the original back step - don't let it stick around. // + // if a process wants to keep track of this itself, it can, but in a different slot. // + //////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // in case we broke from the loop above (e.g., by going directly into a frontend step), once again make sure to lower this flag. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + processState.setIsStepBack(false); } @@ -264,6 +302,12 @@ public class RunProcessAction processFrontendStepFieldDefaultValues(processState, step); processFrontendComponents(processState, step); processState.setNextStepName(step.getName()); + + if(StringUtils.hasContent(step.getBackStepName()) && processState.getBackStepName().isEmpty()) + { + processState.setBackStepName(step.getBackStepName()); + } + return LoopTodo.BREAK; } case SKIP -> @@ -317,6 +361,7 @@ public class RunProcessAction // else run the given lastStepName // ///////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); step = process.getStep(lastStepName); if(step == null) { @@ -398,6 +443,7 @@ public class RunProcessAction // its sub-steps, or, to fall out of the loop and end the process. // ////////////////////////////////////////////////////////////////////////////////////////////////////// processState.clearNextStepName(); + processState.clearBackStepName(); runStateMachineStep(nextStepName.get(), process, processState, stateKey, runProcessInput, runProcessOutput, stackDepth + 1); return; } @@ -621,8 +667,10 @@ public class RunProcessAction /******************************************************************************* ** Get the list of steps which are eligible to run. + ** + ** lastStep will be included in the list, or not, based on includeLastStep. *******************************************************************************/ - private List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep) throws QException + static List getAvailableStepList(ProcessState processState, QProcessMetaData process, String lastStep, boolean includeLastStep) throws QException { if(lastStep == null) { @@ -649,6 +697,10 @@ public class RunProcessAction if(stepName.equals(lastStep)) { foundLastStep = true; + if(includeLastStep) + { + validStepNames.add(stepName); + } } } return (stepNamesToSteps(process, validStepNames)); @@ -660,7 +712,7 @@ public class RunProcessAction /******************************************************************************* ** *******************************************************************************/ - private List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException + private static List stepNamesToSteps(QProcessMetaData process, List stepNames) throws QException { List result = new ArrayList<>(); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java index ad7a0827..c6d07011 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessState.java @@ -40,6 +40,8 @@ public class ProcessState implements Serializable private Map values = new HashMap<>(); private List stepList = new ArrayList<>(); private Optional nextStepName = Optional.empty(); + private Optional backStepName = Optional.empty(); + private boolean isStepBack = false; private ProcessMetaDataAdjustment processMetaDataAdjustment = null; @@ -122,6 +124,39 @@ public class ProcessState implements Serializable + /******************************************************************************* + ** Getter for backStepName + ** + *******************************************************************************/ + public Optional getBackStepName() + { + return backStepName; + } + + + + /******************************************************************************* + ** Setter for backStepName + ** + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = Optional.of(backStepName); + } + + + + /******************************************************************************* + ** clear out the value of backStepName (set the Optional to empty) + ** + *******************************************************************************/ + public void clearBackStepName() + { + this.backStepName = Optional.empty(); + } + + + /******************************************************************************* ** Getter for stepList ** @@ -176,4 +211,35 @@ public class ProcessState implements Serializable } + + /******************************************************************************* + ** Getter for isStepBack + *******************************************************************************/ + public boolean getIsStepBack() + { + return (this.isStepBack); + } + + + + /******************************************************************************* + ** Setter for isStepBack + *******************************************************************************/ + public void setIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + } + + + + /******************************************************************************* + ** Fluent setter for isStepBack + *******************************************************************************/ + public ProcessState withIsStepBack(boolean isStepBack) + { + this.isStepBack = isStepBack; + return (this); + } + + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java index bfaad833..81ff1d77 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunBackendStepInput.java @@ -419,6 +419,17 @@ public class RunBackendStepInput extends AbstractActionInput + /******************************************************************************* + ** Accessor for processState's isStepBack attribute + ** + *******************************************************************************/ + public boolean getIsStepBack() + { + return processState.getIsStepBack(); + } + + + /******************************************************************************* ** Accessor for processState - protected, because we generally want to access ** its members through wrapper methods, we think diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java index a099caaf..c9d500f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/processes/RunProcessInput.java @@ -49,6 +49,7 @@ public class RunProcessInput extends AbstractActionInput private ProcessState processState; private FrontendStepBehavior frontendStepBehavior = FrontendStepBehavior.BREAK; private String startAfterStep; + private String startAtStep; private String processUUID; private AsyncJobCallback asyncJobCallback; @@ -451,4 +452,35 @@ public class RunProcessInput extends AbstractActionInput { return asyncJobCallback; } + + /******************************************************************************* + ** Getter for startAtStep + *******************************************************************************/ + public String getStartAtStep() + { + return (this.startAtStep); + } + + + + /******************************************************************************* + ** Setter for startAtStep + *******************************************************************************/ + public void setStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + } + + + + /******************************************************************************* + ** Fluent setter for startAtStep + *******************************************************************************/ + public RunProcessInput withStartAtStep(String startAtStep) + { + this.startAtStep = startAtStep; + return (this); + } + + } \ No newline at end of file diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java index 0c07043e..514e4fae 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/processes/QFrontendStepMetaData.java @@ -48,6 +48,7 @@ public class QFrontendStepMetaData extends QStepMetaData private Map formFieldMap; private String format; + private String backStepName; private List helpContents; @@ -436,4 +437,35 @@ public class QFrontendStepMetaData extends QStepMetaData } + + /******************************************************************************* + ** Getter for backStepName + *******************************************************************************/ + public String getBackStepName() + { + return (this.backStepName); + } + + + + /******************************************************************************* + ** Setter for backStepName + *******************************************************************************/ + public void setBackStepName(String backStepName) + { + this.backStepName = backStepName; + } + + + + /******************************************************************************* + ** Fluent setter for backStepName + *******************************************************************************/ + public QFrontendStepMetaData withBackStepName(String backStepName) + { + this.backStepName = backStepName; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java index 1d989f96..bf7eeb0a 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessActionTest.java @@ -23,10 +23,15 @@ package com.kingsrook.qqq.backend.core.actions.processes; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessState; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceLambda; @@ -35,6 +40,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.processes.QStateMachineStep; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData; +import com.kingsrook.qqq.backend.core.utils.collections.MultiLevelMapHelper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -73,14 +80,14 @@ class RunProcessActionTest extends BaseTest ///////////////////////////////////////////////////////////////// // two-steps - a, points at b; b has no next-step, so it exits // ///////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -109,8 +116,8 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) - .addStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) + .withStep(QStateMachineStep.frontendOnly("a", new QFrontendStepMetaData().withName("aFrontend")).withDefaultNextStepName("b")) + .withStep(QStateMachineStep.frontendOnly("b", new QFrontendStepMetaData().withName("bFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -150,7 +157,7 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); @@ -193,14 +200,14 @@ class RunProcessActionTest extends BaseTest // since it never goes to the frontend, it'll stack overflow // // (though we'll catch it ourselves before JVM does) // /////////////////////////////////////////////////////////////// - .addStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") + .withStep(QStateMachineStep.backendOnly("a", new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepA"); runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") + .withStep(QStateMachineStep.backendOnly("b", new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> { log.add("in StepB"); @@ -238,7 +245,7 @@ class RunProcessActionTest extends BaseTest { QProcessMetaData process = new QProcessMetaData().withName("test") - .addStep(QStateMachineStep.frontendThenBackend("a", + .withStep(QStateMachineStep.frontendThenBackend("a", new QFrontendStepMetaData().withName("aFrontend"), new QBackendStepMetaData().withName("aBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -247,7 +254,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("b"); })))) - .addStep(QStateMachineStep.frontendThenBackend("b", + .withStep(QStateMachineStep.frontendThenBackend("b", new QFrontendStepMetaData().withName("bFrontend"), new QBackendStepMetaData().withName("bBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -256,7 +263,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("c"); })))) - .addStep(QStateMachineStep.frontendThenBackend("c", + .withStep(QStateMachineStep.frontendThenBackend("c", new QFrontendStepMetaData().withName("cFrontend"), new QBackendStepMetaData().withName("cBackend") .withCode(new QCodeReferenceLambda((runBackendStepInput, runBackendStepOutput) -> @@ -265,7 +272,7 @@ class RunProcessActionTest extends BaseTest runBackendStepOutput.getProcessState().setNextStepName("d"); })))) - .addStep(QStateMachineStep.frontendOnly("d", + .withStep(QStateMachineStep.frontendOnly("d", new QFrontendStepMetaData().withName("dFrontend"))) .withStepFlow(ProcessStepFlow.STATE_MACHINE); @@ -321,7 +328,132 @@ class RunProcessActionTest extends BaseTest runProcessOutput = new RunProcessAction().execute(input); assertEquals(List.of("in StepA", "in StepB", "in StepC"), log); assertThat(runProcessOutput.getProcessState().getNextStepName()).isEmpty(); + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGoingBack() throws QException + { + AtomicInteger backCount = new AtomicInteger(0); + Map stepRunCounts = new HashMap<>(); + + BackendStep backendStep = (runBackendStepInput, runBackendStepOutput) -> + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // shared backend-step lambda, that will do the same thing for both - but using step name to count how many times each is executed. // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + MultiLevelMapHelper.getOrPutAndIncrement(stepRunCounts, runBackendStepInput.getStepName()); + if(runBackendStepInput.getIsStepBack()) + { + backCount.incrementAndGet(); + } + }; + + /////////////////////////////////////////////////////////// + // normal flow here: a -> b -> c // + // but, b can go back to a, as in: a -> b -> a -> b -> c // + /////////////////////////////////////////////////////////// + QProcessMetaData process = new QProcessMetaData().withName("test") + .withStep(new QBackendStepMetaData() + .withName("a") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStep(new QFrontendStepMetaData() + .withName("b") + .withBackStepName("a")) + .withStep(new QBackendStepMetaData() + .withName("c") + .withCode(new QCodeReferenceLambda<>(backendStep))) + .withStepFlow(ProcessStepFlow.LINEAR); + + QContext.getQInstance().addProcess(process); + + RunProcessInput input = new RunProcessInput(); + input.setProcessName("test"); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + + /////////////////////////////////////////////////////////// + // start the process - we should be sent to b (frontend) // + /////////////////////////////////////////////////////////// + RunProcessOutput runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(0, backCount.get()); + assertEquals(Map.of("a", 1), stepRunCounts); + + //////////////////////////////////////////////////////////////// + // resume after b, but in back-mode - should end up back at b // + //////////////////////////////////////////////////////////////// + input.setStartAfterStep(null); + input.setStartAtStep("a"); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isPresent().get() + .isEqualTo("b"); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2), stepRunCounts); + + //////////////////////////////////////////////////////////////////////////// + // resume after b, in regular (forward) mode - should wrap up the process // + //////////////////////////////////////////////////////////////////////////// + input.setStartAfterStep("b"); + input.setStartAtStep(null); + runProcessOutput = new RunProcessAction().execute(input); + assertThat(runProcessOutput.getProcessState().getNextStepName()) + .isEmpty(); + + assertEquals(1, backCount.get()); + assertEquals(Map.of("a", 2, "c", 1), stepRunCounts); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testGetAvailableStepList() throws QException + { + QProcessMetaData process = new QProcessMetaData() + .withStep(new QBackendStepMetaData().withName("A")) + .withStep(new QBackendStepMetaData().withName("B")) + .withStep(new QBackendStepMetaData().withName("C")) + .withStep(new QBackendStepMetaData().withName("D")) + .withStep(new QBackendStepMetaData().withName("E")); + + ProcessState processState = new ProcessState(); + processState.setStepList(process.getStepList().stream().map(s -> s.getName()).toList()); + + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, null, true)); + + assertStepListNames(List.of("B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", false)); + assertStepListNames(List.of("A", "B", "C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "A", true)); + + assertStepListNames(List.of("D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", false)); + assertStepListNames(List.of("C", "D", "E"), RunProcessAction.getAvailableStepList(processState, process, "C", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "E", false)); + assertStepListNames(List.of("E"), RunProcessAction.getAvailableStepList(processState, process, "E", true)); + + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", false)); + assertStepListNames(Collections.emptyList(), RunProcessAction.getAvailableStepList(processState, process, "Z", true)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void assertStepListNames(List expectedNames, List actualSteps) + { + assertEquals(expectedNames, actualSteps.stream().map(s -> s.getName()).toList()); } } \ No newline at end of file diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java index ec1c5bbf..1c0bd7b7 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/backend/javalin/QJavalinProcessHandler.java @@ -89,6 +89,7 @@ import io.javalin.apibuilder.EndpointGroup; import io.javalin.http.Context; import io.javalin.http.UploadedFile; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.BooleanUtils; import org.eclipse.jetty.http.HttpStatus; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; import static io.javalin.apibuilder.ApiBuilder.get; @@ -321,7 +322,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processInit(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.BREAK); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.BREAK); } @@ -335,7 +336,7 @@ public class QJavalinProcessHandler *******************************************************************************/ public static void processRun(Context context) { - doProcessInitOrStep(context, null, null, RunProcessInput.FrontendStepBehavior.SKIP); + doProcessInitOrStep(context, null, null, null, RunProcessInput.FrontendStepBehavior.SKIP); } @@ -343,7 +344,7 @@ public class QJavalinProcessHandler /******************************************************************************* ** *******************************************************************************/ - private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) + private static void doProcessInitOrStep(Context context, String processUUID, String startAfterStep, String startAtStep, RunProcessInput.FrontendStepBehavior frontendStepBehavior) { Map resultForCaller = new HashMap<>(); Exception returningException = null; @@ -357,8 +358,23 @@ public class QJavalinProcessHandler resultForCaller.put("processUUID", processUUID); String processName = context.pathParam("processName"); - LOG.info(startAfterStep == null ? "Initiating process [" + processName + "] [" + processUUID + "]" - : "Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + + if(startAfterStep == null && startAtStep == null) + { + LOG.info("Initiating process [" + processName + "] [" + processUUID + "]"); + } + else if(startAfterStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] after step [" + startAfterStep + "]"); + } + else if(startAtStep != null) + { + LOG.info("Resuming process [" + processName + "] [" + processUUID + "] at step [" + startAtStep + "]"); + } + else + { + LOG.warn("A logical impossibility was reached, regarding the nullity of startAfterStep and startAtStep, at least given how this code was originally written."); + } RunProcessInput runProcessInput = new RunProcessInput(); QJavalinImplementation.setupSession(context, runProcessInput); @@ -367,11 +383,13 @@ public class QJavalinProcessHandler runProcessInput.setFrontendStepBehavior(frontendStepBehavior); runProcessInput.setProcessUUID(processUUID); runProcessInput.setStartAfterStep(startAfterStep); + runProcessInput.setStartAtStep(startAtStep); populateRunProcessRequestWithValuesFromContext(context, runProcessInput); String reportName = ValueUtils.getValueAsString(runProcessInput.getValue("reportName")); QJavalinAccessLogger.logStart(startAfterStep == null ? "processInit" : "processStep", logPair("processName", processName), logPair("processUUID", processUUID), StringUtils.hasContent(startAfterStep) ? logPair("startAfterStep", startAfterStep) : null, + StringUtils.hasContent(startAtStep) ? logPair("startAtStep", startAfterStep) : null, StringUtils.hasContent(reportName) ? logPair("reportName", reportName) : null); ////////////////////////////////////////////////////////////////////////////////////////////////// @@ -460,6 +478,7 @@ public class QJavalinProcessHandler } resultForCaller.put("values", runProcessOutput.getValues()); runProcessOutput.getProcessState().getNextStepName().ifPresent(nextStep -> resultForCaller.put("nextStep", nextStep)); + runProcessOutput.getProcessState().getBackStepName().ifPresent(backStep -> resultForCaller.put("backStep", backStep)); ///////////////////////////////////////////////////////////////////////////////////////////////////////////// // todo - delete after all frontends look for processMetaDataAdjustment instead of updatedFrontendStepList // @@ -660,8 +679,19 @@ public class QJavalinProcessHandler public static void processStep(Context context) { String processUUID = context.pathParam("processUUID"); - String lastStep = context.pathParam("step"); - doProcessInitOrStep(context, processUUID, lastStep, RunProcessInput.FrontendStepBehavior.BREAK); + + String startAfterStep = null; + String startAtStep = null; + if(BooleanUtils.isTrue(ValueUtils.getValueAsBoolean(context.queryParam("isStepBack")))) + { + startAtStep = context.pathParam("step"); + } + else + { + startAfterStep = context.pathParam("step"); + } + + doProcessInitOrStep(context, processUUID, startAfterStep, startAtStep, RunProcessInput.FrontendStepBehavior.BREAK); } diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java index c138b0bc..372b9aac 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepInput.java @@ -45,6 +45,7 @@ public class ProcessInitOrStepInput extends AbstractMiddlewareInput ///////////////////////////////////// private String processUUID; private String startAfterStep; + // todo - add (in next version?) startAtStep (for back) private RunProcessInput.FrontendStepBehavior frontendStepBehavior = RunProcessInput.FrontendStepBehavior.BREAK; diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java index f7d0d4a5..aef6ab44 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/executors/io/ProcessInitOrStepOrStatusOutputInterface.java @@ -58,6 +58,8 @@ public interface ProcessInitOrStepOrStatusOutputInterface extends AbstractMiddle *******************************************************************************/ void setNextStep(String nextStep); + // todo - add (in next version?) backStep + /******************************************************************************* ** Setter for values *******************************************************************************/ From 61582680f37a906e1c204c579c102740d40cd620 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:01:35 -0600 Subject: [PATCH 30/84] CE-1955 - Add support for back to bulk-load process --- .../qqq/backend/core/instances/QInstanceEnricher.java | 4 ++++ .../bulk/insert/BulkInsertReceiveFileMappingStep.java | 1 + 2 files changed, 5 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 7bf3c306..87be37bc 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -898,6 +898,7 @@ public class QInstanceEnricher QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() .withName("fileMapping") .withLabel("File Mapping") + .withBackStepName("upload") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() @@ -911,6 +912,7 @@ public class QInstanceEnricher QFrontendStepMetaData valueMappingScreen = new QFrontendStepMetaData() .withName("valueMapping") .withLabel("Value Mapping") + .withBackStepName("prepareFileMapping") .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM)); QBackendStepMetaData receiveValueMappingStep = new QBackendStepMetaData() @@ -934,7 +936,9 @@ public class QInstanceEnricher // put the bulk-load profile form (e.g., for saving it) on the review & result screens) // ////////////////////////////////////////////////////////////////////////////////////////// process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW) + .withBackStepName("prepareFileMapping") .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); + process.getFrontendStep(StreamedETLWithFrontendProcess.STEP_NAME_RESULT) .getComponents().add(0, new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_PROFILE_FORM)); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java index bc6ae239..afe3c9f1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileMappingStep.java @@ -190,6 +190,7 @@ public class BulkInsertReceiveFileMappingStep implements BackendStep // it's also where the value-mapping loop of steps points. // // and, this will actually be the default (e.g., the step after this one). // ////////////////////////////////////////////////////////////////////////////////// + runBackendStepInput.addValue("valueMappingFieldIndex", -1); } else { From 8ea16db1fc6d7f3480c607c3ee654ae045ce0417 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:10:24 -0600 Subject: [PATCH 31/84] CE-1955 - Checkstyle --- .../implementations/bulk/insert/BulkInsertTransformStep.java | 2 +- .../actions/processes/ProcessSummaryLineInterfaceAssert.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f2d87767..73d35070 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -91,7 +91,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private int rowsProcessed = 0; - private final int EXAMPLE_ROW_LIMIT = 10; + private static final int EXAMPLE_ROW_LIMIT = 10; /******************************************************************************* diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java index 861e21a7..60d4561c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/model/actions/processes/ProcessSummaryLineInterfaceAssert.java @@ -1,4 +1,3 @@ - /* * QQQ - Low-code Application Framework for Engineers. * Copyright (C) 2021-2024. Kingsrook, LLC From a439bffc698a51fd50f1308fe6e90b06383be75a Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:34:37 -0600 Subject: [PATCH 32/84] Add support for OpenAPIEnumSubSet --- .../javalin/schemabuilder/SchemaBuilder.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java index 9b0de98c..a97a7019 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/schemabuilder/SchemaBuilder.java @@ -38,6 +38,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIHasAdditionalProperties; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIIncludeProperties; @@ -123,7 +124,25 @@ public class SchemaBuilder if(c.isEnum()) { schema.withType(Type.STRING); - schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + + if(element.isAnnotationPresent(OpenAPIEnumSubSet.class)) + { + try + { + OpenAPIEnumSubSet enumSubSetAnnotation = element.getAnnotation(OpenAPIEnumSubSet.class); + Class> enumSubSetClass = enumSubSetAnnotation.value(); + OpenAPIEnumSubSet.EnumSubSet enumSubSetContainer = enumSubSetClass.getConstructor().newInstance(); + schema.withEnumValues(enumSubSetContainer.getSubSet().stream().map(e -> String.valueOf(e)).collect(Collectors.toList())); + } + catch(Exception e) + { + throw new QRuntimeException("Error processing OpenAPIEnumSubSet on element: " + element, e); + } + } + else + { + schema.withEnumValues(Arrays.stream(c.getEnumConstants()).map(e -> String.valueOf(e)).collect(Collectors.toList())); + } } else if(c.equals(String.class)) { From 53ca77cde691a7e17abb849f67d2133dfb5e9a84 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:35:10 -0600 Subject: [PATCH 33/84] CE-1955 Update to use an enum-subset (excluding new BULK_LOAD components) --- .../components/FrontendComponent.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java index e28af1ce..f6c15758 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FrontendComponent.java @@ -23,11 +23,13 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.io.Serializable; +import java.util.EnumSet; import java.util.Map; import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIMapKnownEntries; @@ -63,10 +65,39 @@ public class FrontendComponent implements ToSchema + /*************************************************************************** + ** + ***************************************************************************/ + public static class QComponentTypeSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(QComponentType.class); + subSet.remove(QComponentType.BULK_LOAD_FILE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_VALUE_MAPPING_FORM); + subSet.remove(QComponentType.BULK_LOAD_PROFILE_FORM); + QComponentTypeSubSet.subSet = subSet; + } + + return(subSet); + } + } + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("The type of this component. e.g., what kind of UI element(s) should be presented to the user.") + @OpenAPIEnumSubSet(QComponentTypeSubSet.class) public QComponentType getType() { return (this.wrapped.getType()); From 8ec6ccd691462d7e15ac06f57e22e417dc0b4aa0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 27 Nov 2024 15:36:36 -0600 Subject: [PATCH 34/84] CE-1955 added an icon for bulk-load process in example (since it has one now) --- .../src/main/resources/openapi/v1/openapi.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index f89ba784..beda0fc6 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -1483,6 +1483,8 @@ paths: processes: person.bulkInsert: hasPermission: true + icon: + name: "library_add" isHidden: true label: "Person Bulk Insert" name: "person.bulkInsert" From 0e93b90270eed8365d679878f6604aa25f5ddd54 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:50:05 -0600 Subject: [PATCH 35/84] CE-1955 Add mapping and validation of possible-values; refactor error classes some for rollup possible value errors --- .../bulk/insert/BulkInsertTransformStep.java | 18 +- .../AbstractBulkLoadRollableValueError.java | 55 ++++ .../mapping/BulkLoadPossibleValueError.java | 72 ++++++ .../insert/mapping/BulkLoadValueMapper.java | 238 ++++++++++++++++++ .../mapping/BulkLoadValueTypeError.java | 7 +- .../bulk/insert/mapping/FlatRowsToRecord.java | 2 +- .../bulk/insert/mapping/TallRowsToRecord.java | 2 +- .../bulk/insert/mapping/ValueMapper.java | 134 ---------- ...licitFieldNameSuffixIndexBasedMapping.java | 2 +- .../WideRowsToRecordWithSpreadMapping.java | 2 +- ...Test.java => BulkLoadValueMapperTest.java} | 4 +- .../insert/mapping/FlatRowsToRecordTest.java | 45 ++++ 12 files changed, 428 insertions(+), 153 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java delete mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java rename qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/{ValueMapperTest.java => BulkLoadValueMapperTest.java} (96%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index 73d35070..ad531967 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -56,8 +56,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; @@ -210,10 +210,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep { for(QErrorMessage error : record.getErrors()) { - if(error instanceof BulkLoadValueTypeError blvte) + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) { - processSummaryWarningsAndErrorsRollup.addError(blvte.getMessageToUseAsProcessSummaryRollupKey(), null); - addToErrorToExampleRowValueMap(blvte, record); + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); } else { @@ -348,14 +348,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - private void addToErrorToExampleRowValueMap(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + private void addToErrorToExampleRowValueMap(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - String message = bulkLoadValueTypeError.getMessageToUseAsProcessSummaryRollupKey(); + String message = bulkLoadRollableValueError.getMessageToUseAsProcessSummaryRollupKey(); List rowValues = errorToExampleRowValueMap.computeIfAbsent(message, k -> new ArrayList<>()); if(rowValues.size() < EXAMPLE_ROW_LIMIT) { - rowValues.add(new RowValue(bulkLoadValueTypeError, record)); + rowValues.add(new RowValue(bulkLoadRollableValueError, record)); } } @@ -550,9 +550,9 @@ public class BulkInsertTransformStep extends AbstractTransformStep /*************************************************************************** ** ***************************************************************************/ - public RowValue(BulkLoadValueTypeError bulkLoadValueTypeError, QRecord record) + public RowValue(AbstractBulkLoadRollableValueError bulkLoadRollableValueError, QRecord record) { - this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadValueTypeError.getValue())); + this(BulkLoadRecordUtils.getRowNosString(record), ValueUtils.getValueAsString(bulkLoadRollableValueError.getValue())); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java new file mode 100644 index 00000000..74082fd7 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/AbstractBulkLoadRollableValueError.java @@ -0,0 +1,55 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractBulkLoadRollableValueError extends BadInputStatusMessage +{ + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public AbstractBulkLoadRollableValueError(String message) + { + super(message); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract String getMessageToUseAsProcessSummaryRollupKey(); + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract Serializable getValue(); +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java new file mode 100644 index 00000000..199d775f --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadPossibleValueError.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; + + +/******************************************************************************* + ** Specialized error for records, for bulk-load use-cases, where we want to + ** report back info to the user about the field & value. + *******************************************************************************/ +public class BulkLoadPossibleValueError extends AbstractBulkLoadRollableValueError +{ + private final String fieldLabel; + private final Serializable value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadPossibleValueError(String fieldName, Serializable value, String fieldLabel) + { + super("Value [" + value + "] for field [" + fieldLabel + "] is not a valid option"); + this.value = value; + this.fieldLabel = fieldLabel; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String getMessageToUseAsProcessSummaryRollupKey() + { + return ("Unrecognized value for field [" + fieldLabel + "]"); + } + + + + /******************************************************************************* + ** Getter for value + ** + *******************************************************************************/ + @Override + public Serializable getValue() + { + return value; + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java new file mode 100644 index 00000000..cc44e027 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -0,0 +1,238 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.values.SearchPossibleValueSourceAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceInput; +import com.kingsrook.qqq.backend.core.model.actions.values.SearchPossibleValueSourceOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.ListingHash; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadValueMapper +{ + private static final QLogger LOG = QLogger.getLogger(BulkLoadValueMapper.class); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException + { + valueMapping(records, mapping, table, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; + String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; + + Map> possibleValueToRecordMap = new HashMap<>(); + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry valueEntry : record.getValues().entrySet()) + { + QFieldMetaData field = table.getField(valueEntry.getKey()); + Serializable value = valueEntry.getValue(); + + /////////////////// + // value mappin' // + /////////////////// + if(mappingForTable.containsKey(field.getName()) && value != null) + { + Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + if(mappedValue != null) + { + value = mappedValue; + } + } + + ///////////////////// + // type convertin' // + ///////////////////// + if(value != null && !"".equals(value)) + { + if(StringUtils.hasContent(field.getPossibleValueSourceName())) + { + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.computeIfAbsent(field.getName(), k -> new ListingHash<>()); + fieldPossibleValueToRecordMap.add(ValueUtils.getValueAsString(value), record); + } + else + { + QFieldType type = field.getType(); + try + { + value = ValueUtils.getValueAsFieldType(type, value); + } + catch(Exception e) + { + record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); + } + } + } + + record.setValue(field.getName(), value); + } + + ////////////////////////////////////// + // recursively process associations // + ////////////////////////////////////// + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + String associationName = entry.getKey(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); + if(association.isPresent()) + { + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); + } + else + { + throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); + } + } + } + + ////////////////////////////////////////// + // look up and validate possible values // + ////////////////////////////////////////// + for(Map.Entry> entry : possibleValueToRecordMap.entrySet()) + { + String fieldName = entry.getKey(); + QFieldMetaData field = table.getField(fieldName); + ListingHash fieldPossibleValueToRecordMap = possibleValueToRecordMap.get(fieldName); + + handlePossibleValues(field, fieldPossibleValueToRecordMap, associationNamePrefixForFields, tableLabelPrefix); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException + { + Set values = fieldPossibleValueToRecordMap.keySet(); + Map> valuesFound = new HashMap<>(); + Set valuesNotFound = new HashSet<>(values); + + //////////////////////////////////////////////////////// + // do a search, trying to use all given values as ids // + //////////////////////////////////////////////////////// + SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList idList = new ArrayList<>(values); + searchPossibleValueSourceInput.setIdList(idList); + searchPossibleValueSourceInput.setLimit(values.size()); + LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + + //////////////////////////////////////////////////////////////////////////////////////////////////// + // for each possible value found, remove it from the set of ones not-found, and store it as a hit // + //////////////////////////////////////////////////////////////////////////////////////////////////// + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); + valuesFound.put(valueAsString, possibleValue); + valuesNotFound.remove(valueAsString); + } + + /////////////////////////////////////////////////////////////////////////// + // if there are any that weren't found, try to look them up now by label // + /////////////////////////////////////////////////////////////////////////// + if(!valuesNotFound.isEmpty()) + { + searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); + searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); + ArrayList labelList = new ArrayList<>(valuesNotFound); + searchPossibleValueSourceInput.setLabelList(labelList); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); + + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0))); + searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) + { + valuesFound.put(possibleValue.getLabel(), possibleValue); + valuesNotFound.remove(possibleValue.getLabel()); + } + } + + //////////////////////////////////////////////////////////////////////////////// + // for each record now, either set a usable value (e.g., a PV.id) or an error // + //////////////////////////////////////////////////////////////////////////////// + for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) + { + String value = entry.getKey(); + for(QRecord record : entry.getValue()) + { + if(valuesFound.containsKey(value)) + { + record.setValue(field.getName(), valuesFound.get(value).getId()); + } + else + { + record.addError(new BulkLoadPossibleValueError(associationNamePrefixForFields + field.getName(), value, tableLabelPrefix + field.getLabel())); + } + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java index a7e6f371..6bee0449 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -24,17 +24,15 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; /******************************************************************************* ** Specialized error for records, for bulk-load use-cases, where we want to ** report back info to the user about the field & value. *******************************************************************************/ -public class BulkLoadValueTypeError extends BadInputStatusMessage +public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError { private final String fieldLabel; - private final String fieldName; private final Serializable value; private final QFieldType type; @@ -47,7 +45,6 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) { super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); - this.fieldName = fieldName; this.value = value; this.type = type; this.fieldLabel = fieldLabel; @@ -58,6 +55,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage /*************************************************************************** ** ***************************************************************************/ + @Override public String getMessageToUseAsProcessSummaryRollupKey() { return ("Cannot convert value for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); @@ -69,6 +67,7 @@ public class BulkLoadValueTypeError extends BadInputStatusMessage ** Getter for value ** *******************************************************************************/ + @Override public Serializable getValue() { return value; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 243496ef..310fd8db 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -70,7 +70,7 @@ public class FlatRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 8e0443d3..84ab1fa7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -138,7 +138,7 @@ public class TallRowsToRecord implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java deleted file mode 100644 index 44a37f6a..00000000 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; - - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import com.kingsrook.qqq.backend.core.context.QContext; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; -import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; -import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; -import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import com.kingsrook.qqq.backend.core.utils.StringUtils; -import com.kingsrook.qqq.backend.core.utils.ValueUtils; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class ValueMapper -{ - private static final QLogger LOG = QLogger.getLogger(ValueMapper.class); - - - - /*************************************************************************** - ** - ***************************************************************************/ - public static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table) throws QException - { - valueMapping(records, mapping, table, null); - } - - - - /*************************************************************************** - ** - ***************************************************************************/ - private static void valueMapping(List records, BulkInsertMapping mapping, QTableMetaData table, String associationNameChain) throws QException - { - if(CollectionUtils.nullSafeIsEmpty(records)) - { - return; - } - - String associationNamePrefixForFields = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." : ""; - String tableLabelPrefix = StringUtils.hasContent(associationNameChain) ? table.getLabel() + ": " : ""; - - Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); - for(QRecord record : records) - { - for(Map.Entry valueEntry : record.getValues().entrySet()) - { - QFieldMetaData field = table.getField(valueEntry.getKey()); - Serializable value = valueEntry.getValue(); - - /////////////////// - // value mappin' // - /////////////////// - if(mappingForTable.containsKey(field.getName()) && value != null) - { - Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); - if(mappedValue != null) - { - value = mappedValue; - } - } - - ///////////////////// - // type convertin' // - ///////////////////// - if(value != null) - { - QFieldType type = field.getType(); - try - { - value = ValueUtils.getValueAsFieldType(type, value); - } - catch(Exception e) - { - record.addError(new BulkLoadValueTypeError(associationNamePrefixForFields + field.getName(), value, type, tableLabelPrefix + field.getLabel())); - } - } - - record.setValue(field.getName(), value); - } - - ////////////////////////////////////// - // recursively process associations // - ////////////////////////////////////// - for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) - { - String associationName = entry.getKey(); - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); - if(association.isPresent()) - { - QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); - valueMapping(entry.getValue(), mapping, associatedTable, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + associationName : associationName); - } - else - { - throw new QException("Missing association [" + associationName + "] on table [" + table.getName() + "]"); - } - } - } - } - -} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index e79dd2c2..c4231ea7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -75,7 +75,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java index 7b215d2e..e39e95c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithSpreadMapping.java @@ -86,7 +86,7 @@ public class WideRowsToRecordWithSpreadMapping implements RowsToRecordInterface rs.add(record); } - ValueMapper.valueMapping(rs, mapping, table); + BulkLoadValueMapper.valueMapping(rs, mapping, table); return (rs); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java similarity index 96% rename from qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java rename to qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 4348cb2b..687f1f62 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -41,7 +41,7 @@ import static org.assertj.core.api.Assertions.assertThat; /******************************************************************************* ** Unit test for ValueMapper *******************************************************************************/ -class ValueMapperTest extends BaseTest +class BulkLoadValueMapperTest extends BaseTest { /******************************************************************************* @@ -98,7 +98,7 @@ class ValueMapperTest extends BaseTest ); JSONObject expectedJson = recordToJson(expectedRecord); - ValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), mapping, QContext.getQInstance().getTable(TestUtils.TABLE_NAME_ORDER)); JSONObject actualJson = recordToJson(inputRecord); System.out.println("Before"); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index fd1f70c4..5237fcf5 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mode import com.kingsrook.qqq.backend.core.utils.TestUtils; import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -102,6 +103,8 @@ class FlatRowsToRecordTest extends BaseTest assertEquals("Row 5", records.get(0).getBackendDetail("rowNos")); } + + /******************************************************************************* ** *******************************************************************************/ @@ -205,6 +208,48 @@ class FlatRowsToRecordTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValueMappings() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Home State" }, + new Serializable[] { 1, "Homer", "Simpson", 1 }, + new Serializable[] { 2, "Marge", "Simpson", "MO" }, + new Serializable[] { 3, "Bart", "Simpson", null }, + new Serializable[] { 4, "Ned", "Flanders", "Not a state" }, + new Serializable[] { 5, "Mr.", "Burns", 5 } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "homeStateId", "Home State" + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(5, records.size()); + assertEquals(List.of("Homer", "Marge", "Bart", "Ned", "Mr."), getValues(records, "firstName")); + assertEquals(ListBuilder.of(1, 2, null, "Not a state", 5), getValues(records, "homeStateId")); + + assertThat(records.get(0).getErrors()).isNullOrEmpty(); + assertThat(records.get(1).getErrors()).isNullOrEmpty(); + assertThat(records.get(2).getErrors()).isNullOrEmpty(); + assertThat(records.get(3).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + assertThat(records.get(4).getErrors()).hasSize(1).element(0).matches(e -> e.getMessage().contains("not a valid option")); + } + + + /*************************************************************************** ** ***************************************************************************/ From b055913fc8aa1a08456104b9a19cdc8ed3bb42b0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:56:28 -0600 Subject: [PATCH 36/84] CE-1955 Initial checkin --- .../BulkInsertPrepareFileUploadStep.java | 292 ++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java new file mode 100644 index 00000000..c021dcb1 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -0,0 +1,292 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** step before the upload screen, to prepare dynamic help-text for user. + *******************************************************************************/ +public class BulkInsertPrepareFileUploadStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + String tableName = runBackendStepInput.getValueString("tableName"); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); + runBackendStepOutput.addValue("tableStructure", tableStructure); + + List requiredFields = new ArrayList<>(); + List additionalFields = new ArrayList<>(); + for(QFieldMetaData field : tableStructure.getFields()) + { + if(field.getIsRequired()) + { + requiredFields.add(field); + } + else + { + additionalFields.add(field); + } + } + + StringBuilder html; + String childTableLabels = ""; + + StringBuilder tallCSV = new StringBuilder(); + StringBuilder wideCSV = new StringBuilder(); + StringBuilder flatCSV = new StringBuilder(); + + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // potentially this could be a parameter - for now, hard-code false, but keep the code around that did this // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////// + boolean listFieldsInHelpText = false; + + if(!CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) + { + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file, with one row for each record you want to + insert in the ${tableLabel} table.


+ +

Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional (though encouraged) whether you include + a header row in your file. For Excel files, only the first sheet in the workbook will be used.


+ """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + html.append(""" + Template: ${tableLabel}.csv"""); + } + else + { + html.append(""" +

You can download a template file to see the full list of available fields: + ${tableLabel}.csv +

+ """); + } + } + else + { + childTableLabels = StringUtils.joinWithCommasAndAnd(tableStructure.getAssociations().stream().map(a -> a.getLabel()).toList()) + " table" + StringUtils.plural(table.getAssociations()); + + html = new StringBuilder(""" +

Upload either a CSV or Excel (.xlsx) file. Your file can be in one of three layouts:

+ ${openUL} +

  • Flat: Each row in the file will create one record in the ${tableLabel} table.
    • +
  • Wide: Each row in the file will create one record in the ${tableLabel} table, + and optionally one or more records in the ${childTableLabels}, by supplying additional columns + for each sub-record that you want to create.
    • +
  • Tall: Rows with matching values in the fields being used for the ${tableLabel} + table will be used to create one ${tableLabel} record. One or more records will also be built + in the ${childTableLabels} by providing unique values in each row for the sub-records.
    • +
    + +

    Your file can contain any number of columns. You will be prompted to map fields from + the ${tableLabel} table to columns from your file or default values for all records that + you are loading on the next screen. It is optional (though encouraged) whether you include + a header row in your file. For Excel files, only the first sheet in the workbook will be used.


    + """); + + if(listFieldsInHelpText) + { + appendTableRequiredAndAdditionalFields(html, requiredFields, additionalFields); + } + + addCsvFields(tallCSV, requiredFields, additionalFields); + addCsvFields(wideCSV, requiredFields, additionalFields); + + for(BulkLoadTableStructure association : tableStructure.getAssociations()) + { + if(listFieldsInHelpText) + { + html.append(""" +

    You can also add values for these ${childLabel} fields:

    + """.replace("${childLabel}", association.getLabel())); + appendFieldsAsUlToHtml(html, association.getFields()); + } + + addCsvFields(tallCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", ""); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 1"); + addCsvFields(wideCSV, association.getFields(), Collections.emptyList(), association.getLabel() + ": ", " - 2"); + } + + finishCSV(tallCSV); + finishCSV(wideCSV); + + if(listFieldsInHelpText) + { + html.append(""" + Templates: ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv + """); + } + else + { + html.append(""" +

    You can download a template file to see the full list of available fields: + ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv +

    + """); + } + } + + html.insert(0, """ +
    + File Upload Instructions +
    + """); + html.append("
    "); + + addCsvFields(flatCSV, requiredFields, additionalFields); + finishCSV(flatCSV); + + String htmlString = html.toString() + .replace("${tableLabel}", table.getLabel()) + .replace("${childTableLabels}", childTableLabels) + .replace("${flatCSV}", Base64.getEncoder().encodeToString(flatCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${tallCSV}", Base64.getEncoder().encodeToString(tallCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${wideCSV}", Base64.getEncoder().encodeToString(wideCSV.toString().getBytes(StandardCharsets.UTF_8))) + .replace("${openUL}", "
      "); + + runBackendStepOutput.addValue("upload.html", htmlString); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void finishCSV(StringBuilder flatCSV) + { + flatCSV.deleteCharAt(flatCSV.length() - 1); + flatCSV.append("\n"); + flatCSV.append(flatCSV.toString().replaceAll("[^,]", "")); + flatCSV.append("\n"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields) + { + addCsvFields(csv, requiredFields, additionalFields, "", ""); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void addCsvFields(StringBuilder csv, List requiredFields, List additionalFields, String fieldLabelPrefix, String fieldLabelSuffix) + { + for(QFieldMetaData field : requiredFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + + for(QFieldMetaData field : additionalFields) + { + csv.append(fieldLabelPrefix).append(field.getLabel()).append(fieldLabelSuffix).append(","); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendTableRequiredAndAdditionalFields(StringBuilder html, List requiredFields, List additionalFields) + { + if(!requiredFields.isEmpty()) + { + html.append(""" +

      You will be required to supply values (either in a column in the file, or by + choosing a default value on the next screen) for the following ${tableLabel} fields:

      + """); + appendFieldsAsUlToHtml(html, requiredFields); + } + + if(!additionalFields.isEmpty()) + { + if(requiredFields.isEmpty()) + { + html.append(""" +

      You can supply values (either in a column in the file, or by choosing a + default value on the next screen) for the following ${tableLabel} fields:

      + """); + } + else + { + html.append("

      You can also add values for these fields:

      "); + } + + appendFieldsAsUlToHtml(html, additionalFields); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static void appendFieldsAsUlToHtml(StringBuilder html, List additionalFields) + { + html.append("${openUL}"); + for(QFieldMetaData field : additionalFields) + { + html.append("
    • ").append(field.getLabel()).append("
      • "); + } + html.append("

    "); + } + +} From b0cc93cbb79a4dc05263141f1c784990d6600d0d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:56:45 -0600 Subject: [PATCH 37/84] CE-1955 Add FILE_UPLOAD adornment type --- .../model/metadata/fields/AdornmentType.java | 62 +++++++++++++++++++ .../model/metadata/fields/FieldAdornment.java | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java index ba78ceef..1b1941f5 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/AdornmentType.java @@ -41,6 +41,7 @@ public enum AdornmentType RENDER_HTML, REVEAL, FILE_DOWNLOAD, + FILE_UPLOAD, ERROR; ////////////////////////////////////////////////////////////////////////// // keep these values in sync with AdornmentType.ts in qqq-frontend-core // @@ -164,4 +165,65 @@ public enum AdornmentType } } + + + /******************************************************************************* + ** + *******************************************************************************/ + public static class FileUploadAdornment + { + public static String FORMAT = "format"; + public static String WIDTH = "width"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static FieldAdornment newFieldAdornment() + { + return (new FieldAdornment(AdornmentType.FILE_UPLOAD)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatDragAndDrop() + { + return (Pair.of(FORMAT, "dragAndDrop")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair formatButton() + { + return (Pair.of(FORMAT, "button")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthFull() + { + return (Pair.of(WIDTH, "full")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static Pair widthHalf() + { + return (Pair.of(WIDTH, "half")); + } + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java index b8aa82c5..74cc9db7 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/FieldAdornment.java @@ -177,7 +177,7 @@ public class FieldAdornment ** Fluent setter for values ** *******************************************************************************/ - public FieldAdornment withValue(Pair value) + public FieldAdornment withValue(Pair value) { return (withValue(value.getA(), value.getB())); } From 86f8e24d5fc24395d32bf19adc9d07d6083e861d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:57:10 -0600 Subject: [PATCH 38/84] CE-1955 Handle back better; put suggested mapping profile into process value under a dedicated key --- .../BulkInsertPrepareFileMappingStep.java | 19 ++++++++++++++++--- .../BulkInsertPrepareValueMappingStep.java | 6 ++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index f939df3a..80cb49d1 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -38,6 +38,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapp import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfile; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -59,9 +60,20 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); runBackendStepOutput.addValue("tableStructure", tableStructure); - @SuppressWarnings("unchecked") - List headerValues = (List) runBackendStepOutput.getValue("headerValues"); - buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + boolean needSuggestedMapping = true; + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + needSuggestedMapping = false; + + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + } + + if(needSuggestedMapping) + { + @SuppressWarnings("unchecked") + List headerValues = (List) runBackendStepOutput.getValue("headerValues"); + buildSuggestedMapping(headerValues, tableStructure, runBackendStepOutput); + } } @@ -74,6 +86,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep BulkLoadMappingSuggester bulkLoadMappingSuggester = new BulkLoadMappingSuggester(); BulkLoadProfile bulkLoadProfile = bulkLoadMappingSuggester.suggestBulkLoadMappingProfile(tableStructure, headerValues); runBackendStepOutput.addValue("bulkLoadProfile", bulkLoadProfile); + runBackendStepOutput.addValue("suggestedBulkLoadProfile", bulkLoadProfile); } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 19e696e5..d24856c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -53,6 +53,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; @@ -75,6 +76,11 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep { try { + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + } + ///////////////////////////////////////////////////////////// // prep the frontend for what field we're going to map now // ///////////////////////////////////////////////////////////// From 7cd3105ee6a1a2e52ed96215c4dba54764986b81 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:59:05 -0600 Subject: [PATCH 39/84] CE-1955 Add search-by labels - e.g., exact-matches on a single-field used as the PVS's label... definitely not perfect, but a passable first-version for bulk-load to do PVS mapping --- .../SearchPossibleValueSourceAction.java | 130 ++++++++++++++---- .../SearchPossibleValueSourceInput.java | 34 ++++- .../SearchPossibleValueSourceOutput.java | 32 +++++ .../SearchPossibleValueSourceActionTest.java | 71 ++++++++++ 4 files changed, 242 insertions(+), 25 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java index 9cb17a0c..97bcae1c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java @@ -26,8 +26,12 @@ import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; @@ -50,7 +54,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; -import org.apache.commons.lang.NotImplementedException; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -61,6 +65,9 @@ public class SearchPossibleValueSourceAction { private static final QLogger LOG = QLogger.getLogger(SearchPossibleValueSourceAction.class); + private static final Set warnedAboutUnexpectedValueField = Collections.synchronizedSet(new HashSet<>()); + private static final Set warnedAboutUnexpectedNoOfFieldsToSearchByLabel = Collections.synchronizedSet(new HashSet<>()); + private QPossibleValueTranslator possibleValueTranslator; @@ -110,6 +117,7 @@ public class SearchPossibleValueSourceAction List matchingIds = new ArrayList<>(); List inputIdsAsCorrectType = convertInputIdsToEnumIdType(possibleValueSource, input.getIdList()); + Set labels = null; for(QPossibleValue possibleValue : possibleValueSource.getEnumValues()) { @@ -122,12 +130,24 @@ public class SearchPossibleValueSourceAction match = true; } } + else if(input.getLabelList() != null) + { + if(labels == null) + { + labels = input.getLabelList().stream().filter(Objects::nonNull).map(l -> l.toLowerCase()).collect(Collectors.toSet()); + } + + if(labels.contains(possibleValue.getLabel().toLowerCase())) + { + match = true; + } + } else { if(StringUtils.hasContent(input.getSearchTerm())) { match = (Objects.equals(ValueUtils.getValueAsString(possibleValue.getId()).toLowerCase(), input.getSearchTerm().toLowerCase()) - || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); + || possibleValue.getLabel().toLowerCase().startsWith(input.getSearchTerm().toLowerCase())); } else { @@ -168,21 +188,37 @@ public class SearchPossibleValueSourceAction Object anIdFromTheEnum = possibleValueSource.getEnumValues().get(0).getId(); - if(anIdFromTheEnum instanceof Integer) + for(Serializable inputId : inputIdList) { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsInteger(id))); - } - else if(anIdFromTheEnum instanceof String) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsString(id))); - } - else if(anIdFromTheEnum instanceof Boolean) - { - inputIdList.forEach(id -> rs.add(ValueUtils.getValueAsBoolean(id))); - } - else - { - LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + Object properlyTypedId = null; + try + { + if(anIdFromTheEnum instanceof Integer) + { + properlyTypedId = ValueUtils.getValueAsInteger(inputId); + } + else if(anIdFromTheEnum instanceof String) + { + properlyTypedId = ValueUtils.getValueAsString(inputId); + } + else if(anIdFromTheEnum instanceof Boolean) + { + properlyTypedId = ValueUtils.getValueAsBoolean(inputId); + } + else + { + LOG.warn("Unexpected type [" + anIdFromTheEnum.getClass().getSimpleName() + "] for ids in enum: " + possibleValueSource.getName()); + } + } + catch(Exception e) + { + LOG.debug("Error converting possible value id to expected id type", e, logPair("value", inputId)); + } + + if (properlyTypedId != null) + { + rs.add(properlyTypedId); + } } return (rs); @@ -209,6 +245,53 @@ public class SearchPossibleValueSourceAction { queryFilter.addCriteria(new QFilterCriteria(table.getPrimaryKeyField(), QCriteriaOperator.IN, input.getIdList())); } + else if(input.getLabelList() != null) + { + List fieldNames = new ArrayList<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // the 'value fields' will either be 'id' or 'label' (which means, use the fields from the tableMetaData's label fields) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + for(String valueField : possibleValueSource.getValueFields()) + { + if("id".equals(valueField)) + { + fieldNames.add(table.getPrimaryKeyField()); + } + else if("label".equals(valueField)) + { + if(table.getRecordLabelFields() != null) + { + fieldNames.addAll(table.getRecordLabelFields()); + } + } + else + { + String message = "Unexpected valueField defined in possibleValueSource when searching possibleValueSource by label (required: 'id' or 'label')"; + if(!warnedAboutUnexpectedValueField.contains(possibleValueSource.getName())) + { + LOG.warn(message, logPair("valueField", valueField), logPair("possibleValueSource", possibleValueSource.getName())); + warnedAboutUnexpectedValueField.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } + + if(fieldNames.size() == 1) + { + queryFilter.addCriteria(new QFilterCriteria(fieldNames.get(0), QCriteriaOperator.IN, input.getLabelList())); + } + else + { + String message = "Unexpected number of fields found for searching possibleValueSource by label (required: 1, found: " + fieldNames.size() + ")"; + if(!warnedAboutUnexpectedNoOfFieldsToSearchByLabel.contains(possibleValueSource.getName())) + { + LOG.warn(message); + warnedAboutUnexpectedNoOfFieldsToSearchByLabel.add(possibleValueSource.getName()); + } + output.setWarning(message); + } + } else { String searchTerm = input.getSearchTerm(); @@ -269,8 +352,8 @@ public class SearchPossibleValueSourceAction queryFilter = input.getDefaultQueryFilter(); } - // todo - skip & limit as params - queryFilter.setLimit(250); + queryFilter.setLimit(input.getLimit()); + queryFilter.setSkip(input.getSkip()); queryFilter.setOrderBys(possibleValueSource.getOrderByFields()); @@ -288,7 +371,7 @@ public class SearchPossibleValueSourceAction fieldName = table.getPrimaryKeyField(); } - List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); + List ids = queryOutput.getRecords().stream().map(r -> r.getValue(fieldName)).toList(); List> qPossibleValues = possibleValueTranslator.buildTranslatedPossibleValueList(possibleValueSource, ids); output.setResults(qPossibleValues); @@ -301,7 +384,7 @@ public class SearchPossibleValueSourceAction ** *******************************************************************************/ @SuppressWarnings({ "rawtypes", "unchecked" }) - private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) + private SearchPossibleValueSourceOutput searchPossibleValueCustom(SearchPossibleValueSourceInput input, QPossibleValueSource possibleValueSource) throws QException { try { @@ -314,11 +397,10 @@ public class SearchPossibleValueSourceAction } catch(Exception e) { - // LOG.warn("Error sending [" + value + "] for field [" + field + "] through custom code for PVS [" + field.getPossibleValueSourceName() + "]", e); + String message = "Error sending searching custom possible value source [" + input.getPossibleValueSourceName() + "]"; + LOG.warn(message, e); + throw (new QException(message)); } - - throw new NotImplementedException("Not impleemnted"); - // return (null); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java index 65b7c2bc..8976a176 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceInput.java @@ -38,9 +38,10 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen private QQueryFilter defaultQueryFilter; private String searchTerm; private List idList; + private List labelList; private Integer skip = 0; - private Integer limit = 100; + private Integer limit = 250; @@ -281,4 +282,35 @@ public class SearchPossibleValueSourceInput extends AbstractActionInput implemen this.limit = limit; return (this); } + + + /******************************************************************************* + ** Getter for labelList + *******************************************************************************/ + public List getLabelList() + { + return (this.labelList); + } + + + + /******************************************************************************* + ** Setter for labelList + *******************************************************************************/ + public void setLabelList(List labelList) + { + this.labelList = labelList; + } + + + + /******************************************************************************* + ** Fluent setter for labelList + *******************************************************************************/ + public SearchPossibleValueSourceInput withLabelList(List labelList) + { + this.labelList = labelList; + return (this); + } + } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java index e7186614..e60ed262 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/values/SearchPossibleValueSourceOutput.java @@ -35,6 +35,7 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput { private List> results = new ArrayList<>(); + private String warning; /******************************************************************************* @@ -88,4 +89,35 @@ public class SearchPossibleValueSourceOutput extends AbstractActionOutput return (this); } + + /******************************************************************************* + ** Getter for warning + *******************************************************************************/ + public String getWarning() + { + return (this.warning); + } + + + + /******************************************************************************* + ** Setter for warning + *******************************************************************************/ + public void setWarning(String warning) + { + this.warning = warning; + } + + + + /******************************************************************************* + ** Fluent setter for warning + *******************************************************************************/ + public SearchPossibleValueSourceOutput withWarning(String warning) + { + this.warning = warning; + return (this); + } + + } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java index ee2bb104..d71f6335 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceActionTest.java @@ -217,6 +217,63 @@ class SearchPossibleValueSourceActionTest extends BaseTest } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_tableByLabels() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Square", "Circle"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("Square")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(3) && pv.getLabel().equals("Circle")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("notFound"), TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE); + assertEquals(0, output.getResults().size()); + } + } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSearchPvsAction_enumByLabel() throws QException + { + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("IL", "MO", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("Il", "mo", "XX"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(2, output.getResults().size()); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(1) && pv.getLabel().equals("IL")); + assertThat(output.getResults()).anyMatch(pv -> pv.getId().equals(2) && pv.getLabel().equals("MO")); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of(), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + + { + SearchPossibleValueSourceOutput output = getSearchPossibleValueSourceOutputByLabels(List.of("not-found"), TestUtils.POSSIBLE_VALUE_SOURCE_STATE); + assertEquals(0, output.getResults().size()); + } + } + + /******************************************************************************* ** @@ -414,4 +471,18 @@ class SearchPossibleValueSourceActionTest extends BaseTest return output; } + + + /******************************************************************************* + ** + *******************************************************************************/ + private SearchPossibleValueSourceOutput getSearchPossibleValueSourceOutputByLabels(List labels, String possibleValueSourceName) throws QException + { + SearchPossibleValueSourceInput input = new SearchPossibleValueSourceInput(); + input.setLabelList(labels); + input.setPossibleValueSourceName(possibleValueSourceName); + SearchPossibleValueSourceOutput output = new SearchPossibleValueSourceAction().execute(input); + return output; + } + } \ No newline at end of file From a7247b59700c950141042674139d911ef36f8fb4 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 08:59:48 -0600 Subject: [PATCH 40/84] CE-1955 Add method resetValidationFields - to help processes that go 'back' --- .../StreamedETLWithFrontendProcess.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index 8d3e8287..d05bb12c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -193,6 +194,17 @@ public class StreamedETLWithFrontendProcess } + /*************************************************************************** + ** useful for a process step to call upon 'back' + ***************************************************************************/ + public static void resetValidationFields(RunBackendStepInput runBackendStepInput) + { + runBackendStepInput.addValue(FIELD_DO_FULL_VALIDATION, null); + runBackendStepInput.addValue(FIELD_VALIDATION_SUMMARY, null); + runBackendStepInput.addValue(FIELD_PROCESS_SUMMARY, null); + } + + /******************************************************************************* ** From 11db820196db273ccf4d8d7406453a2d5ad5057b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:03:02 -0600 Subject: [PATCH 41/84] CE-1955 Bulk insert updates: Add prepareFileUploadStep; make theFile field use drag&drop adornment --- .../core/instances/QInstanceEnricher.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 87be37bc..da9c9a91 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -42,6 +42,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType.FileUploadAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueBehavior; import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -77,6 +78,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit.BulkEd import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertExtractStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertLoadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileMappingStep; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareFileUploadStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertPrepareValueMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveFileMappingStep; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkInsertReceiveValueMappingStep; @@ -881,14 +883,20 @@ public class QInstanceEnricher .map(QFieldMetaData::getLabel) .collect(Collectors.joining(", ")); + QBackendStepMetaData prepareFileUploadStep = new QBackendStepMetaData() + .withName("prepareFileUpload") + .withCode(new QCodeReference(BulkInsertPrepareFileUploadStep.class)); + QFrontendStepMetaData uploadScreen = new QFrontendStepMetaData() .withName("upload") .withLabel("Upload File") - .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB).withLabel(table.getLabel() + " File").withIsRequired(true)) - .withComponent(new QFrontendComponentMetaData() - .withType(QComponentType.HELP_TEXT) - .withValue("previewText", "file upload instructions") - .withValue("text", "Upload a CSV or Excel (.xlsx) file with the following columns:\n" + fieldsForHelpText)) + .withFormField(new QFieldMetaData("theFile", QFieldType.BLOB) + .withFieldAdornment(FileUploadAdornment.newFieldAdornment() + .withValue(FileUploadAdornment.formatDragAndDrop()) + .withValue(FileUploadAdornment.widthFull())) + .withLabel(table.getLabel() + " File") + .withIsRequired(true)) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.HTML)) .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)); QBackendStepMetaData prepareFileMappingStep = new QBackendStepMetaData() @@ -920,6 +928,7 @@ public class QInstanceEnricher .withCode(new QCodeReference(BulkInsertReceiveValueMappingStep.class)); int i = 0; + process.addStep(i++, prepareFileUploadStep); process.addStep(i++, uploadScreen); process.addStep(i++, prepareFileMappingStep); From 21069e231086659bf22f6fb916d1942404bba1b9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:10:00 -0600 Subject: [PATCH 42/84] CE-1955 Checkstyle! --- .../bulk/insert/BulkInsertPrepareFileUploadStep.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index c021dcb1..5fe61f41 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -161,10 +161,10 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep if(listFieldsInHelpText) { html.append(""" - Templates: ${tableLabel} - Flat.csv - | ${tableLabel} - Tall.csv - | ${tableLabel} - Wide.csv - """); + Templates: ${tableLabel} - Flat.csv + | ${tableLabel} - Tall.csv + | ${tableLabel} - Wide.csv + """); } else { From 7e3592628a4bcf273c86e13c76a996f36e8024b3 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:27:35 -0600 Subject: [PATCH 43/84] CE-1955 Don't put empty-string values into records (in setValueOrDefault) - in general, we might get an empty-string from a file, but let's treat it like a non-value, null. --- .../bulk/insert/mapping/RowsToRecordInterface.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java index 740781c2..6d10b0bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -92,7 +92,7 @@ public interface RowsToRecordInterface value = mapping.getFieldNameToDefaultValueMap().get(fullFieldName); } - if(value != null) + if(value != null && !"".equals(value)) { record.setValue(fieldName, value); } From 2bf12158be0e8b7691495fbf788ddee46774c912 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:27:50 -0600 Subject: [PATCH 44/84] CE-1955 Fix to set tableName before preUpload step --- .../implementations/bulk/insert/BulkInsertFullProcessTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java index b819760e..2ab31159 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertFullProcessTest.java @@ -132,6 +132,7 @@ class BulkInsertFullProcessTest extends BaseTest ///////////////////////////////////////////////////////// RunProcessInput runProcessInput = new RunProcessInput(); runProcessInput.setProcessName(processName); + runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput); String processUUID = runProcessOutput.getProcessUUID(); assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload"); @@ -155,7 +156,6 @@ class BulkInsertFullProcessTest extends BaseTest ////////////////////////// runProcessInput.setProcessUUID(processUUID); runProcessInput.setStartAfterStep("upload"); - runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY); runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput))); runProcessOutput = new RunProcessAction().execute(runProcessInput); assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues")); From 21aeac2def4debbf4440f8b851849b2cf173205d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 09:51:44 -0600 Subject: [PATCH 45/84] CE-1955 Switch fieldMetaData to use a type from in here for FieldAdornment, to include some better docs, but also to exclude new FILE_UPLOAD adornment type enum value --- .../responses/components/FieldAdornment.java | 115 ++++++++++++++++++ .../responses/components/FieldMetaData.java | 18 ++- .../main/resources/openapi/v1/openapi.yaml | 51 ++++---- 3 files changed, 151 insertions(+), 33 deletions(-) create mode 100644 qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java new file mode 100644 index 00000000..2f59ee63 --- /dev/null +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldAdornment.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; + + +import java.io.Serializable; +import java.util.EnumSet; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIEnumSubSet; +import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIExclude; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FieldAdornment implements ToSchema +{ + @OpenAPIExclude() + private com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment(com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment wrapped) + { + this.wrapped = wrapped; + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public FieldAdornment() + { + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static class FieldAdornmentSubSet implements OpenAPIEnumSubSet.EnumSubSet + { + private static EnumSet subSet = null; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EnumSet getSubSet() + { + if(subSet == null) + { + EnumSet subSet = EnumSet.allOf(AdornmentType.class); + subSet.remove(AdornmentType.FILE_UPLOAD); // todo - remove for next version! + FieldAdornmentSubSet.subSet = subSet; + } + + return (subSet); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Type of this adornment") + @OpenAPIEnumSubSet(FieldAdornmentSubSet.class) + public AdornmentType getType() + { + return (this.wrapped == null || this.wrapped.getType() == null ? null : this.wrapped.getType()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @OpenAPIDescription("Values associated with this adornment. Keys and the meanings of their values will differ by type.") + public Map getValues() + { + return (this.wrapped.getValues()); + } + +} diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java index 5c4505ac..b8a75d5f 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/v1/responses/components/FieldMetaData.java @@ -23,7 +23,6 @@ package com.kingsrook.qqq.middleware.javalin.specs.v1.responses.components; import java.util.List; -import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.middleware.javalin.schemabuilder.ToSchema; import com.kingsrook.qqq.middleware.javalin.schemabuilder.annotations.OpenAPIDescription; @@ -61,6 +60,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -82,6 +82,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -92,6 +93,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -102,6 +104,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -112,6 +115,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -122,6 +126,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -143,6 +148,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -153,6 +159,7 @@ public class FieldMetaData implements ToSchema } + /*************************************************************************** ** ***************************************************************************/ @@ -166,6 +173,8 @@ public class FieldMetaData implements ToSchema // todo - inline PVS + + /*************************************************************************** ** ***************************************************************************/ @@ -177,14 +186,17 @@ public class FieldMetaData implements ToSchema // todo behaviors? + + + /*************************************************************************** ** ***************************************************************************/ @OpenAPIDescription("Special UI dressings to add to the field.") - @OpenAPIListItems(value = FieldAdornment.class) // todo! + @OpenAPIListItems(value = FieldAdornment.class, useRef = true) public List getAdornments() { - return (this.wrapped.getAdornments()); + return (this.wrapped.getAdornments() == null ? null : this.wrapped.getAdornments().stream().map(a -> new FieldAdornment(a)).toList()); } // todo help content diff --git a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml index beda0fc6..dd802596 100644 --- a/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml +++ b/qqq-middleware-javalin/src/main/resources/openapi/v1/openapi.yaml @@ -130,26 +130,31 @@ components: description: "Description of the error" type: "string" type: "object" + FieldAdornment: + properties: + type: + description: "Type of this adornment" + enum: + - "LINK" + - "CHIP" + - "SIZE" + - "CODE_EDITOR" + - "RENDER_HTML" + - "REVEAL" + - "FILE_DOWNLOAD" + - "ERROR" + type: "string" + values: + description: "Values associated with this adornment. Keys and the meanings\ + \ of their values will differ by type." + type: "object" + type: "object" FieldMetaData: properties: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." @@ -973,21 +978,7 @@ components: adornments: description: "Special UI dressings to add to the field." items: - properties: - type: - enum: - - "LINK" - - "CHIP" - - "SIZE" - - "CODE_EDITOR" - - "RENDER_HTML" - - "REVEAL" - - "FILE_DOWNLOAD" - - "ERROR" - type: "string" - values: - type: "object" - type: "object" + $ref: "#/components/schemas/FieldAdornment" type: "array" defaultValue: description: "Default value to use in this field." From 1911e27cc0eade0b775d97a379166fec02b8ffe0 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:38:28 -0600 Subject: [PATCH 46/84] CE-1955 clear out uploaded file if user goes back to this step --- .../bulk/insert/BulkInsertPrepareFileUploadStep.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index 5fe61f41..26ebc729 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -52,6 +52,14 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep @Override public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if user has come back here, clear out file (else the storageInput object that it is comes to the frontend, which isn't what we want!) // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(runBackendStepOutput.getProcessState().getIsStepBack()) + { + runBackendStepOutput.addValue("theFile", null); + } + String tableName = runBackendStepInput.getValueString("tableName"); QTableMetaData table = QContext.getQInstance().getTable(tableName); BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(tableName); From b5eae02fa4f96479c3a46d060dc1636ea868f751 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:39:18 -0600 Subject: [PATCH 47/84] CE-1955 populate association structures for record preview validation screen based on table structure associations, not actual mapping (e.g., so lines always appear on orders, even if not being used - to make that clear to user that they aren't being used) --- .../bulk/insert/BulkInsertTransformStep.java | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index ad531967..bd1fb40c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -58,7 +58,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; import com.kingsrook.qqq.backend.core.model.statusmessages.QErrorMessage; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.AbstractBulkLoadRollableValueError; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; -import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadTableStructureBuilder; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadTableStructure; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; @@ -94,6 +95,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep private static final int EXAMPLE_ROW_LIMIT = 10; + /******************************************************************************* ** extension of ProcessSummaryLine for lines where a UniqueKey was violated, ** where we'll collect a sample (or maybe all) of the values that broke the UK. @@ -139,16 +141,20 @@ public class BulkInsertTransformStep extends AbstractTransformStep ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// runBackendStepOutput.addValue("formatPreviewRecordUsingTableLayout", table.getName()); - BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); - if(bulkInsertMapping != null) + BulkLoadTableStructure tableStructure = BulkLoadTableStructureBuilder.buildTableStructure(table.getName()); + if(CollectionUtils.nullSafeHasContents(tableStructure.getAssociations())) { ArrayList previewRecordAssociatedTableNames = new ArrayList<>(); ArrayList previewRecordAssociatedWidgetNames = new ArrayList<>(); ArrayList previewRecordAssociationNames = new ArrayList<>(); - for(String mappedAssociation : bulkInsertMapping.getMappedAssociations()) + //////////////////////////////////////////////////////////// + // note - not recursively processing associations here... // + //////////////////////////////////////////////////////////// + for(BulkLoadTableStructure associatedStructure : tableStructure.getAssociations()) { - Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(mappedAssociation)).findFirst(); + String associationName = associatedStructure.getAssociationPath(); + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationName)).findFirst(); if(association.isPresent()) { for(QFieldSection section : table.getSections()) @@ -518,17 +524,17 @@ public class BulkInsertTransformStep extends AbstractTransformStep String message = entry.getKey(); if(errorToExampleRowValueMap.containsKey(message)) { - ProcessSummaryLine line = entry.getValue(); - List rowValues = errorToExampleRowValueMap.get(message); - String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; + ProcessSummaryLine line = entry.getValue(); + List rowValues = errorToExampleRowValueMap.get(message); + String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); } else if(errorToExampleRowsMap.containsKey(message)) { - ProcessSummaryLine line = entry.getValue(); - List rowDescriptions = errorToExampleRowsMap.get(message); - String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; + ProcessSummaryLine line = entry.getValue(); + List rowDescriptions = errorToExampleRowsMap.get(message); + String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); } From 8157510c04d5a8eb83e037e1f4050273441a28bf Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:39:38 -0600 Subject: [PATCH 48/84] CE-1955 Add fields to bulkLoad fileMapping screen, for helpContent to be associated with --- .../qqq/backend/core/instances/QInstanceEnricher.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index da9c9a91..1d88cb7e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -906,8 +906,10 @@ public class QInstanceEnricher QFrontendStepMetaData fileMappingScreen = new QFrontendStepMetaData() .withName("fileMapping") .withLabel("File Mapping") - .withBackStepName("upload") - .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)); + .withBackStepName("prepareFileUpload") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.BULK_LOAD_FILE_MAPPING_FORM)) + .withFormField(new QFieldMetaData("hasHeaderRow", QFieldType.BOOLEAN)) + .withFormField(new QFieldMetaData("layout", QFieldType.STRING)); // is actually PVS, but, this field is only added to help support helpContent, so :shrug: QBackendStepMetaData receiveFileMappingStep = new QBackendStepMetaData() .withName("receiveFileMapping") From 7bab11ea7ed875cb660a21b94948b01c6c05e9f9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:41:12 -0600 Subject: [PATCH 49/84] CE-1955 Add support for wildcard (at start of) process names - e.g., to support bulkLoad etc processes; update to apply all helpContent to the qInstance that came in as a parameter, rather than the one in context (to work correctly for hot-swaps). --- .../QInstanceHelpContentManager.java | 50 +++++++++++++------ .../QInstanceHelpContentManagerTest.java | 47 ++++++++++++++++- 2 files changed, 81 insertions(+), 16 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java index 02376ff3..e46fb80a 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManager.java @@ -30,7 +30,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; -import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput; @@ -111,7 +110,7 @@ public class QInstanceHelpContentManager } else { - LOG.info("Discarding help content with key that does not contain name:value format", logPair("key", key), logPair("id", record.getValue("id"))); + LOG.info("Discarding help content with key-part that does not contain name:value format", logPair("key", key), logPair("part", part), logPair("id", record.getValue("id"))); } } @@ -150,19 +149,19 @@ public class QInstanceHelpContentManager /////////////////////////////////////////////////////////////////////////////////// if(StringUtils.hasContent(tableName)) { - processHelpContentForTable(key, tableName, sectionName, fieldName, slotName, roles, helpContent); + processHelpContentForTable(qInstance, key, tableName, sectionName, fieldName, slotName, roles, helpContent); } else if(StringUtils.hasContent(processName)) { - processHelpContentForProcess(key, processName, fieldName, stepName, roles, helpContent); + processHelpContentForProcess(qInstance, key, processName, fieldName, stepName, roles, helpContent); } else if(StringUtils.hasContent(widgetName)) { - processHelpContentForWidget(key, widgetName, slotName, roles, helpContent); + processHelpContentForWidget(qInstance, key, widgetName, slotName, roles, helpContent); } else if(nameValuePairs.containsKey("instanceLevel")) { - processHelpContentForInstance(key, slotName, roles, helpContent); + processHelpContentForInstance(qInstance, key, slotName, roles, helpContent); } } catch(Exception e) @@ -176,9 +175,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForTable(String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForTable(QInstance qInstance, String key, String tableName, String sectionName, String fieldName, String slotName, Set roles, QHelpContent helpContent) { - QTableMetaData table = QContext.getQInstance().getTable(tableName); + QTableMetaData table = qInstance.getTable(tableName); if(table == null) { LOG.info("Unrecognized table in help content", logPair("key", key)); @@ -246,9 +245,30 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForProcess(String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) + private static void processHelpContentForProcess(QInstance qInstance, String key, String processName, String fieldName, String stepName, Set roles, QHelpContent helpContent) { - QProcessMetaData process = QContext.getQInstance().getProcess(processName); + if(processName.startsWith("*") && processName.length() > 1) + { + boolean anyMatched = false; + String subName = processName.substring(1); + for(QProcessMetaData process : qInstance.getProcesses().values()) + { + if(process.getName().endsWith(subName)) + { + anyMatched = true; + processHelpContentForProcess(qInstance, key, process.getName(), fieldName, stepName, roles, helpContent); + } + } + + if(!anyMatched) + { + LOG.info("Wildcard process name did not match any processes in help content", logPair("key", key)); + } + + return; + } + + QProcessMetaData process = qInstance.getProcess(processName); if(process == null) { LOG.info("Unrecognized process in help content", logPair("key", key)); @@ -306,9 +326,9 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForWidget(String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForWidget(QInstance qInstance, String key, String widgetName, String slotName, Set roles, QHelpContent helpContent) { - QWidgetMetaDataInterface widget = QContext.getQInstance().getWidget(widgetName); + QWidgetMetaDataInterface widget = qInstance.getWidget(widgetName); if(!StringUtils.hasContent(slotName)) { LOG.info("Missing slot name in help content", logPair("key", key)); @@ -335,7 +355,7 @@ public class QInstanceHelpContentManager /******************************************************************************* ** *******************************************************************************/ - private static void processHelpContentForInstance(String key, String slotName, Set roles, QHelpContent helpContent) + private static void processHelpContentForInstance(QInstance qInstance, String key, String slotName, Set roles, QHelpContent helpContent) { if(!StringUtils.hasContent(slotName)) { @@ -345,11 +365,11 @@ public class QInstanceHelpContentManager { if(helpContent != null) { - QContext.getQInstance().withHelpContent(slotName, helpContent); + qInstance.withHelpContent(slotName, helpContent); } else { - QContext.getQInstance().removeHelpContent(slotName, roles); + qInstance.removeHelpContent(slotName, roles); } } } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java index 51af9eaa..3511326f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceHelpContentManagerTest.java @@ -47,6 +47,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaDataIn import com.kingsrook.qqq.backend.core.model.metadata.help.HelpRole; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpRole; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; import org.junit.jupiter.api.Test; @@ -292,6 +294,49 @@ class QInstanceHelpContentManagerTest extends BaseTest + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testWildcardProcessField() throws QException + { + ///////////////////////////////////// + // get the instance from base test // + ///////////////////////////////////// + QInstance qInstance = QContext.getQInstance(); + new HelpContentMetaDataProvider().defineAll(qInstance, TestUtils.MEMORY_BACKEND_NAME, null); + + HelpContent recordEntity = new HelpContent() + .withId(1) + .withKey("process:*.bulkInsert;step:upload") + .withContent("v1") + .withRole(HelpContentRole.PROCESS_SCREEN.getId()); + new InsertAction().execute(new InsertInput(HelpContent.TABLE_NAME).withRecordEntity(recordEntity)); + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // now - post-insert customizer should have automatically added help content to the instance - to all bulkInsert processes // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + int hitCount = 0; + for(QTableMetaData table : qInstance.getTables().values()) + { + QProcessMetaData process = qInstance.getProcess(table.getName() + ".bulkInsert"); + if(process == null) + { + return; + } + + List helpContents = process.getFrontendStep("upload").getHelpContents(); + assertEquals(1, helpContents.size()); + assertEquals("v1", helpContents.get(0).getContent()); + assertEquals(Set.of(QHelpRole.PROCESS_SCREEN), helpContents.get(0).getRoles()); + hitCount++; + } + + assertThat(hitCount).isGreaterThanOrEqualTo(3); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -411,7 +456,7 @@ class QInstanceHelpContentManagerTest extends BaseTest QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply("foo;bar:baz")); assertThat(collectingLogger.getCollectedMessages()).hasSize(1); - assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key that does not contain name:value format"); + assertThat(collectingLogger.getCollectedMessages().get(0).getMessage()).contains("Discarding help content with key-part that does not contain name:value format"); collectingLogger.clear(); QInstanceHelpContentManager.processHelpContentRecord(qInstance, helpContentCreator.apply(null)); From 8d37ce3c5426d56742bac34cb7fa6b001167af83 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:43:10 -0600 Subject: [PATCH 50/84] CE-1955 add checks for material-dashboard resources before trying to blindly serve them; add field for QJavalinMetaData; --- .../javalin/QApplicationJavalinServer.java | 45 ++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java index d18a6a03..3dfd3507 100644 --- a/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java +++ b/qqq-middleware-javalin/src/main/java/com/kingsrook/qqq/middleware/javalin/QApplicationJavalinServer.java @@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.utils.ClassPathUtils; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; +import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; import com.kingsrook.qqq.middleware.javalin.specs.AbstractMiddlewareVersion; import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; import io.javalin.Javalin; @@ -69,6 +70,7 @@ public class QApplicationJavalinServer private List middlewareVersionList = List.of(new MiddlewareVersionV1()); private List additionalRouteProviders = null; private Consumer javalinConfigurationCustomizer = null; + private QJavalinMetaData javalinMetaData = null; private long lastQInstanceHotSwapMillis; private long millisBetweenHotSwaps = 2500; @@ -100,6 +102,11 @@ public class QApplicationJavalinServer { if(serveFrontendMaterialDashboard) { + if(getClass().getResource("/material-dashboard/index.html") == null) + { + LOG.warn("/material-dashboard/index.html resource was not found. This might happen if you're using a local (e.g., within-IDE) snapshot version... Try updating pom.xml to reference a released version of qfmd?"); + } + //////////////////////////////////////////////////////////////////////////////////////// // If you have any assets to add to the web server (e.g., logos, icons) place them at // // src/main/resources/material-dashboard-overlay (or a directory of your choice // @@ -108,7 +115,10 @@ public class QApplicationJavalinServer // material-dashboard directory, so in case the same file exists in both (e.g., // // favicon.png), the app-specific one will be used. // //////////////////////////////////////////////////////////////////////////////////////// - config.staticFiles.add("/material-dashboard-overlay"); + if(getClass().getResource("/material-dashboard-overlay") != null) + { + config.staticFiles.add("/material-dashboard-overlay"); + } ///////////////////////////////////////////////////////////////////// // tell javalin where to find material-dashboard static web assets // @@ -128,7 +138,7 @@ public class QApplicationJavalinServer { try { - QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); + QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance, javalinMetaData); config.router.apiBuilder(qJavalinImplementation.getRoutes()); } catch(QInstanceValidationException e) @@ -526,4 +536,35 @@ public class QApplicationJavalinServer return (this); } + + /******************************************************************************* + ** Getter for javalinMetaData + *******************************************************************************/ + public QJavalinMetaData getJavalinMetaData() + { + return (this.javalinMetaData); + } + + + + /******************************************************************************* + ** Setter for javalinMetaData + *******************************************************************************/ + public void setJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + } + + + + /******************************************************************************* + ** Fluent setter for javalinMetaData + *******************************************************************************/ + public QApplicationJavalinServer withJavalinMetaData(QJavalinMetaData javalinMetaData) + { + this.javalinMetaData = javalinMetaData; + return (this); + } + + } From 76d7a8a858399d7ad196d78d45efa29970e35b79 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:43:33 -0600 Subject: [PATCH 51/84] CE-1955 Initial checkin --- .../selenium/BaseSampleSeleniumTest.java | 85 +++++++++++++++++ .../selenium/BulkLoadSeleniumTest.java | 95 +++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java create mode 100644 qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java new file mode 100644 index 00000000..a0a904fc --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + */ + +package com.kingsrook.sampleapp.selenium; + + +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.frontend.materialdashboard.selenium.lib.QBaseSeleniumTest; +import com.kingsrook.sampleapp.SampleJavalinServer; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BaseSampleSeleniumTest extends QBaseSeleniumTest +{ + private static final QLogger LOG = QLogger.getLogger(BaseSampleSeleniumTest.class); + + public static final Integer DEFAULT_WAIT_SECONDS = 10; + + private int port = 8011; + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + @BeforeEach + public void beforeEach() + { + super.beforeEach(); + qSeleniumLib.withBaseUrl("http://localhost:" + port); + qSeleniumLib.withWaitSeconds(DEFAULT_WAIT_SECONDS); + + new SampleJavalinServer().startJavalinServer(port); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + protected boolean useInternalJavalin() + { + return (false); + } + + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clickLeftNavMenuItem(String text) + { + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void clickLeftNavMenuItemThenSubItem(String text, String subItemText) + { + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiListItem-root", text).click(); + qSeleniumLib.waitForSelectorContaining(".MuiDrawer-paperAnchorLeft .MuiCollapse-vertical.MuiCollapse-entered .MuiListItem-root", subItemText).click(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void goToPathAndWaitForSelectorContaining(String path, String selector, String text) + { + driver.get(qSeleniumLib.getBaseUrl() + path); + qSeleniumLib.waitForSelectorContaining(selector, text); + } + +} + diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java new file mode 100644 index 00000000..02f2785d --- /dev/null +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.sampleapp.selenium; + + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkLoadSeleniumTest extends BaseSampleSeleniumTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testSimple() throws IOException + { + String email = "jtkirk@starfleet.com"; + String tablePath = "/peopleApp/greetingsApp/person"; + + //////////////////////////////////// + // write a file to be bulk-loaded // + //////////////////////////////////// + String path = "/tmp/" + UUID.randomUUID() + ".csv"; + String csv = String.format(""" + email,firstName,lastName + %s,James T.,Kirk + """, email); + FileUtils.writeStringToFile(new File(path), csv, StandardCharsets.UTF_8); + + goToPathAndWaitForSelectorContaining(tablePath + "/person.bulkInsert", ".MuiTypography-h5", "Person Bulk Insert: Upload File"); + + ////////////////////////////// + // complete the upload form // + ////////////////////////////// + qSeleniumLib.waitForSelector("input[type=file]").sendKeys(path); + qSeleniumLib.waitForSelectorContaining("button", "next").click(); + + ///////////////////////////////////////// + // proceed through file-mapping screen // + ///////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("button", "next").click(); + + ////////////////////////////////////////////////// + // confirm data on preview screen, then proceed // + ////////////////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", email); + qSeleniumLib.waitForSelectorContaining("form#review .MuiTypography-body2 div", "Preview 1 of 1"); + qSeleniumLib.waitForSelectorContaining("button", "arrow_forward").click(); // to avoid the record-preview 'next' button + + /////////////////////////////////////// + // proceed through validation screen // + /////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining("button", "submit").click(); + + //////////////////////////////////////// + // confirm result screen and close it // + //////////////////////////////////////// + qSeleniumLib.waitForSelectorContaining(".MuiListItemText-root", "1 Person record was inserted"); + qSeleniumLib.waitForSelectorContaining("button", "close").click(); + + //////////////////////////////////////////// + // go to the order that was just inserted // + // bonus - also test record-view-by-key // + //////////////////////////////////////////// + goToPathAndWaitForSelectorContaining(tablePath + "/key?email=" + email, "h5", "Viewing Person"); + } + +} From f7bd049b8140dea67a2d3957556675c787301e30 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:44:29 -0600 Subject: [PATCH 52/84] CE-1955 Update qfmd to feature-bulk-upload-v2; add test-dep for qfmd; add slf4j simple and selenium and webdriver. --- qqq-sample-project/pom.xml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/qqq-sample-project/pom.xml b/qqq-sample-project/pom.xml index 2ff7aee7..4db98cd3 100644 --- a/qqq-sample-project/pom.xml +++ b/qqq-sample-project/pom.xml @@ -68,7 +68,14 @@ com.kingsrook.qqq qqq-frontend-material-dashboard - 0.20.0 + feature-bulk-upload-v2-20241203.163149-2 + + + com.kingsrook.qqq + qqq-frontend-material-dashboard + tests + feature-bulk-upload-v2-20241203.163149-2 + test com.h2database @@ -77,7 +84,27 @@ - + + + + org.slf4j + slf4j-simple + 2.0.6 + + + + + org.seleniumhq.selenium + selenium-java + 4.19.1 + test + + + io.github.bonigarcia + webdrivermanager + 5.6.2 + test + From 131da68a38325f50f251d30b966bb5cdc10ba577 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 20:46:37 -0600 Subject: [PATCH 53/84] CE-1955 Update to use new AbstractQQQApplication and QApplicationJavalinServer --- .../sampleapp/SampleJavalinServer.java | 58 +++++++------------ .../metadata/SampleMetaDataProvider.java | 14 ++++- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java index e78227fd..ead70bfc 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/SampleJavalinServer.java @@ -23,11 +23,9 @@ package com.kingsrook.sampleapp; import com.kingsrook.qqq.backend.core.logging.QLogger; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.javalin.QJavalinImplementation; import com.kingsrook.qqq.backend.javalin.QJavalinMetaData; +import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer; import com.kingsrook.sampleapp.metadata.SampleMetaDataProvider; -import io.javalin.Javalin; /******************************************************************************* @@ -39,9 +37,7 @@ public class SampleJavalinServer private static final int PORT = 8000; - private QInstance qInstance; - - private Javalin javalinService; + private QApplicationJavalinServer qApplicationJavalinServer; @@ -59,42 +55,28 @@ public class SampleJavalinServer ** *******************************************************************************/ public void startJavalinServer() + { + startJavalinServer(PORT); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public void startJavalinServer(int port) { try { - qInstance = SampleMetaDataProvider.defineInstance(); - SampleMetaDataProvider.primeTestDatabase("prime-test-database.sql"); - QJavalinImplementation qJavalinImplementation = new QJavalinImplementation(qInstance); - qJavalinImplementation.setJavalinMetaData(new QJavalinMetaData() + qApplicationJavalinServer = new QApplicationJavalinServer(new SampleMetaDataProvider()); + qApplicationJavalinServer.setServeFrontendMaterialDashboard(true); + qApplicationJavalinServer.setServeLegacyUnversionedMiddlewareAPI(true); + qApplicationJavalinServer.setPort(port); + qApplicationJavalinServer.setJavalinMetaData(new QJavalinMetaData() .withUploadedFileArchiveTableName(SampleMetaDataProvider.UPLOAD_FILE_ARCHIVE_TABLE_NAME)); - - javalinService = Javalin.create(config -> - { - config.router.apiBuilder(qJavalinImplementation.getRoutes()); - // todo - not all? - config.bundledPlugins.enableCors(cors -> cors.addRule(corsRule -> corsRule.anyHost())); - }).start(PORT); - - ///////////////////////////////////////////////////////////////// - // set the server to hot-swap the q instance before all routes // - ///////////////////////////////////////////////////////////////// - QJavalinImplementation.setQInstanceHotSwapSupplier(() -> - { - try - { - return (SampleMetaDataProvider.defineInstance()); - } - catch(Exception e) - { - LOG.warn("Error hot-swapping meta data", e); - return (null); - } - }); - javalinService.before(QJavalinImplementation::hotSwapQInstance); - - javalinService.after(ctx -> ctx.res().setHeader("Access-Control-Allow-Origin", "http://localhost:3000")); + qApplicationJavalinServer.start(); } catch(Exception e) { @@ -109,9 +91,9 @@ public class SampleJavalinServer *******************************************************************************/ public void stopJavalinServer() { - if(javalinService != null) + if(qApplicationJavalinServer != null) { - javalinService.stop(); + qApplicationJavalinServer.stop(); } } } diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 77b05601..062d8721 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.QuickSightChartR import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QValueException; +import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -94,7 +95,7 @@ import org.apache.commons.io.IOUtils; /******************************************************************************* ** *******************************************************************************/ -public class SampleMetaDataProvider +public class SampleMetaDataProvider extends AbstractQQQApplication { public static boolean USE_MYSQL = false; @@ -128,6 +129,17 @@ public class SampleMetaDataProvider + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QInstance defineQInstance() throws QException + { + return defineInstance(); + } + + + /******************************************************************************* ** *******************************************************************************/ From 164d9e1de50186b72366d4bdf61f5daef499dcb1 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 21:46:49 -0600 Subject: [PATCH 54/84] CE-1955 Checkstyle --- .../selenium/BaseSampleSeleniumTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java index a0a904fc..5c45c3de 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BaseSampleSeleniumTest.java @@ -1,5 +1,22 @@ /* - * Copyright © 2022-2023. ColdTrack . All Rights Reserved. + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . */ package com.kingsrook.sampleapp.selenium; From eec1924113ff19c645c501b5885bf47ad374d835 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 3 Dec 2024 22:03:03 -0600 Subject: [PATCH 55/84] CE-1955 add browser-tools orb, to try to fix selenium/chrome version mismatch --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fffc467..a3e27fbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: localstack: localstack/platform@2.1 + browser-tools: circleci/browser-tools@1.4.7 commands: store_jacoco_site: @@ -38,6 +39,8 @@ commands: - restore_cache: keys: - v1-dependencies-{{ checksum "pom.xml" }} + - browser-tools/install-chrome + - browser-tools/install-chromedriver - run: name: Write .env command: | From 434d1587761416e2f61195506ddc71f205f21d79 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 07:12:10 -0600 Subject: [PATCH 56/84] CE-1955 disable until ci selenium fixed --- .../com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java index 02f2785d..e97055f5 100644 --- a/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java +++ b/qqq-sample-project/src/test/java/com/kingsrook/sampleapp/selenium/BulkLoadSeleniumTest.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.UUID; import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -39,6 +40,7 @@ public class BulkLoadSeleniumTest extends BaseSampleSeleniumTest ** *******************************************************************************/ @Test + @Disabled("selenium not working in circleci at this time...") void testSimple() throws IOException { String email = "jtkirk@starfleet.com"; From c4583f16a94b92ffa996798034f3bf3a6ffa29ee Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 14:58:34 -0600 Subject: [PATCH 57/84] CE-1955 Fix to re-set the position of the review step, upon going back --- .../BulkInsertPrepareFileMappingStep.java | 2 +- .../BulkInsertPrepareValueMappingStep.java | 2 +- .../StreamedETLWithFrontendProcess.java | 18 +++++++++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java index 80cb49d1..c72df5fd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileMappingStep.java @@ -65,7 +65,7 @@ public class BulkInsertPrepareFileMappingStep implements BackendStep { needSuggestedMapping = false; - StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); } if(needSuggestedMapping) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index d24856c0..471f0661 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -78,7 +78,7 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep { if(runBackendStepOutput.getProcessState().getIsStepBack()) { - StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput); + StreamedETLWithFrontendProcess.resetValidationFields(runBackendStepInput, runBackendStepOutput); } ///////////////////////////////////////////////////////////// diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java index d05bb12c..09e621ce 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLWithFrontendProcess.java @@ -23,11 +23,14 @@ package com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwit import java.io.Serializable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; @@ -70,6 +73,8 @@ import com.kingsrook.qqq.backend.core.processes.implementations.basepull.Basepul *******************************************************************************/ public class StreamedETLWithFrontendProcess { + private static final QLogger LOG = QLogger.getLogger(StreamedETLWithFrontendProcess.class); + public static final String STEP_NAME_PREVIEW = "preview"; public static final String STEP_NAME_REVIEW = "review"; public static final String STEP_NAME_VALIDATE = "validate"; @@ -197,11 +202,22 @@ public class StreamedETLWithFrontendProcess /*************************************************************************** ** useful for a process step to call upon 'back' ***************************************************************************/ - public static void resetValidationFields(RunBackendStepInput runBackendStepInput) + public static void resetValidationFields(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) { runBackendStepInput.addValue(FIELD_DO_FULL_VALIDATION, null); runBackendStepInput.addValue(FIELD_VALIDATION_SUMMARY, null); runBackendStepInput.addValue(FIELD_PROCESS_SUMMARY, null); + + ////////////////////////////////////////////////////////////////////////////////////////////////// + // in case, on the first time forward, the review step got moved after the validation step // + // (see BaseStreamedETLStep.moveReviewStepAfterValidateStep) - then un-do that upon going back. // + ////////////////////////////////////////////////////////////////////////////////////////////////// + ArrayList stepList = new ArrayList<>(runBackendStepOutput.getProcessState().getStepList()); + LOG.debug("Resetting step list. It was:" + stepList); + stepList.removeIf(s -> s.equals(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW)); + stepList.add(stepList.indexOf(StreamedETLWithFrontendProcess.STEP_NAME_PREVIEW) + 1, StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + runBackendStepOutput.getProcessState().setStepList(stepList); + LOG.debug("... and now step list is: " + stepList); } From 271f2dc25be8fcbbdddf933211b34b154e06e5bb Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 4 Dec 2024 14:59:53 -0600 Subject: [PATCH 58/84] CE-1955 Add a display-value for the mappingJSON in saved bulk-load-profiles --- ...ProfileJsonFieldDisplayValueFormatter.java | 141 ++++++++++++++++++ .../SavedBulkLoadProfileMetaDataProvider.java | 7 +- 2 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java new file mode 100644 index 00000000..843a1063 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileJsonFieldDisplayValueFormatter.java @@ -0,0 +1,141 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.values.ValueBehaviorApplier; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldDisplayBehavior; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class SavedBulkLoadProfileJsonFieldDisplayValueFormatter implements FieldDisplayBehavior +{ + private static SavedBulkLoadProfileJsonFieldDisplayValueFormatter savedReportJsonFieldDisplayValueFormatter = null; + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private SavedBulkLoadProfileJsonFieldDisplayValueFormatter() + { + + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static SavedBulkLoadProfileJsonFieldDisplayValueFormatter getInstance() + { + if(savedReportJsonFieldDisplayValueFormatter == null) + { + savedReportJsonFieldDisplayValueFormatter = new SavedBulkLoadProfileJsonFieldDisplayValueFormatter(); + } + return (savedReportJsonFieldDisplayValueFormatter); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public SavedBulkLoadProfileJsonFieldDisplayValueFormatter getDefault() + { + return getInstance(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void apply(ValueBehaviorApplier.Action action, List recordList, QInstance instance, QTableMetaData table, QFieldMetaData field) + { + for(QRecord record : CollectionUtils.nonNullList(recordList)) + { + if(field.getName().equals("mappingJson")) + { + String mappingJson = record.getValueString("mappingJson"); + if(StringUtils.hasContent(mappingJson)) + { + try + { + record.setDisplayValue("mappingJson", jsonToDisplayValue(mappingJson)); + } + catch(Exception e) + { + record.setDisplayValue("mappingJson", "Invalid Mapping..."); + } + } + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private String jsonToDisplayValue(String mappingJson) + { + JSONObject jsonObject = new JSONObject(mappingJson); + + List parts = new ArrayList<>(); + + if(jsonObject.has("fieldList")) + { + JSONArray fieldListArray = jsonObject.getJSONArray("fieldList"); + parts.add(fieldListArray.length() + " field" + StringUtils.plural(fieldListArray.length())); + } + + if(jsonObject.has("hasHeaderRow")) + { + boolean hasHeaderRow = jsonObject.getBoolean("hasHeaderRow"); + parts.add((hasHeaderRow ? "With" : "Without") + " header row"); + } + + if(jsonObject.has("layout")) + { + String layout = jsonObject.getString("layout"); + parts.add("Layout: " + StringUtils.allCapsToMixedCase(layout)); + } + + return StringUtils.join("; ", parts); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java index 23826835..1438dd37 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedbulkloadprofiles/SavedBulkLoadProfileMetaDataProvider.java @@ -113,12 +113,11 @@ public class SavedBulkLoadProfileMetaDataProvider .withFieldsFromEntity(SavedBulkLoadProfile.class) .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD)) .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "label", "tableName"))) - .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("mappingJson")).withIsHidden(true)) - .withSection(new QFieldSection("hidden", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId")).withIsHidden(true)) + .withSection(new QFieldSection("details", new QIcon().withName("text_snippet"), Tier.T2, List.of("userId", "mappingJson"))) .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate"))); - // todo - want one of these? - // table.getField("queryFilterJson").withBehavior(SavedReportJsonFieldDisplayValueFormatter.getInstance()); + table.getField("mappingJson").withBehavior(SavedBulkLoadProfileJsonFieldDisplayValueFormatter.getInstance()); + table.getField("mappingJson").setLabel("Mapping"); table.withShareableTableMetaData(new ShareableTableMetaData() .withSharedRecordTableName(SharedSavedBulkLoadProfile.TABLE_NAME) From 000226c30a18d0a4c7e76763b838be2d9f1d5257 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 20 Dec 2024 09:38:20 -0600 Subject: [PATCH 59/84] Make unique id on pet species enum --- .../kingsrook/sampleapp/metadata/SampleMetaDataProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java index 062d8721..c416e358 100644 --- a/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java +++ b/qqq-sample-project/src/main/java/com/kingsrook/sampleapp/metadata/SampleMetaDataProvider.java @@ -778,7 +778,7 @@ public class SampleMetaDataProvider extends AbstractQQQApplication public enum PetSpecies implements PossibleValueEnum { DOG(1, "Dog"), - CAT(1, "Cat"); + CAT(2, "Cat"); private final Integer id; private final String label; From db526009d2b1d353b97eb70dab00710f2dcfd534 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:27:21 -0600 Subject: [PATCH 60/84] CE-1955 - more flexible handling of inbound types for looking up possible values --- .../insert/mapping/BulkLoadValueMapper.java | 46 +++++++++++++++---- .../mapping/BulkLoadValueMapperTest.java | 43 +++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index cc44e027..4fa7a414 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -40,6 +40,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; @@ -170,20 +171,44 @@ public class BulkLoadValueMapper ***************************************************************************/ private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException { - Set values = fieldPossibleValueToRecordMap.keySet(); - Map> valuesFound = new HashMap<>(); - Set valuesNotFound = new HashSet<>(values); + QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); + + Set values = fieldPossibleValueToRecordMap.keySet(); + Map valuesToValueInPvsIdTypeMap = new HashMap<>(); + Map> valuesFound = new HashMap<>(); + Set valuesNotFound = new HashSet<>(); //////////////////////////////////////////////////////// // do a search, trying to use all given values as ids // //////////////////////////////////////////////////////// SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList idList = new ArrayList<>(values); + + ArrayList idList = new ArrayList<>(); + for(String value : values) + { + Serializable valueInPvsIdType = value; + + try + { + valueInPvsIdType = ValueUtils.getValueAsFieldType(possibleValueSource.getIdType(), value); + } + catch(Exception e) + { + //////////////////////////// + // leave as original type // + //////////////////////////// + } + + valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType); + idList.add(valueInPvsIdType); + valuesNotFound.add(valueInPvsIdType); + } + searchPossibleValueSourceInput.setIdList(idList); searchPossibleValueSourceInput.setLimit(values.size()); LOG.debug("Searching possible value source by ids during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfIds", idList.size()), logPair("firstId", () -> idList.get(0))); - SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); + SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); //////////////////////////////////////////////////////////////////////////////////////////////////// // for each possible value found, remove it from the set of ones not-found, and store it as a hit // @@ -202,7 +227,7 @@ public class BulkLoadValueMapper { searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList labelList = new ArrayList<>(valuesNotFound); + List labelList = valuesNotFound.stream().map(ValueUtils::getValueAsString).toList(); searchPossibleValueSourceInput.setLabelList(labelList); searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); @@ -220,12 +245,15 @@ public class BulkLoadValueMapper //////////////////////////////////////////////////////////////////////////////// for(Map.Entry> entry : fieldPossibleValueToRecordMap.entrySet()) { - String value = entry.getKey(); + String value = entry.getKey(); + Serializable valueInPvsIdType = valuesToValueInPvsIdTypeMap.get(entry.getKey()); + String pvsIdAsString = ValueUtils.getValueAsString(valueInPvsIdType); + for(QRecord record : entry.getValue()) { - if(valuesFound.containsKey(value)) + if(valuesFound.containsKey(pvsIdAsString)) { - record.setValue(field.getName(), valuesFound.get(value).getId()); + record.setValue(field.getName(), valuesFound.get(pvsIdAsString).getId()); } else { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 687f1f62..26a4528f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.map import java.io.Serializable; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -36,6 +37,7 @@ import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; /******************************************************************************* @@ -113,6 +115,47 @@ class BulkLoadValueMapperTest extends BaseTest + /*************************************************************************** + ** + ***************************************************************************/ + void testPossibleValue(Serializable inputValue, Serializable expectedValue, boolean expectErrors) throws QException + { + QRecord inputRecord = new QRecord().withValue("homeStateId", inputValue); + BulkLoadValueMapper.valueMapping(List.of(inputRecord), new BulkInsertMapping(), QContext.getQInstance().getTable(TestUtils.TABLE_NAME_PERSON_MEMORY)); + assertEquals(expectedValue, inputRecord.getValue("homeStateId")); + + if(expectErrors) + { + assertThat(inputRecord.getErrors().get(0)).isInstanceOf(BulkLoadPossibleValueError.class); + } + else + { + assertThat(inputRecord.getErrors()).isNullOrEmpty(); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPossibleValues() throws QException + { + testPossibleValue(1, 1, false); + testPossibleValue("1", 1, false); + testPossibleValue("1.0", 1, false); + testPossibleValue(new BigDecimal("1.0"), 1, false); + testPossibleValue("IL", 1, false); + + testPossibleValue(512, 512, true); // an id, but not in the PVS + testPossibleValue("USA", "USA", true); + testPossibleValue(true, true, true); + testPossibleValue(new BigDecimal("4.7"), new BigDecimal("4.7"), true); + } + + + /*************************************************************************** ** ***************************************************************************/ From 2b0b176ced8219614b8f2f2934fae8a6871723c9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:27:47 -0600 Subject: [PATCH 61/84] CE-1955 - only handle a single level deep of associations... --- .../mapping/BulkLoadTableStructureBuilder.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java index 0ac81f00..26575be3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadTableStructureBuilder.java @@ -123,8 +123,18 @@ public class BulkLoadTableStructureBuilder for(Association childAssociation : CollectionUtils.nonNullList(table.getAssociations())) { - BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, parentAssociationPath); - tableStructure.addAssociation(associatedStructure); + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // at this time, we are not prepared to handle 3-level deep associations, so, only process them from the top level... // + // main challenge being, wide-mode. so, maybe we should just only support 3-level+ associations for tall? // + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(association == null) + { + String nextLevelPath = + (StringUtils.hasContent(parentAssociationPath) ? parentAssociationPath + "." : "") + + (association != null ? association.getName() : ""); + BulkLoadTableStructure associatedStructure = buildTableStructure(childAssociation.getAssociatedTableName(), childAssociation, nextLevelPath); + tableStructure.addAssociation(associatedStructure); + } } return (tableStructure); From 7e475e2c18e29eb6bc3be6f40bed151fa972586d Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 16 Dec 2024 08:29:22 -0600 Subject: [PATCH 62/84] CE-1955 - add idType to possibleValueSource - used by bulk load possible-value mapping --- .../core/instances/QInstanceEnricher.java | 60 +++++++++++++++++++ .../core/instances/QInstanceValidator.java | 2 + .../possiblevalues/QPossibleValueSource.java | 34 +++++++++++ 3 files changed, 96 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index 1d88cb7e..a0d926c0 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.instances; import java.io.Serializable; +import java.lang.reflect.Method; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -33,8 +36,10 @@ import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -89,6 +94,7 @@ import com.kingsrook.qqq.backend.core.scheduler.QScheduleManager; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -1347,6 +1353,60 @@ public class QInstanceEnricher } } + if(possibleValueSource.getIdType() == null) + { + QTableMetaData table = qInstance.getTable(possibleValueSource.getTableName()); + if(table != null) + { + String primaryKeyField = table.getPrimaryKeyField(); + QFieldMetaData primaryKeyFieldMetaData = table.getFields().get(primaryKeyField); + if(primaryKeyFieldMetaData != null) + { + possibleValueSource.setIdType(primaryKeyFieldMetaData.getType()); + } + } + } + } + else if(QPossibleValueSourceType.ENUM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + if(CollectionUtils.nullSafeHasContents(possibleValueSource.getEnumValues())) + { + Object id = possibleValueSource.getEnumValues().get(0).getId(); + try + { + possibleValueSource.setIdType(QFieldType.fromClass(id.getClass())); + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first enum value", e, logPair("possibleValueSource", possibleValueSource.getName()), logPair("id", id)); + } + } + } + } + else if(QPossibleValueSourceType.CUSTOM.equals(possibleValueSource.getType())) + { + if(possibleValueSource.getIdType() == null) + { + try + { + QCustomPossibleValueProvider customPossibleValueProvider = QCodeLoader.getCustomPossibleValueProvider(possibleValueSource); + + Method getPossibleValueMethod = customPossibleValueProvider.getClass().getDeclaredMethod("getPossibleValue", Serializable.class); + Type returnType = getPossibleValueMethod.getGenericReturnType(); + Type idType = ((ParameterizedType) returnType).getActualTypeArguments()[0]; + + if(idType instanceof Class c) + { + possibleValueSource.setIdType(QFieldType.fromClass(c)); + } + } + catch(Exception e) + { + LOG.warn("Error enriching possible value source with idType based on first custom value", e, logPair("possibleValueSource", possibleValueSource.getName())); + } + } } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 58618c3e..a1edd5a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -2075,6 +2075,8 @@ public class QInstanceValidator default -> errors.add("Unexpected possibleValueSource type: " + possibleValueSource.getType()); } + assertCondition(possibleValueSource.getIdType() != null, "possibleValueSource " + name + " is missing its idType."); + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java index e8fc860b..00b121b4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValueSource.java @@ -28,6 +28,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.TopLevelMetaDataInterface; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; /******************************************************************************* @@ -42,6 +43,8 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface private String label; private QPossibleValueSourceType type; + private QFieldType idType; + private String valueFormat = PVSValueFormatAndFields.LABEL_ONLY.getFormat(); private List valueFields = PVSValueFormatAndFields.LABEL_ONLY.getFields(); private String valueFormatIfNotFound = null; @@ -662,4 +665,35 @@ public class QPossibleValueSource implements TopLevelMetaDataInterface return (this); } + + /******************************************************************************* + ** Getter for idType + *******************************************************************************/ + public QFieldType getIdType() + { + return (this.idType); + } + + + + /******************************************************************************* + ** Setter for idType + *******************************************************************************/ + public void setIdType(QFieldType idType) + { + this.idType = idType; + } + + + + /******************************************************************************* + ** Fluent setter for idType + *******************************************************************************/ + public QPossibleValueSource withIdType(QFieldType idType) + { + this.idType = idType; + return (this); + } + + } From 6b7d3ac26da42f83440bf6e563d8b9f1776b6dae Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 18:53:27 -0600 Subject: [PATCH 63/84] CE-1955 propagate errors from child (association) records up to main record --- .../bulk/insert/BulkInsertTransformStep.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index bd1fb40c..f113f76f 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -197,6 +197,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep List recordsWithSomeErrors = new ArrayList<>(); for(QRecord record : runBackendStepInput.getRecords()) { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + if(CollectionUtils.nullSafeHasContents(record.getErrors())) { recordsWithSomeErrors.add(record); @@ -351,6 +359,34 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + ** + ***************************************************************************/ + private List getErrorsFromAssociations(QRecord record) + { + List rs = null; + for(Map.Entry> entry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + for(QRecord associatedRecord : CollectionUtils.nonNullList(entry.getValue())) + { + if(CollectionUtils.nullSafeHasContents(associatedRecord.getErrors())) + { + rs = Objects.requireNonNullElseGet(rs, () -> new ArrayList<>()); + rs.addAll(associatedRecord.getErrors()); + + List childErrors = getErrorsFromAssociations(associatedRecord); + if(CollectionUtils.nullSafeHasContents(childErrors)) + { + rs.addAll(childErrors); + } + } + } + } + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ From 9cfc7fafc12c29dceaafb466086822425ba72e0b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:08:01 -0600 Subject: [PATCH 64/84] CE-1955 case-insenitiveKey map, to help with bulk load possible value case-insensitvity --- .../collections/CaseInsensitiveKeyMap.java | 53 +++ .../utils/collections/TransformedKeyMap.java | 400 ++++++++++++++++++ .../CaseInsensitiveKeyMapTest.java | 52 +++ .../collections/TransformedKeyMapTest.java | 199 +++++++++ 4 files changed, 704 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java create mode 100644 qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java new file mode 100644 index 00000000..8cbcb54d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMap.java @@ -0,0 +1,53 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Map; +import java.util.function.Supplier; + + +/******************************************************************************* + ** Version of map where string keys are handled case-insensitively. e.g., + ** map.put("One", 1); map.get("ONE") == 1. + *******************************************************************************/ +public class CaseInsensitiveKeyMap extends TransformedKeyMap +{ + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap() + { + super(key -> key.toLowerCase()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public CaseInsensitiveKeyMap(Supplier> supplier) + { + super(key -> key.toLowerCase(), supplier); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java new file mode 100644 index 00000000..afe1116d --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java @@ -0,0 +1,400 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + + +/******************************************************************************* + ** Version of a map that uses a transformation function on keys. The original + ** idea being, e.g., to support case-insensitive keys via a toLowerCase transform. + ** e.g., map.put("One", 1); map.get("ONE") == 1. + ** + ** But, implemented generically to support any transformation function. + ** + ** keySet() and entries() should give only the first version of a key that overlapped. + ** e.g., map.put("One", 1); map.put("one", 1); map.keySet() == Set.of("One"); + *******************************************************************************/ +public class TransformedKeyMap implements Map +{ + private Function keyTransformer; + private Map wrappedMap; + + private Map originalKeys = new HashMap<>(); + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = new HashMap<>(); + } + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMap(Function keyTransformer, Supplier> supplier) + { + this.keyTransformer = keyTransformer; + this.wrappedMap = supplier.get(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int size() + { + return (wrappedMap.size()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEmpty() + { + return (wrappedMap.isEmpty()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsKey(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.containsKey(transformed); + } + catch(Exception e) + { + return (false); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean containsValue(Object value) + { + return (wrappedMap.containsValue(value)); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V get(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + return wrappedMap.get(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @Nullable V put(OK key, V value) + { + TK transformed = keyTransformer.apply(key); + originalKeys.putIfAbsent(transformed, key); + return wrappedMap.put(transformed, value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public V remove(Object key) + { + try + { + TK transformed = keyTransformer.apply((OK) key); + originalKeys.remove(transformed); + return wrappedMap.remove(transformed); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void putAll(@NotNull Map m) + { + for(Entry entry : m.entrySet()) + { + put(entry.getKey(), entry.getValue()); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void clear() + { + wrappedMap.clear(); + originalKeys.clear(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set keySet() + { + return new HashSet<>(originalKeys.values()); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Collection values() + { + return wrappedMap.values(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public @NotNull Set> entrySet() + { + Set> wrappedEntries = wrappedMap.entrySet(); + Set> originalEntries; + try + { + originalEntries = wrappedEntries.getClass().getConstructor().newInstance(); + } + catch(Exception e) + { + originalEntries = new HashSet<>(); + } + + for(Entry wrappedEntry : wrappedEntries) + { + OK originalKey = originalKeys.get(wrappedEntry.getKey()); + originalEntries.add(new TransformedKeyMapEntry<>(originalKey, wrappedEntry.getValue())); + } + + return (originalEntries); + } + + // methods with a default implementation below here // + + + + /* + @Override + public V getOrDefault(Object key, V defaultValue) + { + return Map.super.getOrDefault(key, defaultValue); + } + + + + @Override + public void forEach(BiConsumer action) + { + Map.super.forEach(action); + } + + + + @Override + public void replaceAll(BiFunction function) + { + Map.super.replaceAll(function); + } + + + + @Override + public @Nullable V putIfAbsent(OK key, V value) + { + return Map.super.putIfAbsent(key, value); + } + + + + @Override + public boolean remove(Object key, Object value) + { + return Map.super.remove(key, value); + } + + + + @Override + public boolean replace(OK key, V oldValue, V newValue) + { + return Map.super.replace(key, oldValue, newValue); + } + + + + @Override + public @Nullable V replace(OK key, V value) + { + return Map.super.replace(key, value); + } + + + + @Override + public V computeIfAbsent(OK key, @NotNull Function mappingFunction) + { + return Map.super.computeIfAbsent(key, mappingFunction); + } + + + + @Override + public V computeIfPresent(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.computeIfPresent(key, remappingFunction); + } + + + + @Override + public V compute(OK key, @NotNull BiFunction remappingFunction) + { + return Map.super.compute(key, remappingFunction); + } + + + + @Override + public V merge(OK key, @NotNull V value, @NotNull BiFunction remappingFunction) + { + return Map.super.merge(key, value, remappingFunction); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + public static class TransformedKeyMapEntry implements Map.Entry + { + private final EK key; + private EV value; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TransformedKeyMapEntry(EK key, EV value) + { + this.key = key; + this.value = value; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public EK getKey() + { + return (key); + } + + + + @Override + public EV getValue() + { + return (value); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public EV setValue(EV value) + { + throw (new UnsupportedOperationException("Setting value in an entry of a TransformedKeyMap is not supported.")); + } + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java new file mode 100644 index 00000000..63919551 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/CaseInsensitiveKeyMapTest.java @@ -0,0 +1,52 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for CaseInsensitiveKeyMap + *******************************************************************************/ +class CaseInsensitiveKeyMapTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + CaseInsensitiveKeyMap map = new CaseInsensitiveKeyMap<>(); + map.put("One", 1); + map.put("one", 1); + map.put("ONE", 1); + assertEquals(1, map.get("one")); + assertEquals(1, map.get("One")); + assertEquals(1, map.get("oNe")); + assertEquals(1, map.size()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java new file mode 100644 index 00000000..f319706d --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java @@ -0,0 +1,199 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.utils.collections; + + +import java.math.BigDecimal; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + + +/******************************************************************************* + ** Unit test for TransformedKeyMap + *******************************************************************************/ +@SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" }) +class TransformedKeyMapTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCaseInsensitiveKeyMap() + { + TransformedKeyMap caseInsensitiveKeys = new TransformedKeyMap<>(key -> key.toLowerCase()); + caseInsensitiveKeys.put("One", 1); + caseInsensitiveKeys.put("one", 1); + caseInsensitiveKeys.put("ONE", 1); + assertEquals(1, caseInsensitiveKeys.get("one")); + assertEquals(1, caseInsensitiveKeys.get("One")); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(1, caseInsensitiveKeys.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", caseInsensitiveKeys.entrySet().iterator().next().getKey()); + assertEquals("One", caseInsensitiveKeys.keySet().iterator().next()); + + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + + for(String key : caseInsensitiveKeys.keySet()) + { + assertEquals(1, caseInsensitiveKeys.get(key)); + } + + for(Map.Entry entry : caseInsensitiveKeys.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(1, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + caseInsensitiveKeys.put("Two", 2); + assertEquals(2, caseInsensitiveKeys.size()); + assertEquals(2, caseInsensitiveKeys.entrySet().size()); + assertEquals(2, caseInsensitiveKeys.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + caseInsensitiveKeys.remove("TWO"); + assertNull(caseInsensitiveKeys.get("Two")); + assertNull(caseInsensitiveKeys.get("two")); + assertEquals(1, caseInsensitiveKeys.size()); + assertEquals(1, caseInsensitiveKeys.keySet().size()); + assertEquals(1, caseInsensitiveKeys.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + caseInsensitiveKeys.clear(); + assertNull(caseInsensitiveKeys.get("one")); + assertEquals(0, caseInsensitiveKeys.size()); + assertEquals(0, caseInsensitiveKeys.keySet().size()); + assertEquals(0, caseInsensitiveKeys.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + caseInsensitiveKeys.putAll(Map.of("One", 1, "one", 1, "ONE", 1, "TwO", 2, "tWo", 2, "three", 3)); + assertEquals(1, caseInsensitiveKeys.get("oNe")); + assertEquals(2, caseInsensitiveKeys.get("two")); + assertEquals(3, caseInsensitiveKeys.get("Three")); + assertEquals(3, caseInsensitiveKeys.size()); + assertEquals(3, caseInsensitiveKeys.entrySet().size()); + assertEquals(3, caseInsensitiveKeys.keySet().size()); + } + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testStringToNumberMap() + { + BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); + BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); + + TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> + switch (key.toLowerCase()) + { + case "one", "uno", "eins" -> 1; + case "two", "dos", "zwei" -> 2; + case "three", "tres", "drei" -> 3; + default -> null; + }); + multiLingualWordToNumber.put("One", BigDecimal.ONE); + multiLingualWordToNumber.put("uno", BigDecimal.ONE); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("uno")); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("eins")); + assertEquals(1, multiLingualWordToNumber.size()); + + ////////////////////////////////////////////////// + // get back the first way it was put in the map // + ////////////////////////////////////////////////// + assertEquals("One", multiLingualWordToNumber.entrySet().iterator().next().getKey()); + assertEquals("One", multiLingualWordToNumber.keySet().iterator().next()); + + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + + for(String key : multiLingualWordToNumber.keySet()) + { + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get(key)); + } + + for(Map.Entry entry : multiLingualWordToNumber.entrySet()) + { + assertEquals("One", entry.getKey()); + assertEquals(BigDecimal.ONE, entry.getValue()); + } + + ///////////////////////////// + // add a second unique key // + ///////////////////////////// + multiLingualWordToNumber.put("Two", BIG_DECIMAL_TWO); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("Dos")); + assertEquals(2, multiLingualWordToNumber.size()); + assertEquals(2, multiLingualWordToNumber.entrySet().size()); + assertEquals(2, multiLingualWordToNumber.keySet().size()); + + //////////////////////////////////////// + // make sure remove works as expected // + //////////////////////////////////////// + multiLingualWordToNumber.remove("ZWEI"); + assertNull(multiLingualWordToNumber.get("Two")); + assertNull(multiLingualWordToNumber.get("Dos")); + assertEquals(1, multiLingualWordToNumber.size()); + assertEquals(1, multiLingualWordToNumber.keySet().size()); + assertEquals(1, multiLingualWordToNumber.entrySet().size()); + + /////////////////////////////////////// + // make sure clear works as expected // + /////////////////////////////////////// + multiLingualWordToNumber.clear(); + assertNull(multiLingualWordToNumber.get("eins")); + assertNull(multiLingualWordToNumber.get("One")); + assertEquals(0, multiLingualWordToNumber.size()); + assertEquals(0, multiLingualWordToNumber.keySet().size()); + assertEquals(0, multiLingualWordToNumber.entrySet().size()); + + ///////////////////////////////////////// + // make sure put-all works as expected // + ///////////////////////////////////////// + multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei",BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); + assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe")); + assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos")); + assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei")); + assertEquals(3, multiLingualWordToNumber.size()); + assertEquals(3, multiLingualWordToNumber.entrySet().size()); + assertEquals(3, multiLingualWordToNumber.keySet().size()); + } + +} \ No newline at end of file From a4499219c8837e712356c722e036746555b3cad7 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:09:41 -0600 Subject: [PATCH 65/84] CE-1955 Update fastexcel version; Update XlsxFileToRows to read formats, and then do a better job of handling numbers as date-time, date, int, or decimal (hopefully) --- qqq-backend-core/pom.xml | 2 +- .../insert/filehandling/XlsxFileToRows.java | 192 ++++++++++++++---- .../filehandling/XlsxFileToRowsTest.java | 59 +++++- 3 files changed, 210 insertions(+), 43 deletions(-) diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml index 0944e1e9..21723b83 100644 --- a/qqq-backend-core/pom.xml +++ b/qqq-backend-core/pom.xml @@ -100,7 +100,7 @@ org.dhatim fastexcel - 0.12.15 + 0.18.4 org.dhatim diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index 206ca722..de0b1ccb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -26,14 +26,16 @@ import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.math.BigDecimal; -import java.math.MathContext; -import java.time.LocalDateTime; import java.util.Optional; +import java.util.regex.Pattern; import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.ReadingOptions; +import org.dhatim.fastexcel.reader.Row; import org.dhatim.fastexcel.reader.Sheet; @@ -42,6 +44,10 @@ import org.dhatim.fastexcel.reader.Sheet; *******************************************************************************/ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface { + private static final QLogger LOG = QLogger.getLogger(XlsxFileToRows.class); + + private static final Pattern DAY_PATTERN = Pattern.compile(".*\\b(d|dd)\\b.*"); + private ReadableWorkbook workbook; private Stream rows; @@ -55,7 +61,7 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows - { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... // - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - Optional dateTime = readerRow.getCellAsDate(i); - if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100) - { - yield dateTime.get(); - } - - Optional optionalBigDecimal = readerRow.getCellAsNumber(i); - if(optionalBigDecimal.isPresent()) - { - BigDecimal bigDecimal = optionalBigDecimal.get(); - if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0) - { - yield bigDecimal.intValue(); - } - - yield bigDecimal; - } - - yield (null); - } - case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null); - case STRING, FORMULA -> cell.getText(); - case EMPTY, ERROR -> null; - }; - } + values[i] = processCell(readerRow, i); } return new BulkLoadFileRow(values, getRowNo()); @@ -121,6 +93,150 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + { + ///////////////////////////////////////////////////////////////////////////////////// + // dates, date-times, integers, and decimals are all identified as type = "number" // + // so go through this process to try to identify what user means it as // + ///////////////////////////////////////////////////////////////////////////////////// + if(isDateTimeFormat(dataFormatString)) + { + //////////////////////////////////////////////////////////////////////////////////////// + // first - if it has a date-time looking format string, then treat it as a date-time. // + //////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate()); + } + else if(isDateFormat(dataFormatString)) + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // second, if it has a date looking format string (which is a sub-set of date-time), then treat as date. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + return (cell.asDate().toLocalDate()); + } + else + { + //////////////////////////////////////////////////////////////////////////////////////// + // now assume it's a number - but in case this optional is empty (why?) return a null // + //////////////////////////////////////////////////////////////////////////////////////// + Optional bigDecimal = readerRow.getCellAsNumber(columnIndex); + if(bigDecimal.isEmpty()) + { + return (null); + } + + try + { + //////////////////////////////////////////////////////////// + // now if the bigDecimal is an exact integer, return that // + //////////////////////////////////////////////////////////// + Integer i = bigDecimal.get().intValueExact(); + return (i); + } + catch(ArithmeticException e) + { + ///////////////////////////////// + // else, end up with a decimal // + ///////////////////////////////// + return (bigDecimal.get()); + } + } + } + case STRING -> + { + return cell.asString(); + } + case BOOLEAN -> + { + return cell.asBoolean(); + } + case EMPTY, ERROR, FORMULA -> + { + LOG.debug("cell type: " + cell.getType() + " had value string: " + cell.asString()); + return (null); + } + default -> + { + return (null); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateTimeFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString) && hasHour(dataFormatString)) + { + return (true); + } + + return false; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasHour(String dataFormatString) + { + return dataFormatString.contains("h"); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean hasDay(String dataFormatString) + { + return DAY_PATTERN.matcher(dataFormatString).matches(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + static boolean isDateFormat(String dataFormatString) + { + if(dataFormatString == null) + { + return (false); + } + + if(hasDay(dataFormatString)) + { + return (true); + } + + return false; + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java index 95052d56..8d144ad6 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.time.LocalDate; -import java.time.LocalDateTime; import java.time.Month; import java.util.Map; import com.kingsrook.qqq.backend.core.BaseTest; @@ -45,6 +44,7 @@ import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; /******************************************************************************* @@ -61,13 +61,14 @@ class XlsxFileToRowsTest extends BaseTest { byte[] byteArray = writeExcelBytes(); - FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray)); + FileToRowsInterface fileToRowsInterface = new XlsxFileToRows(); + fileToRowsInterface.init(new ByteArrayInputStream(byteArray)); BulkLoadFileRow headerRow = fileToRowsInterface.next(); BulkLoadFileRow bodyRow = fileToRowsInterface.next(); - assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name", "Birth Date"}, 1), headerRow); - assertEquals(new BulkLoadFileRow(new Serializable[] {1, "Darin", "Jonson", LocalDateTime.of(1980, Month.JANUARY, 31, 0, 0)}, 2), bodyRow); + assertEquals(new BulkLoadFileRow(new String[] { "Id", "First Name", "Last Name", "Birth Date" }, 1), headerRow); + assertEquals(new BulkLoadFileRow(new Serializable[] { 1, "Darin", "Jonson", LocalDate.of(1980, Month.JANUARY, 31) }, 2), bodyRow); /////////////////////////////////////////////////////////////////////////////////////// // make sure there's at least a limit (less than 20) to how many more rows there are // @@ -107,4 +108,54 @@ class XlsxFileToRowsTest extends BaseTest return byteArray; } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testDateTimeFormats() + { + assertFormatDateAndOrDateTime(true, false, "dddd, m/d/yy at h:mm"); + assertFormatDateAndOrDateTime(true, false, "h PM, ddd mmm dd"); + assertFormatDateAndOrDateTime(true, false, "dd/mm/yyyy hh:mm"); + assertFormatDateAndOrDateTime(true, false, "yyyy-mm-dd hh:mm:ss.000"); + assertFormatDateAndOrDateTime(true, false, "hh:mm dd/mm/yyyy"); + + assertFormatDateAndOrDateTime(false, true, "yyyy-mm-dd"); + assertFormatDateAndOrDateTime(false, true, "mmmm d \\[dddd\\]"); + assertFormatDateAndOrDateTime(false, true, "mmm dd, yyyy"); + assertFormatDateAndOrDateTime(false, true, "d-mmm"); + assertFormatDateAndOrDateTime(false, true, "dd.mm.yyyy"); + + assertFormatDateAndOrDateTime(false, false, "yyyy"); + assertFormatDateAndOrDateTime(false, false, "mmm-yyyy"); + assertFormatDateAndOrDateTime(false, false, "hh"); + assertFormatDateAndOrDateTime(false, false, "hh:mm"); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void assertFormatDateAndOrDateTime(boolean expectDateTime, boolean expectDate, String format) + { + if(XlsxFileToRows.isDateTimeFormat(format)) + { + assertTrue(expectDateTime, format + " was considered a dateTime, but wasn't expected to."); + assertFalse(expectDate, format + " was considered a dateTime, but was expected to be a date."); + } + else if(XlsxFileToRows.isDateFormat(format)) + { + assertFalse(expectDateTime, format + " was considered a date, but was expected to be a dateTime."); + assertTrue(expectDate, format + " was considered a date, but was expected to."); + } + else + { + assertFalse(expectDateTime, format + " was not considered a dateTime, but was expected to."); + assertFalse(expectDate, format + " was considered a date, but was expected to."); + } + } + } \ No newline at end of file From 7f67eda2e310bd3245640078e874ae74ee660e2f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:11:41 -0600 Subject: [PATCH 66/84] CE-1955 do case-insensitive lookups of possible values by label --- .../insert/mapping/BulkLoadValueMapper.java | 63 ++++++++++++++----- .../mapping/BulkLoadValueMapperTest.java | 4 ++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index 4fa7a414..f6b01561 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -48,6 +48,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.ListingHash; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.collections.CaseInsensitiveKeyMap; import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; @@ -167,24 +168,57 @@ public class BulkLoadValueMapper /*************************************************************************** + ** Given a listingHash of Strings from the bulk-load file, to QRecords, + ** we will either: + ** - make sure the value set for the field is valid in the PV type + ** - or put an error in the record (leaving the original value from the file in the field) ** + ** We'll do potentially 2 possible-value searches - the first "by id" - + ** type-converting the input strings to the PV's id type. Then, if any + ** values weren't found by id, a second search by "labels"... which might + ** be a bit suspicious, e.g., if the PV has a multi-field label... ***************************************************************************/ private static void handlePossibleValues(QFieldMetaData field, ListingHash fieldPossibleValueToRecordMap, String associationNamePrefixForFields, String tableLabelPrefix) throws QException { QPossibleValueSource possibleValueSource = QContext.getQInstance().getPossibleValueSource(field.getPossibleValueSourceName()); - Set values = fieldPossibleValueToRecordMap.keySet(); - Map valuesToValueInPvsIdTypeMap = new HashMap<>(); - Map> valuesFound = new HashMap<>(); - Set valuesNotFound = new HashSet<>(); + //////////////////////////////////////////////////////////// + // String from file -> List that have that value // + //////////////////////////////////////////////////////////// + Set values = fieldPossibleValueToRecordMap.keySet(); + + ///////////////////////////////////////////////////////////////////////////////// + // String from file -> Integer (for example) - that is the type-converted // + // version of the PVS's idType (but before any lookups were done with that id) // + // e.g., "42" -> 42 // + // e.g., "SOME_CONST" -> "SOME_CONST" (for PVS w/ string ids) // + ///////////////////////////////////////////////////////////////////////////////// + Map valuesToValueInPvsIdTypeMap = new HashMap<>(); + + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // String versions of EITHER ids or values found in searchPossibleValueSource call (depending on what was searched by) // + // e.g., "42" -> QPossibleValue(42, "Forty-two") (when searched by id) // + // e.g., "Forty-Two" -> QPossibleValue(42, "Forty-two") (when searched by label) // + // e.g., "SOME_CONST" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by id) // + // e.g., "Some Const" -> QPossibleValue("SOME_CONST", "Some Const") (when searched by label) // + // goal being - file could have "42" or "Forty-Two" (or "forty two") and those would all map to QPossibleValue(42, "Forty-two") // + // or - file could have "SOME_CONST" or "Some Const" (or "some const") and those would all map to QPossibleValue("SOME_CONST", "Some Const") // + // this is also why using CaseInsensitiveKeyMap! // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + CaseInsensitiveKeyMap> valuesFoundAsStrings = new CaseInsensitiveKeyMap<>(); + + ///////////////////////////////////////////////////////// + // String values (from file) that still need looked up // + ///////////////////////////////////////////////////////// + Set valuesNotFound = new HashSet<>(); //////////////////////////////////////////////////////// // do a search, trying to use all given values as ids // //////////////////////////////////////////////////////// + ArrayList idList = new ArrayList<>(); SearchPossibleValueSourceInput searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - ArrayList idList = new ArrayList<>(); for(String value : values) { Serializable valueInPvsIdType = value; @@ -202,7 +236,7 @@ public class BulkLoadValueMapper valuesToValueInPvsIdTypeMap.put(value, valueInPvsIdType); idList.add(valueInPvsIdType); - valuesNotFound.add(valueInPvsIdType); + valuesNotFound.add(value); } searchPossibleValueSourceInput.setIdList(idList); @@ -211,12 +245,12 @@ public class BulkLoadValueMapper SearchPossibleValueSourceOutput searchPossibleValueSourceOutput = idList.isEmpty() ? new SearchPossibleValueSourceOutput() : new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); //////////////////////////////////////////////////////////////////////////////////////////////////// - // for each possible value found, remove it from the set of ones not-found, and store it as a hit // + // for each possible value found, store it as a hit, and remove it from the set of ones not-found // //////////////////////////////////////////////////////////////////////////////////////////////////// for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { String valueAsString = ValueUtils.getValueAsString(possibleValue.getId()); - valuesFound.put(valueAsString, possibleValue); + valuesFoundAsStrings.put(valueAsString, possibleValue); valuesNotFound.remove(valueAsString); } @@ -227,15 +261,14 @@ public class BulkLoadValueMapper { searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); - List labelList = valuesNotFound.stream().map(ValueUtils::getValueAsString).toList(); - searchPossibleValueSourceInput.setLabelList(labelList); - searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); + searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound)); + searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); // todo - a little sus... leaves no room for any dupes, which, can they happen? - LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", labelList.size()), logPair("firstLabel", () -> labelList.get(0))); + LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next())); searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { - valuesFound.put(possibleValue.getLabel(), possibleValue); + valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue); valuesNotFound.remove(possibleValue.getLabel()); } } @@ -251,9 +284,9 @@ public class BulkLoadValueMapper for(QRecord record : entry.getValue()) { - if(valuesFound.containsKey(pvsIdAsString)) + if(valuesFoundAsStrings.containsKey(pvsIdAsString)) { - record.setValue(field.getName(), valuesFound.get(pvsIdAsString).getId()); + record.setValue(field.getName(), valuesFoundAsStrings.get(pvsIdAsString).getId()); } else { diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 26a4528f..2435a059 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -30,6 +30,7 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -148,6 +149,9 @@ class BulkLoadValueMapperTest extends BaseTest testPossibleValue(new BigDecimal("1.0"), 1, false); testPossibleValue("IL", 1, false); + BackendQueryFilterUtils.setCaseSensitive(true); + testPossibleValue("il", 1, false); + testPossibleValue(512, 512, true); // an id, but not in the PVS testPossibleValue("USA", "USA", true); testPossibleValue(true, true, true); From f57df2be86c5dff1720dbcbea0c0cd127944c50c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:12:01 -0600 Subject: [PATCH 67/84] CE-1955 change type-argument to be extends-Serializable --- .../model/metadata/possiblevalues/PossibleValueEnum.java | 5 ++++- .../core/model/metadata/possiblevalues/QPossibleValue.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java index fc61b2df..97585230 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/PossibleValueEnum.java @@ -22,11 +22,14 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; + + /******************************************************************************* ** Interface to be implemented by enums which can be used as a PossibleValueSource. ** *******************************************************************************/ -public interface PossibleValueEnum +public interface PossibleValueEnum { /******************************************************************************* ** diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java index a36fb981..929a7800 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/possiblevalues/QPossibleValue.java @@ -22,12 +22,15 @@ package com.kingsrook.qqq.backend.core.model.metadata.possiblevalues; +import java.io.Serializable; + + /******************************************************************************* ** An actual possible value - an id and label. ** ** Type parameter `T` is the type of the id (often Integer, maybe String) *******************************************************************************/ -public class QPossibleValue +public class QPossibleValue { private final T id; private final String label; From 8b00e8c877c1c72dd4072088c12bc4c4e8e2a962 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 26 Dec 2024 19:58:09 -0600 Subject: [PATCH 68/84] checkstyle --- .../utils/collections/TransformedKeyMap.java | 1 + .../collections/TransformedKeyMapTest.java | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java index afe1116d..ae424e66 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMap.java @@ -344,6 +344,7 @@ public class TransformedKeyMap implements Map { return Map.super.merge(key, value, remappingFunction); } + */ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java index f319706d..594aeabc 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/collections/TransformedKeyMapTest.java @@ -36,6 +36,10 @@ import static org.junit.jupiter.api.Assertions.assertNull; @SuppressWarnings({ "RedundantCollectionOperation", "RedundantOperationOnEmptyContainer" }) class TransformedKeyMapTest extends BaseTest { + private static final BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); + private static final BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); + + /******************************************************************************* ** @@ -111,23 +115,21 @@ class TransformedKeyMapTest extends BaseTest assertEquals(3, caseInsensitiveKeys.keySet().size()); } + + /******************************************************************************* ** *******************************************************************************/ @Test void testStringToNumberMap() { - BigDecimal BIG_DECIMAL_TWO = BigDecimal.valueOf(2); - BigDecimal BIG_DECIMAL_THREE = BigDecimal.valueOf(3); - - TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> - switch (key.toLowerCase()) - { - case "one", "uno", "eins" -> 1; - case "two", "dos", "zwei" -> 2; - case "three", "tres", "drei" -> 3; - default -> null; - }); + TransformedKeyMap multiLingualWordToNumber = new TransformedKeyMap<>(key -> switch(key.toLowerCase()) + { + case "one", "uno", "eins" -> 1; + case "two", "dos", "zwei" -> 2; + case "three", "tres", "drei" -> 3; + default -> null; + }); multiLingualWordToNumber.put("One", BigDecimal.ONE); multiLingualWordToNumber.put("uno", BigDecimal.ONE); assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("one")); @@ -187,7 +189,7 @@ class TransformedKeyMapTest extends BaseTest ///////////////////////////////////////// // make sure put-all works as expected // ///////////////////////////////////////// - multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei",BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); + multiLingualWordToNumber.putAll(Map.of("One", BigDecimal.ONE, "Uno", BigDecimal.ONE, "EINS", BigDecimal.ONE, "dos", BIG_DECIMAL_TWO, "zwei", BIG_DECIMAL_TWO, "tres", BIG_DECIMAL_THREE)); assertEquals(BigDecimal.ONE, multiLingualWordToNumber.get("oNe")); assertEquals(BIG_DECIMAL_TWO, multiLingualWordToNumber.get("dos")); assertEquals(BIG_DECIMAL_THREE, multiLingualWordToNumber.get("drei")); From 21982e8f53571af6e906e0dc2d7f2019557a7d29 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 27 Dec 2024 08:54:22 -0600 Subject: [PATCH 69/84] Remove uncommitted BackendQueryFilterUtils.setCaseSensitive --- .../bulk/insert/mapping/BulkLoadValueMapperTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java index 2435a059..c73a2147 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapperTest.java @@ -30,7 +30,6 @@ import com.kingsrook.qqq.backend.core.BaseTest; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.modules.backend.implementations.utils.BackendQueryFilterUtils; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkInsertMapping; import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.TestUtils; @@ -148,8 +147,6 @@ class BulkLoadValueMapperTest extends BaseTest testPossibleValue("1.0", 1, false); testPossibleValue(new BigDecimal("1.0"), 1, false); testPossibleValue("IL", 1, false); - - BackendQueryFilterUtils.setCaseSensitive(true); testPossibleValue("il", 1, false); testPossibleValue(512, 512, true); // an id, but not in the PVS From 048ee2e332bfef04b32a8399adad099a7b428681 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 27 Dec 2024 09:08:13 -0600 Subject: [PATCH 70/84] Expand cases hit due to new idType requirement in possible values --- .../core/instances/QInstanceValidatorTest.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java index 89c69734..760a0991 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidatorTest.java @@ -1185,10 +1185,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have searchFields", "should not have orderByFields", "should not have a customCodeReference", - "is missing enum values"); + "is missing enum values", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_STATE).setEnumValues(new ArrayList<>()), - "is missing enum values"); + "is missing enum values", + "is missing its idType."); } @@ -1213,10 +1215,12 @@ public class QInstanceValidatorTest extends BaseTest "should not have a customCodeReference", "is missing a tableName", "is missing searchFields", - "is missing orderByFields"); + "is missing orderByFields", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setTableName("Not a table"), - "Unrecognized table"); + "Unrecognized table", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_SHAPE).setSearchFields(List.of("id", "notAField", "name")), "unrecognized searchField: notAField"); @@ -1244,11 +1248,13 @@ public class QInstanceValidatorTest extends BaseTest "should not have a tableName", "should not have searchFields", "should not have orderByFields", - "is missing a customCodeReference"); + "is missing a customCodeReference", + "is missing its idType."); assertValidationFailureReasons((qInstance) -> qInstance.getPossibleValueSource(TestUtils.POSSIBLE_VALUE_SOURCE_CUSTOM).setCustomCodeReference(new QCodeReference()), "missing a code reference name", - "missing a code type"); + "missing a code type", + "is missing its idType."); } From 3fda1a1eda3400570fd66143ed7125ab3e1f3a01 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 12:57:49 -0600 Subject: [PATCH 71/84] CE-1955 Add handling for associations w/ some vs. all values coming from defaults instead of columns; --- .../bulk/insert/mapping/TallRowsToRecord.java | 21 ++++- ...licitFieldNameSuffixIndexBasedMapping.java | 80 +++++++++++++++++-- .../bulk/insert/model/BulkInsertMapping.java | 34 ++++++++ .../insert/mapping/TallRowsToRecordTest.java | 55 +++++++++++++ ...tFieldNameSuffixIndexBasedMappingTest.java | 53 ++++++++++++ 5 files changed, 233 insertions(+), 10 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index 84ab1fa7..bf67522c 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -32,6 +32,7 @@ import java.util.Optional; import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; @@ -43,6 +44,7 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.core.utils.Pair; import com.kingsrook.qqq.backend.core.utils.StringUtils; import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -50,6 +52,8 @@ import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; *******************************************************************************/ public class TallRowsToRecord implements RowsToRecordInterface { + private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class); + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); @@ -166,9 +170,9 @@ public class TallRowsToRecord implements RowsToRecordInterface Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); - ////////////////////////////////////////////////////// - // get all rows for the main table from the 0th row // - ////////////////////////////////////////////////////// + //////////////////////////////////////////////////////// + // get all values for the main table from the 0th row // + //////////////////////////////////////////////////////// BulkLoadFileRow row = rows.get(0); for(QFieldMetaData field : table.getFields().values()) { @@ -258,6 +262,17 @@ public class TallRowsToRecord implements RowsToRecordInterface // throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); } + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // special case here - if there are no group-by-indexes for the row, it means there are no fields coming from columns in the file. // + // but, if any fields for this association have a default value - then - make a row using just default values. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + LOG.info("Handling case of an association with no fields from the file, but rather only defaults", logPair("associationName", associationName)); + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, List.of(row))); + break; + } + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index c4231ea7..86a5faf2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -22,6 +22,7 @@ package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -71,7 +72,7 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem while(fileToRowsInterface.hasNext() && rs.size() < limit) { BulkLoadFileRow row = fileToRowsInterface.next(); - QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>()); + QRecord record = makeRecordFromRow(mapping, table, "", row, fieldIndexes, headerRow, new ArrayList<>(), false); rs.add(record); } @@ -84,8 +85,42 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem /*************************************************************************** ** may return null, if there were no values in the row for this (sub-wide) record. + ** more specifically: + ** + ** the param `rowOfOnlyDefaultValues` - should be false for the header table, + ** and true for an association iff all mapped fields are using 'default values' + ** (e.g., not values from the file). + ** + ** So this method will return null, indicating "no child row to build" if: + ** - when doing a rowOfOnlyDefaultValues - only if there actually weren't any + ** default values, which, probably never happens! + ** - else (doing a row with at least 1 value from the file) - then, null is + ** returned if there were NO values from the file. + ** + ** The goal here is to support these cases: + ** + ** Case A (a row of not only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = column: sku2 + ** - lineItem.qty,1 = column: qty2 + ** - lineItem.lineNo,1 = Default: 2 + ** then a file row with no values for sku2 & qty2 - we don't want a row + ** in that case (which would only have the default value of lineNo=2) + ** + ** Case B (a row of only-default-values): + ** - lineItem.sku,0 = column: sku1 + ** - lineItem.qty,0 = column: qty1 + ** - lineItem.lineNo,0 = Default: 1 + ** - lineItem.sku,1 = Default: SUPPLEMENT + ** - lineItem.qty,1 = Default: 1 + ** - lineItem.lineNo,1 = Default: 2 + ** we want every parent (order) to include a 2nd line item - with 3 + ** default values (sku=SUPPLEMENT, qty=q, lineNo=2). + ** ***************************************************************************/ - private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes) throws QException + private QRecord makeRecordFromRow(BulkInsertMapping mapping, QTableMetaData table, String associationNameChain, BulkLoadFileRow row, Map fieldIndexes, BulkLoadFileRow headerRow, List wideAssociationIndexes, boolean rowOfOnlyDefaultValues) throws QException { ////////////////////////////////////////////////////// // start by building the record with its own fields // @@ -93,15 +128,35 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem QRecord record = new QRecord(); BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean hadAnyValuesInRowFromFile = false; boolean hadAnyValuesInRow = false; for(QFieldMetaData field : table.getFields().values()) { - hadAnyValuesInRow = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRow; + hadAnyValuesInRowFromFile = setValueOrDefault(record, field, associationNameChain, mapping, row, fieldIndexes.get(field.getName()), wideAssociationIndexes) || hadAnyValuesInRowFromFile; + + ///////////////////////////////////////////////////////////////////////////////////// + // for wide mode (different from tall) - allow a row that only has default values. // + // e.g., Line Item (2) might be a default to add to every order // + ///////////////////////////////////////////////////////////////////////////////////// + if(record.getValue(field.getName()) != null) + { + hadAnyValuesInRow = true; + } } - if(!hadAnyValuesInRow) + if(rowOfOnlyDefaultValues) { - return (null); + if(!hadAnyValuesInRow) + { + return (null); + } + } + else + { + if(!hadAnyValuesInRowFromFile) + { + return (null); + } } ///////////////////////////// @@ -149,12 +204,23 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem // todo - doesn't support grand-children List wideAssociationIndexes = List.of(i); Map fieldIndexes = mapping.getFieldIndexes(associatedTable, associationNameChainForRecursiveCalls, headerRow, wideAssociationIndexes); + + boolean rowOfOnlyDefaultValues = false; if(fieldIndexes.isEmpty()) { - break; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if there aren't any field-indexes for this (i) value (e.g., no columns mapped for Line Item: X (2)), we can still build a // + // child record here if there are any default values - so check for them - and only if they are empty, then break the loop. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + Map fieldDefaultValues = mapping.getFieldDefaultValues(associatedTable, associationNameChainForRecursiveCalls, wideAssociationIndexes); + if(!CollectionUtils.nullSafeHasContents(fieldDefaultValues)) + { + break; + } + rowOfOnlyDefaultValues = true; } - QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes); + QRecord record = makeRecordFromRow(mapping, associatedTable, associationNameChainForRecursiveCalls, row, fieldIndexes, headerRow, wideAssociationIndexes, rowOfOnlyDefaultValues); if(record != null) { rs.add(record); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java index d0164b3f..e391b5f9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/model/BulkInsertMapping.java @@ -165,6 +165,40 @@ public class BulkInsertMapping implements Serializable + /*************************************************************************** + ** get a map of default-values for fields in a given table (at the specified + ** association chain and wide-indexes). Will only include fields using a + ** default value. + ***************************************************************************/ + @JsonIgnore + public Map getFieldDefaultValues(QTableMetaData table, String associationNameChain, List wideAssociationIndexes) throws QException + { + Map rs = new HashMap<>(); + + String wideAssociationSuffix = ""; + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + wideAssociationSuffix = "," + StringUtils.join(".", wideAssociationIndexes); + } + + /////////////////////////////////////////////////////////////////////////// + // loop over fields - adding them to the rs if they have a default value // + /////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + Serializable defaultValue = fieldNameToDefaultValueMap.get(fieldNamePrefix + field.getName() + wideAssociationSuffix); + if(defaultValue != null) + { + rs.put(field.getName(), defaultValue); + } + } + + return (rs); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java index 126f500b..b59f3b1d 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -105,6 +105,61 @@ class TallRowsToRecordTest extends BaseTest assertEquals(2, ((List) order.getBackendDetail("fileRows")).size()); } + + + /******************************************************************************* + ** test to show that we can do 1 default line item (child record) for each + ** header record. + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName + 1, Homer, Simpson + 2, Ned, Flanders + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku", "NUCLEAR-ROD", + "orderLine.quantity", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 2", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(1, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals("Row 3", order.getBackendDetail("rowNos")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + } + /******************************************************************************* ** *******************************************************************************/ diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 38488683..62eeb607 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -98,6 +98,59 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); } + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithLineValuesFromDefaults() throws QException + { + String csv = """ + orderNo, Ship To, lastName, SKU 1, Quantity 1 + 1, Homer, Simpson, DONUT, 12, + 2, Ned, Flanders, + """; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping rowsToRecord = new WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku,0", "SKU 1", + "orderLine.quantity,0", "Quantity 1" + )) + .withFieldNameToDefaultValueMap(Map.of( + "orderLine.sku,1", "NUCLEAR-ROD", + "orderLine.quantity,1", 1 + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(12, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("NUCLEAR-ROD"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of(1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); + assertEquals("Row 3", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); + } + /******************************************************************************* ** *******************************************************************************/ From e9fc5f81d236e39298a8af58ccbf900cb4de4e24 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 3 Jan 2025 12:58:12 -0600 Subject: [PATCH 72/84] CE-1955 Add some room for a PVS search to return duplicates... room to improve here though. --- .../bulk/insert/mapping/BulkLoadValueMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index f6b01561..641e11bb 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -262,12 +262,13 @@ public class BulkLoadValueMapper searchPossibleValueSourceInput = new SearchPossibleValueSourceInput(); searchPossibleValueSourceInput.setPossibleValueSourceName(field.getPossibleValueSourceName()); searchPossibleValueSourceInput.setLabelList(new ArrayList<>(valuesNotFound)); - searchPossibleValueSourceInput.setLimit(valuesNotFound.size()); // todo - a little sus... leaves no room for any dupes, which, can they happen? + searchPossibleValueSourceInput.setLimit(valuesNotFound.size() * 10); // todo - a little sus... leaves some room for dupes, which, can they happen? LOG.debug("Searching possible value source by labels during bulk load mapping", logPair("pvsName", field.getPossibleValueSourceName()), logPair("noOfLabels", valuesNotFound.size()), logPair("firstLabel", () -> valuesNotFound.iterator().next())); searchPossibleValueSourceOutput = new SearchPossibleValueSourceAction().execute(searchPossibleValueSourceInput); for(QPossibleValue possibleValue : searchPossibleValueSourceOutput.getResults()) { + // todo - deal with multiple values found - and maybe... if some end up not-found, but some dupes happened, should we try another search, in case we hit the limit? valuesFoundAsStrings.put(possibleValue.getLabel(), possibleValue); valuesNotFound.remove(possibleValue.getLabel()); } From f54b2b79db45965d3ffffd0d17a9be63ede4f38e Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Mon, 6 Jan 2025 16:35:06 -0600 Subject: [PATCH 73/84] CE-1955 Add support for value-mapping on wide-mode associated fields --- .../BulkInsertPrepareValueMappingStep.java | 22 ++++++++++++++++--- .../insert/mapping/BulkLoadValueMapper.java | 11 ++++++++-- ...licitFieldNameSuffixIndexBasedMapping.java | 10 +++++++++ ...tFieldNameSuffixIndexBasedMappingTest.java | 7 ++++-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java index 471f0661..8309531e 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareValueMappingStep.java @@ -106,8 +106,13 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep runBackendStepInput.addValue("valueMappingFieldIndex", valueMappingFieldIndex); - String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); - TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fullFieldName); + String fullFieldName = fieldNamesToDoValueMapping.get(valueMappingFieldIndex); + String fieldNameWithoutWideSuffix = fullFieldName; + if(fieldNameWithoutWideSuffix.contains(",")) + { + fieldNameWithoutWideSuffix = fieldNameWithoutWideSuffix.replaceFirst(",.*", ""); + } + TableAndField tableAndField = getTableAndField(runBackendStepInput.getValueString("tableName"), fieldNameWithoutWideSuffix); runBackendStepInput.addValue("valueMappingField", new QFrontendFieldMetaData(tableAndField.field())); runBackendStepInput.addValue("valueMappingFullFieldName", fullFieldName); @@ -213,6 +218,17 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepInput.getValue("bulkInsertMapping"); + List wideAssociationIndexes = null; + if(fullFieldName.contains(",")) + { + wideAssociationIndexes = new ArrayList<>(); + String indexes = fullFieldName.substring(fullFieldName.lastIndexOf(",") + 1); + for(String index : indexes.split("\\.")) + { + wideAssociationIndexes.add(Integer.parseInt(index)); + } + } + String associationNameChain = null; if(fullFieldName.contains(".")) { @@ -227,7 +243,7 @@ public class BulkInsertPrepareValueMappingStep implements BackendStep { Set values = new LinkedHashSet<>(); BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; - Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow); + Map fieldIndexes = bulkInsertMapping.getFieldIndexes(table, associationNameChain, headerRow, wideAssociationIndexes); int index = fieldIndexes.get(field.getName()); while(fileToRowsInterface.hasNext()) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java index 641e11bb..32b30c29 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueMapper.java @@ -94,12 +94,19 @@ public class BulkLoadValueMapper QFieldMetaData field = table.getField(valueEntry.getKey()); Serializable value = valueEntry.getValue(); + String fieldNamePlusWideIndex = field.getName(); + if(record.getBackendDetail("wideAssociationIndexes") != null) + { + ArrayList indexes = (ArrayList) record.getBackendDetail("wideAssociationIndexes"); + fieldNamePlusWideIndex += "," + StringUtils.join(",", indexes); + } + /////////////////// // value mappin' // /////////////////// - if(mappingForTable.containsKey(field.getName()) && value != null) + if(mappingForTable.containsKey(fieldNamePlusWideIndex) && value != null) { - Serializable mappedValue = mappingForTable.get(field.getName()).get(ValueUtils.getValueAsString(value)); + Serializable mappedValue = mappingForTable.get(fieldNamePlusWideIndex).get(ValueUtils.getValueAsString(value)); if(mappedValue != null) { value = mappedValue; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java index 86a5faf2..2767c061 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; +import com.google.gson.reflect.TypeToken; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -185,6 +186,15 @@ public class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMapping implem } } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // stash the wide-association indexes in records, so that in the value mapper, we know if if this is, e.g., ,1, or ,2.3, for value-mapping // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(CollectionUtils.nullSafeHasContents(wideAssociationIndexes)) + { + ArrayList indexesArrayList = CollectionUtils.useOrWrap(wideAssociationIndexes, new TypeToken<>() {}); + record.addBackendDetail("wideAssociationIndexes", indexesArrayList); + } + return record; } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java index 62eeb607..3a8677bb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest.java @@ -53,7 +53,7 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B { String csv = """ orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 - 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, two 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 """; @@ -74,6 +74,9 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B "orderLine.quantity,2", "Quantity 3" )) .withMappedAssociations(List.of("orderLine")) + .withFieldNameToValueMapping(Map.of( + "orderLine.quantity,2", Map.of("two", 2) + )) .withTableName(TestUtils.TABLE_NAME_ORDER) .withLayout(BulkInsertMapping.Layout.WIDE) .withHasHeaderRow(true); @@ -85,7 +88,7 @@ class WideRowsToRecordWithExplicitFieldNameSuffixIndexBasedMappingTest extends B assertEquals(1, order.getValueInteger("orderNo")); assertEquals("Homer", order.getValueString("shipToName")); assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); - assertEquals(List.of(12, 500, 1), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(12, 500, 2), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); assertEquals(1, ((List) order.getBackendDetail("fileRows")).size()); assertEquals("Row 2", order.getAssociatedRecords().get("orderLine").get(0).getBackendDetail("rowNos")); From 5171af1c95968b72f25873e28269502be3ae9c7f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 7 Jan 2025 11:22:33 -0600 Subject: [PATCH 74/84] CE-1955 Adjust help text re: headers --- .../bulk/insert/BulkInsertPrepareFileUploadStep.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java index 26ebc729..ea4810a9 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertPrepareFileUploadStep.java @@ -99,8 +99,9 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep

    Your file can contain any number of columns. You will be prompted to map fields from the ${tableLabel} table to columns from your file or default values for all records that - you are loading on the next screen. It is optional (though encouraged) whether you include - a header row in your file. For Excel files, only the first sheet in the workbook will be used.


    + you are loading on the next screen. It is optional whether you include a header row in your + file (though it is encouraged, and is the only way to received suggested field mappings). + For Excel files, only the first sheet in the workbook will be used.


    """); if(listFieldsInHelpText) @@ -136,8 +137,9 @@ public class BulkInsertPrepareFileUploadStep implements BackendStep

    Your file can contain any number of columns. You will be prompted to map fields from the ${tableLabel} table to columns from your file or default values for all records that - you are loading on the next screen. It is optional (though encouraged) whether you include - a header row in your file. For Excel files, only the first sheet in the workbook will be used.


    + you are loading on the next screen. It is optional whether you include a header row in your + file (though it is encouraged, and is the only way to received suggested field mappings). + For Excel files, only the first sheet in the workbook will be used.


    """); if(listFieldsInHelpText) From bcedb566ff53bf6640574ceccac338b80f0e377c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 7 Jan 2025 11:22:49 -0600 Subject: [PATCH 75/84] CE-1955 Add info summary line re: number of records processed; also return early if all records have mapping errors... this could lead to some spoon feeding, but, is working better now, so. --- .../bulk/insert/BulkInsertTransformStep.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f113f76f..f406608b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -241,8 +241,10 @@ public class BulkInsertTransformStep extends AbstractTransformStep { ///////////////////////////////////////////////////////////////////////////////// // skip the rest of this method if there aren't any records w/o errors in them // + // but, advance our counter before we return. // ///////////////////////////////////////////////////////////////////////////////// this.rowsProcessed += recordsInThisPage; + return; } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -508,6 +510,15 @@ public class BulkInsertTransformStep extends AbstractTransformStep ArrayList rs = new ArrayList<>(); String tableLabel = table == null ? "" : table.getLabel(); + ProcessSummaryLine recordsProcessedLine = new ProcessSummaryLine(Status.INFO); + recordsProcessedLine.setCount(rowsProcessed); + rs.add(recordsProcessedLine); + recordsProcessedLine.withMessageSuffix(" processed from the file."); + recordsProcessedLine.withSingularFutureMessage("record was"); + recordsProcessedLine.withSingularPastMessage("record was"); + recordsProcessedLine.withPluralFutureMessage("records were"); + recordsProcessedLine.withPluralPastMessage("records were"); + String noWarningsSuffix = processSummaryWarningsAndErrorsRollup.countWarnings() == 0 ? "" : " with no warnings"; okSummary.setSingularFutureMessage(tableLabel + " record will be inserted" + noWarningsSuffix + "."); okSummary.setPluralFutureMessage(tableLabel + " records will be inserted" + noWarningsSuffix + "."); From f7cbf9d1c240f46ed767642bc41f9e58dc0fbc8f Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 7 Jan 2025 11:30:19 -0600 Subject: [PATCH 76/84] CE-1955 Make sure to skip blank rows (e.g., no columns had a value) --- .../bulk/insert/mapping/FlatRowsToRecord.java | 14 ++++++++++++-- .../bulk/insert/mapping/FlatRowsToRecordTest.java | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java index 310fd8db..1da11588 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -62,12 +62,22 @@ public class FlatRowsToRecord implements RowsToRecordInterface QRecord record = new QRecord(); BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, row); + boolean anyValuesFromFileUsed = false; for(QFieldMetaData field : table.getFields().values()) { - setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName())); + if(setValueOrDefault(record, field, null, mapping, row, fieldIndexes.get(field.getName()))) + { + anyValuesFromFileUsed = true; + } } - rs.add(record); + ////////////////////////////////////////////////////////////////////////// + // avoid building empty records (e.g., "past the end" of an Excel file) // + ////////////////////////////////////////////////////////////////////////// + if(anyValuesFromFileUsed) + { + rs.add(record); + } } BulkLoadValueMapper.valueMapping(rs, mapping, table); diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java index 5237fcf5..ab9486a9 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -56,7 +56,8 @@ class FlatRowsToRecordTest extends BaseTest new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, new Serializable[] { 2, "Marge", "Simpson", false, "" }, new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, - new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" }, + new Serializable[] { "", "", "", "", "" } // all blank row (we can get these at the bottoms of files) - make sure it doesn't become a record. )); BulkLoadFileRow header = fileToRows.next(); From 5ad42164346d1fbcfa70be145fc106c9e423a968 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 7 Jan 2025 11:30:48 -0600 Subject: [PATCH 77/84] CE-1955 Replace _ with space in allCapsToMixedCase (for common use-case of an enum constant) --- .../com/kingsrook/qqq/backend/core/utils/StringUtils.java | 4 ++-- .../com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java index a8348756..751caf25 100755 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/StringUtils.java @@ -82,7 +82,7 @@ public class StringUtils /******************************************************************************* - ** allCapsToMixedCase - ie, UNIT CODE -> Unit Code + ** allCapsToMixedCase - ie, UNIT_CODE -> Unit Code ** ** @param input ** @return @@ -127,7 +127,7 @@ public class StringUtils return (input); } - return (rs.toString()); + return (rs.toString().replace('_', ' ')); } diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java index b2cad605..f13ccf4f 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/StringUtilsTest.java @@ -105,6 +105,7 @@ class StringUtilsTest extends BaseTest assertEquals("Foo bar", StringUtils.allCapsToMixedCase("FOo bar")); assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOo BAr")); assertEquals("foo bar", StringUtils.allCapsToMixedCase("foo bar")); + assertEquals("Foo Bar", StringUtils.allCapsToMixedCase("FOO_BAR")); } From 387804acffe25a1245f19f43719409174d41ab85 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 7 Jan 2025 11:34:39 -0600 Subject: [PATCH 78/84] CE-1955 Add "H" to pattern check for date-time/hours (doesn't appear in docs i can find, but does appear in a file i'm working with, so... probably valid) --- .../bulk/insert/filehandling/XlsxFileToRows.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index de0b1ccb..a980f6dd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -202,7 +202,7 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows Date: Fri, 10 Jan 2025 15:42:17 -0600 Subject: [PATCH 79/84] CE-1955 Revert splitting out records with mapping errors, to help do less spoon-feeding; also, avoid double-running customizer --- .../core/actions/tables/InsertAction.java | 12 +- .../bulk/insert/BulkInsertTransformStep.java | 131 ++++++++---------- .../mapping/BulkLoadValueTypeError.java | 2 +- .../insert/BulkInsertTransformStepTest.java | 58 ++++++++ 4 files changed, 120 insertions(+), 83 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java index 1c799b34..64a4ddc3 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/InsertAction.java @@ -122,7 +122,7 @@ public class InsertAction extends AbstractQActionFunction preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); - if(preInsertCustomizer.isPresent()) - { - runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); - } + Optional preInsertCustomizer = didAlreadyRunCustomizer ? Optional.empty() : QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + runPreInsertCustomizerIfItIsTime(insertInput, isPreview, preInsertCustomizer, AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); setDefaultValuesInRecords(table, insertInput.getRecords()); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java index f406608b..410fccf2 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStep.java @@ -187,65 +187,8 @@ public class BulkInsertTransformStep extends AbstractTransformStep @Override public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException { - int recordsInThisPage = runBackendStepInput.getRecords().size(); - QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); - - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // split the records into 2 lists: those w/ errors (e.g., from the bulk-load mapping), and those that are okay // - ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutAnyErrors = new ArrayList<>(); - List recordsWithSomeErrors = new ArrayList<>(); - for(QRecord record : runBackendStepInput.getRecords()) - { - List errorsFromAssociations = getErrorsFromAssociations(record); - if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) - { - List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); - recordErrors.addAll(errorsFromAssociations); - record.setErrors(recordErrors); - } - - if(CollectionUtils.nullSafeHasContents(record.getErrors())) - { - recordsWithSomeErrors.add(record); - } - else - { - recordsWithoutAnyErrors.add(record); - } - } - - ////////////////////////////////////////////////////////////////// - // propagate errors that came into this step out to the summary // - ////////////////////////////////////////////////////////////////// - if(!recordsWithSomeErrors.isEmpty()) - { - for(QRecord record : recordsWithSomeErrors) - { - for(QErrorMessage error : record.getErrors()) - { - if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) - { - processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); - addToErrorToExampleRowValueMap(rollableValueError, record); - } - else - { - processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); - } - } - } - } - - if(recordsWithoutAnyErrors.isEmpty()) - { - ///////////////////////////////////////////////////////////////////////////////// - // skip the rest of this method if there aren't any records w/o errors in them // - // but, advance our counter before we return. // - ///////////////////////////////////////////////////////////////////////////////// - this.rowsProcessed += recordsInThisPage; - return; - } + QTableMetaData table = QContext.getQInstance().getTable(runBackendStepInput.getTableName()); + List records = runBackendStepInput.getRecords(); ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // set up an insert-input, which will be used as input to the pre-customizer as well as for additional validations // @@ -253,7 +196,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep InsertInput insertInput = new InsertInput(); insertInput.setInputSource(QInputSource.USER); insertInput.setTableName(runBackendStepInput.getTableName()); - insertInput.setRecords(recordsWithoutAnyErrors); + insertInput.setRecords(records); insertInput.setSkipUniqueKeyCheck(true); ////////////////////////////////////////////////////////////////////// @@ -262,34 +205,41 @@ public class BulkInsertTransformStep extends AbstractTransformStep // we do this, in case it needs to, for example, adjust values that // // are part of a unique key // ////////////////////////////////////////////////////////////////////// - Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); + boolean didAlreadyRunCustomizer = false; + Optional preInsertCustomizer = QCodeLoader.getTableCustomizer(table, TableCustomizers.PRE_INSERT_RECORD.getRole()); if(preInsertCustomizer.isPresent()) { AbstractPreInsertCustomizer.WhenToRun whenToRun = preInsertCustomizer.get().whenToRunPreInsert(insertInput, true); if(WhenToRun.BEFORE_ALL_VALIDATIONS.equals(whenToRun) || WhenToRun.BEFORE_UNIQUE_KEY_CHECKS.equals(whenToRun)) { - List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, recordsWithoutAnyErrors, true); + List recordsAfterCustomizer = preInsertCustomizer.get().preInsert(insertInput, records, true); runBackendStepInput.setRecords(recordsAfterCustomizer); - /////////////////////////////////////////////////////////////////////////////////////// - // todo - do we care if the customizer runs both now, and in the validation below? // - // right now we'll let it run both times, but maybe that should be protected against // - /////////////////////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // so we used to have a comment here asking "do we care if the customizer runs both now, and in the validation below?" // + // when implementing Bulk Load V2, we were seeing that some customizers were adding errors to records, both now, and // + // when they ran below. so, at that time, we added this boolean, to track and avoid the double-run... // + // we could also imagine this being a setting on the pre-insert customizer, similar to its whenToRun attribute... // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + didAlreadyRunCustomizer = true; } } + /////////////////////////////////////////////////////////////////////////////// + // If the table has unique keys - then capture all values on these records // + // for each key and set up a processSummaryLine for each of the table's UK's // + /////////////////////////////////////////////////////////////////////////////// Map>> existingKeys = new HashMap<>(); List uniqueKeys = CollectionUtils.nonNullList(table.getUniqueKeys()); for(UniqueKey uniqueKey : uniqueKeys) { - existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, recordsWithoutAnyErrors, uniqueKey).keySet()); + existingKeys.put(uniqueKey, UniqueKeyHelper.getExistingKeys(null, table, records, uniqueKey).keySet()); ukErrorSummaries.computeIfAbsent(uniqueKey, x -> new ProcessSummaryLineWithUKSampleValues(Status.ERROR)); } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// // on the validate step, we haven't read the full file, so we don't know how many rows there are - thus // // record count is null, and the ValidateStep won't be setting status counters - so - do it here in that case. // - // todo - move this up (before the early return?) // ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// if(runBackendStepInput.getStepName().equals(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE)) { @@ -311,14 +261,14 @@ public class BulkInsertTransformStep extends AbstractTransformStep // Note, we want to do our own UK checking here, even though InsertAction also tries to do it, because InsertAction // // will only be getting the records in pages, but in here, we'll track UK's across pages!! // ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(recordsWithoutAnyErrors, existingKeys, uniqueKeys, table); + List recordsWithoutUkErrors = getRecordsWithoutUniqueKeyErrors(records, existingKeys, uniqueKeys, table); ///////////////////////////////////////////////////////////////////////////////// // run all validation from the insert action - in Preview mode (boolean param) // ///////////////////////////////////////////////////////////////////////////////// insertInput.setRecords(recordsWithoutUkErrors); InsertAction insertAction = new InsertAction(); - insertAction.performValidations(insertInput, true); + insertAction.performValidations(insertInput, true, didAlreadyRunCustomizer); List validationResultRecords = insertInput.getRecords(); ///////////////////////////////////////////////////////////////// @@ -327,12 +277,28 @@ public class BulkInsertTransformStep extends AbstractTransformStep List outputRecords = new ArrayList<>(); for(QRecord record : validationResultRecords) { + List errorsFromAssociations = getErrorsFromAssociations(record); + if(CollectionUtils.nullSafeHasContents(errorsFromAssociations)) + { + List recordErrors = Objects.requireNonNullElseGet(record.getErrors(), () -> new ArrayList<>()); + recordErrors.addAll(errorsFromAssociations); + record.setErrors(recordErrors); + } + if(CollectionUtils.nullSafeHasContents(record.getErrors())) { for(QErrorMessage error : record.getErrors()) { - processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); - addToErrorToExampleRowMap(error.getMessage(), record); + if(error instanceof AbstractBulkLoadRollableValueError rollableValueError) + { + processSummaryWarningsAndErrorsRollup.addError(rollableValueError.getMessageToUseAsProcessSummaryRollupKey(), null); + addToErrorToExampleRowValueMap(rollableValueError, record); + } + else + { + processSummaryWarningsAndErrorsRollup.addError(error.getMessage(), null); + addToErrorToExampleRowMap(error.getMessage(), record); + } } } else if(CollectionUtils.nullSafeHasContents(record.getWarnings())) @@ -356,7 +322,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep } runBackendStepOutput.setRecords(outputRecords); - this.rowsProcessed += recordsInThisPage; + this.rowsProcessed += records.size(); } @@ -574,7 +540,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ProcessSummaryLine line = entry.getValue(); List rowValues = errorToExampleRowValueMap.get(message); String exampleOrFull = rowValues.size() < line.getCount() ? "Example " : ""; - line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Values:"); + line.setMessageSuffix(line.getMessageSuffix() + periodIfNeeded(line.getMessageSuffix()) + " " + exampleOrFull + "Values:"); line.setBulletsOfText(new ArrayList<>(rowValues.stream().map(String::valueOf).toList())); } else if(errorToExampleRowsMap.containsKey(message)) @@ -582,7 +548,7 @@ public class BulkInsertTransformStep extends AbstractTransformStep ProcessSummaryLine line = entry.getValue(); List rowDescriptions = errorToExampleRowsMap.get(message); String exampleOrFull = rowDescriptions.size() < line.getCount() ? "Example " : ""; - line.setMessageSuffix(line.getMessageSuffix() + ". " + exampleOrFull + "Records:"); + line.setMessageSuffix(line.getMessageSuffix() + periodIfNeeded(line.getMessageSuffix()) + " " + exampleOrFull + "Records:"); line.setBulletsOfText(new ArrayList<>(rowDescriptions.stream().map(String::valueOf).toList())); } } @@ -594,6 +560,21 @@ public class BulkInsertTransformStep extends AbstractTransformStep + /*************************************************************************** + * + ***************************************************************************/ + private String periodIfNeeded(String input) + { + if(input != null && input.matches(".*\\. *$")) + { + return (""); + } + + return ("."); + } + + + /*************************************************************************** ** ***************************************************************************/ diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java index 6bee0449..5add5f9b 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkLoadValueTypeError.java @@ -44,7 +44,7 @@ public class BulkLoadValueTypeError extends AbstractBulkLoadRollableValueError *******************************************************************************/ public BulkLoadValueTypeError(String fieldName, Serializable value, QFieldType type, String fieldLabel) { - super("Value [" + value + "] for field [" + fieldLabel + "] could not be converted to type [" + type + "]"); + super("Cannot convert value [" + value + "] for field [" + fieldLabel + "] to type [" + type.getMixedCaseLabel() + "]"); this.value = value; this.type = type; this.fieldLabel = fieldLabel; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java index 5a8ae6d3..386050eb 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertTransformStepTest.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadRecordUtils; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkLoadValueTypeError; @@ -367,4 +368,61 @@ class BulkInsertTransformStepTest extends BaseTest .hasCount(1); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testPropagationOfErrorsFromAssociations() throws QException + { + //////////////////////////////////////////////// + // set line item lineNumber field as required // + //////////////////////////////////////////////// + QInstance instance = TestUtils.defineInstance(); + instance.getTable(TestUtils.TABLE_NAME_LINE_ITEM).getField("lineNumber").setIsRequired(true); + reInitInstanceInContext(instance); + + QContext.getQSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_TYPE_STORE, 1); + + /////////////////////////////////////////// + // setup & run the bulk insert transform // + /////////////////////////////////////////// + BulkInsertTransformStep bulkInsertTransformStep = new BulkInsertTransformStep(); + RunBackendStepInput input = new RunBackendStepInput(); + RunBackendStepOutput output = new RunBackendStepOutput(); + + input.setTableName(TestUtils.TABLE_NAME_ORDER); + input.setStepName(StreamedETLWithFrontendProcess.STEP_NAME_VALIDATE); + input.setRecords(ListBuilder.of( + new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord()), + new QRecord().withValue("storeId", 1).withAssociatedRecord("orderLine", new QRecord().withError(new BadInputStatusMessage("some mapping error"))) + )); + + bulkInsertTransformStep.preRun(input, output); + bulkInsertTransformStep.runOnePage(input, output); + ArrayList processSummary = bulkInsertTransformStep.getProcessSummary(output, false); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("some mapping error") + .hasMessageContaining("Records:") + .hasStatus(Status.ERROR) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("records were processed from the file") + .hasStatus(Status.INFO) + .hasCount(2); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + + ProcessSummaryAssert.assertThat(processSummary) + .hasLineWithMessageContaining("Order Line record will be inserted") + .hasStatus(Status.OK) + .hasCount(1); + } + } \ No newline at end of file From 70b569c2cacb3c94496561abedd73f835de9e775 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 Jan 2025 15:43:38 -0600 Subject: [PATCH 80/84] CE-1955 Memoize groupByAllIndexesFromTable to avoid wasting lots of arrayLists; add todo about maybe only doing grouping if there is a mapped child table... --- .../bulk/insert/mapping/TallRowsToRecord.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java index bf67522c..9aab3ab8 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -55,6 +55,7 @@ public class TallRowsToRecord implements RowsToRecordInterface private static final QLogger LOG = QLogger.getLogger(TallRowsToRecord.class); private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + private Memoization> groupByAllIndexesFromTableMemoization = new Memoization<>(); @@ -96,6 +97,10 @@ public class TallRowsToRecord implements RowsToRecordInterface continue; } + /////////////////////////////////////////////////////////////////////////////// + // maybe todo - some version of - only do this if there are mapped children? // + /////////////////////////////////////////////////////////////////////////////// + if(rowsForCurrentRecord.isEmpty()) { /////////////////////////////////// @@ -154,8 +159,11 @@ public class TallRowsToRecord implements RowsToRecordInterface ***************************************************************************/ private List groupByAllIndexesFromTable(BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow headerRow, String name) throws QException { - Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); - return new ArrayList<>(fieldIndexes.values()); + return ((groupByAllIndexesFromTableMemoization.getResult(table.getName(), (n) -> + { + Map fieldIndexes = mapping.getFieldIndexes(table, name, headerRow); + return new ArrayList<>(fieldIndexes.values()); + })).orElse(null)); } @@ -166,6 +174,7 @@ public class TallRowsToRecord implements RowsToRecordInterface private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException { QRecord record = new QRecord(); + record.setTableName(table.getName()); BulkLoadRecordUtils.addBackendDetailsAboutFileRows(record, CollectionUtils.useOrWrap(rows, new TypeToken>() {})); Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); @@ -273,6 +282,10 @@ public class TallRowsToRecord implements RowsToRecordInterface break; } + /////////////////////////////////////////////////////////////////////////////// + // maybe todo - some version of - only do this if there are mapped children? // + /////////////////////////////////////////////////////////////////////////////// + List rowGroupByValues = getGroupByValues(row, groupByIndexes); if(rowGroupByValues == null) { From e2c7748a4bc64a60041bbcd64d012fe9ecb64fb9 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Fri, 10 Jan 2025 16:11:14 -0600 Subject: [PATCH 81/84] CE-1955 Update getValueAsInstant to handle a single-digit hour, by assuming a leading 0 on it. --- .../qqq/backend/core/utils/ValueUtils.java | 62 ++++++++++++++----- .../backend/core/utils/ValueUtilsTest.java | 3 + 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java index f2ee6802..1e791abd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/utils/ValueUtils.java @@ -590,11 +590,35 @@ public class ValueUtils try { + ///////////////////////////////////////////////////////////////////////////////////// + // first assume the instant is perfectly formatted, as in: 2007-12-03T10:15:30.00Z // + ///////////////////////////////////////////////////////////////////////////////////// return Instant.parse(s); } catch(DateTimeParseException e) { - return tryAlternativeInstantParsing(s, e); + try + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // if the string isn't quite the right format, try some alternates that are common and fairly un-vague // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + return tryAlternativeInstantParsing(s, e); + } + catch(DateTimeParseException dtpe) + { + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // we commonly receive date-times with only a single-digit hour after the space, which fails tryAlternativeInstantParsing. // + // so if we see what looks like that pattern, zero-pad the hour, and try the alternative parse patterns again. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(s.matches(".* \\d:.*")) + { + return tryAlternativeInstantParsing(s.replaceFirst(" (\\d):", " 0$1:"), e); + } + else + { + throw (dtpe); + } + } } } else @@ -617,11 +641,12 @@ public class ValueUtils /******************************************************************************* ** *******************************************************************************/ - private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) + private static Instant tryAlternativeInstantParsing(String s, DateTimeParseException e) throws DateTimeParseException { - ////////////////////// - // 1999-12-31T12:59 // - ////////////////////// + //////////////////////////////////////////////////////////////////// + // 1999-12-31T12:59 // + // missing seconds & zone - but we're happy to assume :00 and UTC // + //////////////////////////////////////////////////////////////////// if(s.matches("^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}$")) { ////////////////////////// @@ -630,27 +655,30 @@ public class ValueUtils return Instant.parse(s + ":00Z"); } - /////////////////////////// - // 1999-12-31 12:59:59.0 // - /////////////////////////// + /////////////////////////////////////////////////////////////// + // 1999-12-31 12:59:59.0 // + // fractional seconds and no zone - truncate, and assume UTC // + /////////////////////////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.0$")) { s = s.replaceAll(" ", "T").replaceAll("\\..*$", "Z"); return Instant.parse(s); } - ///////////////////////// - // 1999-12-31 12:59:59 // - ///////////////////////// + //////////////////////////////////////////// + // 1999-12-31 12:59:59 // + // Missing 'T' and 'Z', so just add those // + //////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$")) { s = s.replaceAll(" ", "T") + "Z"; return Instant.parse(s); } - ////////////////////// - // 1999-12-31 12:59 // - ////////////////////// + ///////////////////////////////////////////// + // 1999-12-31 12:59 // + // missing T, seconds, and Z - add 'em all // + ///////////////////////////////////////////// else if(s.matches("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}$")) { s = s.replaceAll(" ", "T") + ":00Z"; @@ -661,12 +689,18 @@ public class ValueUtils { try { + //////////////////////////////////////////////////////// + // such as '2011-12-03T10:15:30+01:00[Europe/Paris]'. // + //////////////////////////////////////////////////////// return LocalDateTime.parse(s, DateTimeFormatter.ISO_ZONED_DATE_TIME).toInstant(ZoneOffset.UTC); } catch(DateTimeParseException e2) { try { + /////////////////////////////////////////////////////// + // also includes such as '2011-12-03T10:15:30+01:00' // + /////////////////////////////////////////////////////// return LocalDateTime.parse(s, DateTimeFormatter.ISO_DATE_TIME).toInstant(ZoneOffset.UTC); } catch(Exception e3) diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java index d61a56c1..ee1f69f1 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/ValueUtilsTest.java @@ -251,6 +251,9 @@ class ValueUtilsTest extends BaseTest assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("a,b")); assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant("1980/05/31")); assertThat(assertThrows(QValueException.class, () -> ValueUtils.getValueAsInstant(new Object())).getMessage()).contains("Unsupported class"); + + expected = Instant.parse("1980-05-31T01:30:00Z"); + assertEquals(expected, ValueUtils.getValueAsInstant("1980-05-31 1:30:00")); } From b397c4da080bf61b20ccd03bc75d6a48ed3edfbc Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Sat, 11 Jan 2025 20:13:51 -0600 Subject: [PATCH 82/84] CE-1955 Add plugins for QInstanceEnricher --- .../core/instances/QInstanceEnricher.java | 85 +++++++++++++++++++ .../QInstanceEnricherPluginInterface.java | 40 +++++++++ .../core/instances/QInstanceEnricherTest.java | 46 ++++++++++ 3 files changed, 171 insertions(+) create mode 100644 qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index a0d926c0..fbe0ea54 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -27,6 +27,7 @@ import java.lang.reflect.Method; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -34,6 +35,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; @@ -41,6 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -52,6 +55,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.DynamicDefaultValueB import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; @@ -120,6 +124,8 @@ public class QInstanceEnricher ////////////////////////////////////////////////////////////////////////////////////////////////// private static final Map labelMappings = new LinkedHashMap<>(); + private static ListingHash, QInstanceEnricherPluginInterface> enricherPlugins = new ListingHash<>(); + /******************************************************************************* @@ -181,6 +187,7 @@ public class QInstanceEnricher } enrichJoins(); + enrichInstance(); ////////////////////////////////////////////////////////////////////////////// // if the instance DOES have 1 or more scheduler, but no schedulable types, // @@ -197,6 +204,16 @@ public class QInstanceEnricher + /*************************************************************************** + ** + ***************************************************************************/ + private void enrichInstance() + { + runPlugins(QInstance.class, qInstance, qInstance); + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -261,6 +278,14 @@ public class QInstanceEnricher } } } + + /////////////////////////////////////////// + // run plugins on joins if there are any // + /////////////////////////////////////////// + for(QJoinMetaData join : qInstance.getJoins().values()) + { + runPlugins(QJoinMetaData.class, join, qInstance); + } } catch(Exception e) { @@ -276,6 +301,7 @@ public class QInstanceEnricher private void enrichWidget(QWidgetMetaDataInterface widgetMetaData) { enrichPermissionRules(widgetMetaData); + runPlugins(QWidgetMetaDataInterface.class, widgetMetaData, qInstance); } @@ -286,6 +312,7 @@ public class QInstanceEnricher private void enrichBackend(QBackendMetaData qBackendMetaData) { qBackendMetaData.enrich(); + runPlugins(QBackendMetaData.class, qBackendMetaData, qInstance); } @@ -326,6 +353,7 @@ public class QInstanceEnricher enrichPermissionRules(table); enrichAuditRules(table); + runPlugins(QTableMetaData.class, table, qInstance); } @@ -416,6 +444,7 @@ public class QInstanceEnricher } enrichPermissionRules(process); + runPlugins(QProcessMetaData.class, process, qInstance); } @@ -537,6 +566,8 @@ public class QInstanceEnricher field.withBehavior(DynamicDefaultValueBehavior.MODIFY_DATE); } } + + runPlugins(QFieldMetaData.class, field, qInstance); } @@ -608,6 +639,7 @@ public class QInstanceEnricher ensureAppSectionMembersAreAppChildren(app); enrichPermissionRules(app); + runPlugins(QAppMetaData.class, app, qInstance); } @@ -755,6 +787,7 @@ public class QInstanceEnricher } enrichPermissionRules(report); + runPlugins(QReportMetaData.class, report, qInstance); } @@ -1408,6 +1441,58 @@ public class QInstanceEnricher } } } + + runPlugins(QPossibleValueSource.class, possibleValueSource, qInstance); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void addEnricherPlugin(QInstanceEnricherPluginInterface plugin) + { + Optional enrichMethod = Arrays.stream(plugin.getClass().getDeclaredMethods()) + .filter(m -> m.getName().equals("enrich") + && m.getParameterCount() == 2 + && !m.getParameterTypes()[0].equals(Object.class) + && m.getParameterTypes()[1].equals(QInstance.class) + ).findFirst(); + + if(enrichMethod.isPresent()) + { + Class parameterType = enrichMethod.get().getParameterTypes()[0]; + enricherPlugins.add(parameterType, plugin); + } + else + { + LOG.warn("Could not find enrich method on enricher plugin [" + plugin.getClass().getName() + "] (to infer type being enriched) - this plugin will not be used."); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + public static void removeAllEnricherPlugins() + { + enricherPlugins.clear(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void runPlugins(Class c, T t, QInstance qInstance) + { + for(QInstanceEnricherPluginInterface plugin : CollectionUtils.nonNullList(enricherPlugins.get(c))) + { + @SuppressWarnings("unchecked") + QInstanceEnricherPluginInterface castedPlugin = (QInstanceEnricherPluginInterface) plugin; + castedPlugin.enrich(t, qInstance); + } } diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java new file mode 100644 index 00000000..9a900772 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java @@ -0,0 +1,40 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.instances.validation.plugins; + + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** Interface for additional / optional enrichment to be done on q instance members. + ** Some may be provided by QQQ - others can be defined by applications. + *******************************************************************************/ +public interface QInstanceEnricherPluginInterface +{ + + /******************************************************************************* + ** + *******************************************************************************/ + void enrich(T object, QInstance qInstance); + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 20f87686..7b356218 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; @@ -47,6 +48,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_GREETINGS; import static com.kingsrook.qqq.backend.core.utils.TestUtils.APP_NAME_MISCELLANEOUS; @@ -66,6 +68,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; class QInstanceEnricherTest extends BaseTest { + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + QInstanceEnricher.removeAllEnricherPlugins(); + } + + + /******************************************************************************* ** Test that a table missing a label gets the default label applied (name w/ UC-first). ** @@ -572,4 +585,37 @@ class QInstanceEnricherTest extends BaseTest assertEquals("My Field", qInstance.getProcess("test").getFrontendStep("screen").getViewFields().get(0).getLabel()); } + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldPlugIn() + { + QInstance qInstance = TestUtils.defineInstance(); + + QInstanceEnricher.addEnricherPlugin(new QInstanceEnricherPluginInterface() + { + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void enrich(QFieldMetaData field, QInstance qInstance) + { + if(field != null) + { + field.setLabel(field.getLabel() + " Plugged"); + } + } + }); + + new QInstanceEnricher(qInstance).enrich(); + + qInstance.getTables().values().forEach(table -> table.getFields().values().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + qInstance.getProcesses().values().forEach(process -> process.getInputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + qInstance.getProcesses().values().forEach(process -> process.getOutputFields().forEach(field -> assertThat(field.getLabel()).endsWith("Plugged"))); + + } + } From 1fae1c5e2a3aa33807ae7867bb4d70cc381721da Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 14 Jan 2025 10:08:42 -0600 Subject: [PATCH 83/84] Move enricher plugin to enrichment package --- .../qqq/backend/core/instances/QInstanceEnricher.java | 2 +- .../plugins/QInstanceEnricherPluginInterface.java | 4 ++-- .../qqq/backend/core/instances/QInstanceEnricherTest.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/{validation => enrichment}/plugins/QInstanceEnricherPluginInterface.java (93%) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java index fbe0ea54..809508cd 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java @@ -43,7 +43,7 @@ import com.kingsrook.qqq.backend.core.actions.metadata.JoinGraph; import com.kingsrook.qqq.backend.core.actions.permissions.BulkTableActionProcessPermissionChecker; import com.kingsrook.qqq.backend.core.actions.values.QCustomPossibleValueProvider; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java similarity index 93% rename from qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java rename to qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java index 9a900772..30b07f61 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/validation/plugins/QInstanceEnricherPluginInterface.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/enrichment/plugins/QInstanceEnricherPluginInterface.java @@ -1,6 +1,6 @@ /* * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2024. Kingsrook, LLC + * Copyright (C) 2021-2025. Kingsrook, LLC * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States * contact@kingsrook.com * https://github.com/Kingsrook/ @@ -19,7 +19,7 @@ * along with this program. If not, see . */ -package com.kingsrook.qqq.backend.core.instances.validation.plugins; +package com.kingsrook.qqq.backend.core.instances.enrichment.plugins; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java index 7b356218..1817b57c 100644 --- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricherTest.java @@ -27,7 +27,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import com.kingsrook.qqq.backend.core.BaseTest; -import com.kingsrook.qqq.backend.core.instances.validation.plugins.QInstanceEnricherPluginInterface; +import com.kingsrook.qqq.backend.core.instances.enrichment.plugins.QInstanceEnricherPluginInterface; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; From c91a7903ba9241db7e6a7b09c480761385516d9c Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 16 Jan 2025 10:52:59 -0600 Subject: [PATCH 84/84] Haandle FORMULA type by using 'raw value' as string (seems to be the evaluated value) --- .../bulk/insert/filehandling/XlsxFileToRows.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java index a980f6dd..289b90ee 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -32,11 +32,13 @@ import java.util.stream.Stream; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.ReadingOptions; import org.dhatim.fastexcel.reader.Row; import org.dhatim.fastexcel.reader.Sheet; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -163,9 +165,13 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows + case FORMULA -> { - LOG.debug("cell type: " + cell.getType() + " had value string: " + cell.asString()); + return (ValueUtils.getValueAsString(cell.getRawValue())); + } + case EMPTY, ERROR -> + { + LOG.debug("Empty or Error cell", logPair("type", cell.getType()), logPair("rawValue", () -> cell.getRawValue())); return (null); } default ->