From 872dec3177b413be64bc69ad1329f4ecb2743633 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Thu, 28 Dec 2023 16:38:40 -0600 Subject: [PATCH] CE-773 change fileNameFieldName and contentsFieldName to default as null - add validation to tableBackendDetails, specifically implemented in filesystem module --- .../core/instances/QInstanceValidator.java | 5 + .../metadata/tables/QTableBackendDetails.java | 15 ++ ...AbstractFilesystemTableBackendDetails.java | 48 ++++- .../backend/module/filesystem/TestUtils.java | 7 +- ...ractFilesystemTableBackendDetailsTest.java | 195 ++++++++++++++++++ 5 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java index 4cd6bafd..725b9c4d 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java @@ -480,6 +480,11 @@ public class QInstanceValidator validateTableCustomizer(tableName, entry.getKey(), entry.getValue()); } + if(table.getBackendDetails() != null) + { + table.getBackendDetails().validate(qInstance, table, this); + } + validateTableAutomationDetails(qInstance, table); validateTableUniqueKeys(table); validateAssociatedScripts(table); diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java index 77344ec0..732fa9e4 100644 --- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java @@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer; import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface; @@ -100,4 +103,16 @@ public abstract class QTableBackendDetails return (this); } + + + /******************************************************************************* + ** + *******************************************************************************/ + public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator) + { + //////////////////////// + // noop in base class // + //////////////////////// + } + } diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java index 95d4b438..f50f1b4c 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java @@ -22,7 +22,11 @@ package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* @@ -35,11 +39,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails private RecordFormat recordFormat; private Cardinality cardinality; - /////////////////////////////////////////////////////////////////////////////////////////////////// - // todo default these to null, and give validation error if not set for a cardinality=ONE table? // - /////////////////////////////////////////////////////////////////////////////////////////////////// - private String contentsFieldName = "contents"; - private String fileNameFieldName = "fileName"; + private String contentsFieldName; + private String fileNameFieldName; + /******************************************************************************* @@ -243,4 +245,40 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails } + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator) + { + super.validate(qInstance, table, qInstanceValidator); + + String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - "; + if(qInstanceValidator.assertCondition(cardinality != null, prefix + "missing cardinality")) + { + if(cardinality.equals(Cardinality.ONE)) + { + if(qInstanceValidator.assertCondition(StringUtils.hasContent(contentsFieldName), prefix + "missing contentsFieldName, which is required for Cardinality ONE")) + { + qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(contentsFieldName), prefix + "contentsFieldName [" + contentsFieldName + "] is not a field on this table."); + } + + if(qInstanceValidator.assertCondition(StringUtils.hasContent(fileNameFieldName), prefix + "missing fileNameFieldName, which is required for Cardinality ONE")) + { + qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(fileNameFieldName), prefix + "fileNameFieldName [" + fileNameFieldName + "] is not a field on this table."); + } + + qInstanceValidator.assertCondition(recordFormat == null, prefix + "has a recordFormat, which is not allowed for Cardinality ONE"); + } + + if(cardinality.equals(Cardinality.MANY)) + { + qInstanceValidator.assertCondition(!StringUtils.hasContent(contentsFieldName), prefix + "has a contentsFieldName, which is not allowed for Cardinality MANY"); + qInstanceValidator.assertCondition(!StringUtils.hasContent(fileNameFieldName), prefix + "has a fileNameFieldName, which is not allowed for Cardinality MANY"); + qInstanceValidator.assertCondition(recordFormat != null, prefix + "missing recordFormat, which is required for Cardinality MANY"); + } + } + + } } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java index c209f6a5..4510ecd5 100644 --- a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java @@ -26,7 +26,6 @@ import java.io.File; import java.io.IOException; import java.util.List; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; @@ -142,8 +141,6 @@ public class TestUtils qInstance.addTable(defineMockPersonTable()); qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess()); - new QInstanceValidator().validate(qInstance); - return (qInstance); } @@ -249,6 +246,8 @@ public class TestUtils .withBackendDetails(new FilesystemTableBackendDetails() .withBasePath("blobs") .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") ); } @@ -269,6 +268,8 @@ public class TestUtils .withBackendDetails(new S3TableBackendDetails() .withBasePath("blobs") .withCardinality(Cardinality.ONE) + .withFileNameFieldName("fileName") + .withContentsFieldName("contents") ); } diff --git a/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java new file mode 100644 index 00000000..7fc78997 --- /dev/null +++ b/qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2023. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata; + + +import java.util.function.Consumer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.module.filesystem.BaseTest; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + + +/******************************************************************************* + ** Unit test for AbstractFilesystemTableBackendDetails + *******************************************************************************/ +class AbstractFilesystemTableBackendDetailsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testValidInstancePasses() throws QInstanceValidationException + { + new QInstanceValidator().validate(QContext.getQInstance()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMissingCardinality() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3).withBackendDetails(new FilesystemTableBackendDetails()); + }, false, "missing cardinality"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCardinalityOneIssues() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + ); + }, false, "missing contentsFieldName", "missing fileNameFieldName"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + .withContentsFieldName("foo") + .withFileNameFieldName("bar") + ); + }, false, "contentsFieldName [foo] is not a field", "fileNameFieldName [bar] is not a field"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.ONE) + .withContentsFieldName("contents") + .withFileNameFieldName("fileName") + .withRecordFormat(RecordFormat.CSV) + ); + }, false, "has a recordFormat"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testCardinalityManyIssues() throws QException + { + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.MANY) + ); + }, false, "missing recordFormat"); + + assertValidationFailureReasons((QInstance qInstance) -> + { + qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails() + .withCardinality(Cardinality.MANY) + .withRecordFormat(RecordFormat.CSV) + .withContentsFieldName("foo") + .withFileNameFieldName("bar") + ); + }, false, "has a contentsFieldName", "has a fileNameFieldName"); + } + + + + /******************************************************************************* + ** Implementation for the overloads of this name. + *******************************************************************************/ + private void assertValidationFailureReasons(Consumer setup, boolean allowExtraReasons, String... reasons) throws QException + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + setup.accept(qInstance); + new QInstanceValidator().validate(qInstance); + fail("Should have thrown validationException"); + } + catch(QInstanceValidationException e) + { + if(!allowExtraReasons) + { + int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size(); + assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons) + + "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--")); + } + + for(String reason : reasons) + { + assertReason(reason, e); + } + } + } + + + + /******************************************************************************* + ** Assert that an instance is valid! + *******************************************************************************/ + private void assertValidationSuccess(Consumer setup) throws QException + { + try + { + QInstance qInstance = TestUtils.defineInstance(); + setup.accept(qInstance); + new QInstanceValidator().validate(qInstance); + } + catch(QInstanceValidationException e) + { + fail("Expected no validation errors, but received: " + e.getMessage()); + } + } + + + + /******************************************************************************* + ** utility method for asserting that a specific reason string is found within + ** the list of reasons in the QInstanceValidationException. + ** + *******************************************************************************/ + private void assertReason(String reason, QInstanceValidationException e) + { + assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)"); + assertThat(e.getReasons()) + .withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason) + .anyMatch(s -> s.contains(reason)); + } + +} \ No newline at end of file