QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-773 change fileNameFieldName and contentsFieldName to default as null - add validation to tableBackendDetails, specifically implemented in filesystem module

Browse Source
This commit is contained in:
darin.kelkhoff
2023-12-28 16:38:40 -06:00
parent 2da6878e70
commit 872dec3177
5 changed files with 262 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java
View File

@ -480,6 +480,11 @@ public class QInstanceValidator
validateTableCustomizer(tableName, entry.getKey(), entry.getValue());
}
if(table.getBackendDetails() != null)
{
table.getBackendDetails().validate(qInstance, table, this);
}
validateTableAutomationDetails(qInstance, table);
validateTableUniqueKeys(table);
validateAssociatedScripts(table);

15
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/tables/QTableBackendDetails.java
View File

@ -23,6 +23,9 @@ package com.kingsrook.qqq.backend.core.model.metadata.tables;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.serialization.QTableBackendDetailsDeserializer;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleInterface;
@ -100,4 +103,16 @@ public abstract class QTableBackendDetails
return (this);
}
/*******************************************************************************
**
*******************************************************************************/
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
{
////////////////////////
// noop in base class //
////////////////////////
}
}

48
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetails.java
View File

@ -22,7 +22,11 @@
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableBackendDetails;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
@ -35,11 +39,9 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
private RecordFormat recordFormat;
private Cardinality cardinality;
///////////////////////////////////////////////////////////////////////////////////////////////////
// todo default these to null, and give validation error if not set for a cardinality=ONE table? //
///////////////////////////////////////////////////////////////////////////////////////////////////
private String contentsFieldName = "contents";
private String fileNameFieldName = "fileName";
private String contentsFieldName;
private String fileNameFieldName;
/*******************************************************************************
@ -243,4 +245,40 @@ public class AbstractFilesystemTableBackendDetails extends QTableBackendDetails
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public void validate(QInstance qInstance, QTableMetaData table, QInstanceValidator qInstanceValidator)
{
super.validate(qInstance, table, qInstanceValidator);
String prefix = "Table " + (table == null ? "null" : table.getName()) + " backend details - ";
if(qInstanceValidator.assertCondition(cardinality != null, prefix + "missing cardinality"))
{
if(cardinality.equals(Cardinality.ONE))
{
if(qInstanceValidator.assertCondition(StringUtils.hasContent(contentsFieldName), prefix + "missing contentsFieldName, which is required for Cardinality ONE"))
{
qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(contentsFieldName), prefix + "contentsFieldName [" + contentsFieldName + "] is not a field on this table.");
}
if(qInstanceValidator.assertCondition(StringUtils.hasContent(fileNameFieldName), prefix + "missing fileNameFieldName, which is required for Cardinality ONE"))
{
qInstanceValidator.assertCondition(table != null && table.getFields().containsKey(fileNameFieldName), prefix + "fileNameFieldName [" + fileNameFieldName + "] is not a field on this table.");
}
qInstanceValidator.assertCondition(recordFormat == null, prefix + "has a recordFormat, which is not allowed for Cardinality ONE");
}
if(cardinality.equals(Cardinality.MANY))
{
qInstanceValidator.assertCondition(!StringUtils.hasContent(contentsFieldName), prefix + "has a contentsFieldName, which is not allowed for Cardinality MANY");
qInstanceValidator.assertCondition(!StringUtils.hasContent(fileNameFieldName), prefix + "has a fileNameFieldName, which is not allowed for Cardinality MANY");
qInstanceValidator.assertCondition(recordFormat != null, prefix + "missing recordFormat, which is required for Cardinality MANY");
}
}
}
}

7
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/TestUtils.java
View File

@ -26,7 +26,6 @@ import java.io.File;
import java.io.IOException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -142,8 +141,6 @@ public class TestUtils
qInstance.addTable(defineMockPersonTable());
qInstance.addProcess(defineStreamedLocalCsvToMockETLProcess());
new QInstanceValidator().validate(qInstance);
return (qInstance);
}
@ -249,6 +246,8 @@ public class TestUtils
.withBackendDetails(new FilesystemTableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
);
}
@ -269,6 +268,8 @@ public class TestUtils
.withBackendDetails(new S3TableBackendDetails()
.withBasePath("blobs")
.withCardinality(Cardinality.ONE)
.withFileNameFieldName("fileName")
.withContentsFieldName("contents")
);
}

195
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/base/model/metadata/AbstractFilesystemTableBackendDetailsTest.java Normal file
View File

@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2023. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.base.model.metadata;
import java.util.function.Consumer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.fail;
/*******************************************************************************
** Unit test for AbstractFilesystemTableBackendDetails
*******************************************************************************/
class AbstractFilesystemTableBackendDetailsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testValidInstancePasses() throws QInstanceValidationException
{
new QInstanceValidator().validate(QContext.getQInstance());
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testMissingCardinality() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_S3).withBackendDetails(new FilesystemTableBackendDetails());
}, false, "missing cardinality");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCardinalityOneIssues() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
);
}, false, "missing contentsFieldName", "missing fileNameFieldName");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
.withContentsFieldName("foo")
.withFileNameFieldName("bar")
);
}, false, "contentsFieldName [foo] is not a field", "fileNameFieldName [bar] is not a field");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_BLOB_LOCAL_FS).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.ONE)
.withContentsFieldName("contents")
.withFileNameFieldName("fileName")
.withRecordFormat(RecordFormat.CSV)
);
}, false, "has a recordFormat");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testCardinalityManyIssues() throws QException
{
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.MANY)
);
}, false, "missing recordFormat");
assertValidationFailureReasons((QInstance qInstance) ->
{
qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_CSV).withBackendDetails(new FilesystemTableBackendDetails()
.withCardinality(Cardinality.MANY)
.withRecordFormat(RecordFormat.CSV)
.withContentsFieldName("foo")
.withFileNameFieldName("bar")
);
}, false, "has a contentsFieldName", "has a fileNameFieldName");
}
/*******************************************************************************
** Implementation for the overloads of this name.
*******************************************************************************/
private void assertValidationFailureReasons(Consumer<QInstance> setup, boolean allowExtraReasons, String... reasons) throws QException
{
try
{
QInstance qInstance = TestUtils.defineInstance();
setup.accept(qInstance);
new QInstanceValidator().validate(qInstance);
fail("Should have thrown validationException");
}
catch(QInstanceValidationException e)
{
if(!allowExtraReasons)
{
int noOfReasons = e.getReasons() == null ? 0 : e.getReasons().size();
assertEquals(reasons.length, noOfReasons, "Expected number of validation failure reasons.\nExpected reasons: " + String.join(",", reasons)
+ "\nActual reasons: " + (noOfReasons > 0 ? String.join("\n", e.getReasons()) : "--"));
}
for(String reason : reasons)
{
assertReason(reason, e);
}
}
}
/*******************************************************************************
** Assert that an instance is valid!
*******************************************************************************/
private void assertValidationSuccess(Consumer<QInstance> setup) throws QException
{
try
{
QInstance qInstance = TestUtils.defineInstance();
setup.accept(qInstance);
new QInstanceValidator().validate(qInstance);
}
catch(QInstanceValidationException e)
{
fail("Expected no validation errors, but received: " + e.getMessage());
}
}
/*******************************************************************************
** utility method for asserting that a specific reason string is found within
** the list of reasons in the QInstanceValidationException.
**
*******************************************************************************/
private void assertReason(String reason, QInstanceValidationException e)
{
assertNotNull(e.getReasons(), "Expected there to be a reason for the failure (but there was not)");
assertThat(e.getReasons())
.withFailMessage("Expected any of:\n%s\nTo match: [%s]", e.getReasons(), reason)
.anyMatch(s -> s.contains(reason));
}
}