QQQ-16 increasing test coverage

This commit is contained in:
2022-06-23 16:49:17 -05:00
parent 93a06cf1ab
commit daacfa6849
11 changed files with 213 additions and 41 deletions

View File

@ -51,7 +51,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-backend-core</artifactId>
<version>0.0.0-20220623.193535-9</version>
<version>0.0.0-20220623.214704-10</version>
</dependency>
<!-- 3rd party deps specifically for this module -->

View File

@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
@ -41,12 +40,13 @@ public class FilesystemDeleteAction implements DeleteInterface
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
throw new NotImplementedException("Filesystem delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
throw new NotImplementedException("Filesystem delete not implemented");
// return rs;
}
@ -54,6 +54,7 @@ public class FilesystemDeleteAction implements DeleteInterface
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

View File

@ -22,13 +22,9 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
@ -44,6 +40,8 @@ public class FilesystemInsertAction implements InsertInterface
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
throw new NotImplementedException("Filesystem insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
@ -52,14 +50,13 @@ public class FilesystemInsertAction implements InsertInterface
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("Filesystem insert not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
}
}

View File

@ -22,13 +22,9 @@
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
@ -44,6 +40,8 @@ public class FilesystemUpdateAction implements UpdateInterface
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
throw new NotImplementedException("Filesystem update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
@ -52,7 +50,6 @@ public class FilesystemUpdateAction implements UpdateInterface
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
throw new NotImplementedException("Filesystem update not implemented");
// return rs;
}
@ -60,6 +57,7 @@ public class FilesystemUpdateAction implements UpdateInterface
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

View File

@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
@ -51,12 +52,19 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
{
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
@Override
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
{
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName());
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
{
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
@ -67,17 +75,17 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
.collect(Collectors.toSet());
for(String sourceFile : sourceFiles)
{
String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete");
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
if("delete".equals(moveOrDelete))
{
filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
}
else if("move".equals(moveOrDelete))
{
String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves");
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
if(!StringUtils.hasContent(destinationForMoves))
{
throw (new QException("Field [destinationForMoves] is missing a value."));
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
}
File file = new File(sourceFile);
String destinationPath = destinationForMoves + File.separator + file.getName();
@ -85,7 +93,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
}
else
{
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete]."));
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete]."));
}
}
}

View File

@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
import org.apache.commons.lang.NotImplementedException;
@ -41,12 +40,13 @@ public class S3DeleteAction implements DeleteInterface
*******************************************************************************/
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
{
throw new NotImplementedException("S3 delete not implemented");
/*
try
{
DeleteResult rs = new DeleteResult();
QTableMetaData table = deleteRequest.getTable();
throw new NotImplementedException("S3 delete not implemented");
// return rs;
}
@ -54,6 +54,7 @@ public class S3DeleteAction implements DeleteInterface
{
throw new QException("Error executing delete: " + e.getMessage(), e);
}
*/
}
}

View File

@ -22,13 +22,9 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
import org.apache.commons.lang.NotImplementedException;
@ -44,6 +40,8 @@ public class S3InsertAction implements InsertInterface
*******************************************************************************/
public InsertResult execute(InsertRequest insertRequest) throws QException
{
throw new NotImplementedException("S3 insert not implemented");
/*
try
{
InsertResult rs = new InsertResult();
@ -52,7 +50,6 @@ public class S3InsertAction implements InsertInterface
List<QRecord> recordsWithStatus = new ArrayList<>();
rs.setRecords(recordsWithStatus);
throw new NotImplementedException("S3 insert not implemented");
// return rs;
}
@ -60,6 +57,7 @@ public class S3InsertAction implements InsertInterface
{
throw new QException("Error executing insert: " + e.getMessage(), e);
}
*/
}
}

View File

@ -22,13 +22,9 @@
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
import org.apache.commons.lang.NotImplementedException;
@ -44,6 +40,8 @@ public class S3UpdateAction implements UpdateInterface
*******************************************************************************/
public UpdateResult execute(UpdateRequest updateRequest) throws QException
{
throw new NotImplementedException("S3 update not implemented");
/*
try
{
UpdateResult rs = new UpdateResult();
@ -52,14 +50,13 @@ public class S3UpdateAction implements UpdateInterface
List<QRecord> records = new ArrayList<>();
rs.setRecords(records);
throw new NotImplementedException("S3 update not implemented");
// return rs;
}
catch(Exception e)
{
throw new QException("Error executing update: " + e.getMessage(), e);
}
*/
}
}

View File

@ -26,10 +26,13 @@ import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
@ -43,7 +46,9 @@ import org.apache.commons.io.FileUtils;
*******************************************************************************/
public class TestUtils
{
public static final String TABLE_NAME_PERSON = "person";
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
public static final String BACKEND_NAME_S3 = "s3";
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
///////////////////////////////////////////////////////////////////
@ -103,6 +108,7 @@ public class TestUtils
public static QInstance defineInstance() throws QInstanceValidationException
{
QInstance qInstance = new QInstance();
qInstance.setAuthentication(defineAuthentication());
qInstance.addBackend(defineLocalFilesystemBackend());
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
qInstance.addBackend(defineS3Backend());
@ -115,6 +121,19 @@ public class TestUtils
/*******************************************************************************
** Define the authentication used in standard tests - using 'mock' type.
**
*******************************************************************************/
public static QAuthenticationMetaData defineAuthentication()
{
return new QAuthenticationMetaData()
.withName("mock")
.withType("mock");
}
/*******************************************************************************
**
*******************************************************************************/
@ -122,7 +141,7 @@ public class TestUtils
{
return (new FilesystemBackendMetaData()
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
.withName("local-filesystem"));
.withName(BACKEND_NAME_LOCAL_FS));
}
@ -133,7 +152,7 @@ public class TestUtils
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
{
return new QTableMetaData()
.withName(TABLE_NAME_PERSON)
.withName(TABLE_NAME_PERSON_LOCAL_FS)
.withLabel("Person")
.withBackendName(defineLocalFilesystemBackend().getName())
.withPrimaryKeyField("id")
@ -161,7 +180,7 @@ public class TestUtils
return (new S3BackendMetaData()
.withBucketName(BaseS3Test.BUCKET_NAME)
.withBasePath(BaseS3Test.TEST_FOLDER)
.withName("s3"));
.withName(BACKEND_NAME_S3));
}
@ -189,4 +208,14 @@ public class TestUtils
);
}
/*******************************************************************************
**
*******************************************************************************/
public static QSession getMockSession()
{
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
return (mockAuthenticationModule.createSession(null));
}
}

View File

@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then delete one, then re-list, and assert that we have one fewer //
@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest
public void testDeleteFileDoesNotExist() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
@ -119,7 +119,7 @@ public class FilesystemBackendModuleTest
public void testMoveFile() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
@ -156,7 +156,7 @@ public class FilesystemBackendModuleTest
public void testMoveFileDoesNotExit() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
String subPath = basePath + File.separator + "subdir";
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));

View File

@ -0,0 +1,143 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
import com.kingsrook.qqq.backend.core.callbacks.NoopCallback;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesFunctionTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testDelete() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete",
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash"));
assertNull(runFunctionResult.getError());
assertFalse(new File(filePath).exists(), "File should have been deleted.");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
public void testMove() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
String trashDir = File.separator + "tmp" + File.separator + "trash";
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move",
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
assertNull(runFunctionResult.getError());
assertFalse(new File(filePath).exists(), "File should have been moved.");
String movedPath = trashDir + File.separator + (new File(filePath).getName());
assertTrue(new File(movedPath).exists(), "File should have been moved.");
}
/*******************************************************************************
**
*******************************************************************************/
private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map<String, String> values) throws Exception
{
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
qInstance.addProcess(qProcessMetaData);
File file = new File(filePath);
FileUtils.writeStringToFile(file, "content");
List<QRecord> records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath));
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
runFunctionRequest.setProcessName(qProcessMetaData.getName());
runFunctionRequest.setCallback(new NoopCallback());
runFunctionRequest.setRecords(records);
runFunctionRequest.setSession(TestUtils.getMockSession());
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
for(Map.Entry<String, String> entry : values.entrySet())
{
runFunctionRequest.addValue(entry.getKey(), entry.getValue());
}
RunFunctionAction runFunctionAction = new RunFunctionAction();
return (runFunctionAction.execute(runFunctionRequest));
}
/*******************************************************************************
**
*******************************************************************************/
private String getRandomFilePathPersonTable(QInstance qInstance)
{
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath();
String filePath = tablePath + File.separator + UUID.randomUUID();
return filePath;
}
}