mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
QQQ-16 increasing test coverage
This commit is contained in:
2
pom.xml
2
pom.xml
@ -51,7 +51,7 @@
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.kingsrook.qqq</groupId>
|
<groupId>com.kingsrook.qqq</groupId>
|
||||||
<artifactId>qqq-backend-core</artifactId>
|
<artifactId>qqq-backend-core</artifactId>
|
||||||
<version>0.0.0-20220623.193535-9</version>
|
<version>0.0.0-20220623.214704-10</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- 3rd party deps specifically for this module -->
|
<!-- 3rd party deps specifically for this module -->
|
||||||
|
@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
|||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -41,12 +40,13 @@ public class FilesystemDeleteAction implements DeleteInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("Filesystem delete not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
DeleteResult rs = new DeleteResult();
|
DeleteResult rs = new DeleteResult();
|
||||||
QTableMetaData table = deleteRequest.getTable();
|
QTableMetaData table = deleteRequest.getTable();
|
||||||
|
|
||||||
throw new NotImplementedException("Filesystem delete not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
@ -54,6 +54,7 @@ public class FilesystemDeleteAction implements DeleteInterface
|
|||||||
{
|
{
|
||||||
throw new QException("Error executing delete: " + e.getMessage(), e);
|
throw new QException("Error executing delete: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,13 +22,9 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||||
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -44,6 +40,8 @@ public class FilesystemInsertAction implements InsertInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public InsertResult execute(InsertRequest insertRequest) throws QException
|
public InsertResult execute(InsertRequest insertRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("Filesystem insert not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
InsertResult rs = new InsertResult();
|
InsertResult rs = new InsertResult();
|
||||||
@ -52,14 +50,13 @@ public class FilesystemInsertAction implements InsertInterface
|
|||||||
List<QRecord> recordsWithStatus = new ArrayList<>();
|
List<QRecord> recordsWithStatus = new ArrayList<>();
|
||||||
rs.setRecords(recordsWithStatus);
|
rs.setRecords(recordsWithStatus);
|
||||||
|
|
||||||
throw new NotImplementedException("Filesystem insert not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
catch(Exception e)
|
catch(Exception e)
|
||||||
{
|
{
|
||||||
throw new QException("Error executing insert: " + e.getMessage(), e);
|
throw new QException("Error executing insert: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,13 +22,9 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
package com.kingsrook.qqq.backend.module.filesystem.local.actions;
|
||||||
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -44,6 +40,8 @@ public class FilesystemUpdateAction implements UpdateInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("Filesystem update not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
UpdateResult rs = new UpdateResult();
|
UpdateResult rs = new UpdateResult();
|
||||||
@ -52,7 +50,6 @@ public class FilesystemUpdateAction implements UpdateInterface
|
|||||||
List<QRecord> records = new ArrayList<>();
|
List<QRecord> records = new ArrayList<>();
|
||||||
rs.setRecords(records);
|
rs.setRecords(records);
|
||||||
|
|
||||||
throw new NotImplementedException("Filesystem update not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
@ -60,6 +57,7 @@ public class FilesystemUpdateAction implements UpdateInterface
|
|||||||
{
|
{
|
||||||
throw new QException("Error executing update: " + e.getMessage(), e);
|
throw new QException("Error executing update: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
|
|||||||
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
import com.kingsrook.qqq.backend.core.interfaces.FunctionBody;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeReference;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeType;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
|
import com.kingsrook.qqq.backend.core.model.metadata.QCodeUsage;
|
||||||
@ -51,12 +52,19 @@ import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendD
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
|
public class BasicETLCleanupSourceFilesFunction implements FunctionBody
|
||||||
{
|
{
|
||||||
|
public static final String FIELD_MOVE_OR_DELETE = "moveOrDelete";
|
||||||
|
public static final String FIELD_DESTINATION_FOR_MOVES = "destinationForMoves";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
public void run(RunFunctionRequest runFunctionRequest, RunFunctionResult runFunctionResult) throws QException
|
||||||
{
|
{
|
||||||
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
|
String sourceTableName = runFunctionRequest.getValueString(BasicETLProcess.FIELD_SOURCE_TABLE);
|
||||||
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
|
QTableMetaData table = runFunctionRequest.getInstance().getTable(sourceTableName);
|
||||||
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(table.getBackendName());
|
QBackendMetaData backend = runFunctionRequest.getInstance().getBackendForTable(sourceTableName);
|
||||||
|
QBackendModuleInterface module = new QBackendModuleDispatcher().getQBackendModule(backend);
|
||||||
|
|
||||||
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
|
if(!(module instanceof FilesystemBackendModuleInterface filesystemModule))
|
||||||
{
|
{
|
||||||
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
|
throw (new QException("Backend " + table.getBackendName() + " for table " + sourceTableName + " does not support this function."));
|
||||||
@ -67,17 +75,17 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
|
|||||||
.collect(Collectors.toSet());
|
.collect(Collectors.toSet());
|
||||||
for(String sourceFile : sourceFiles)
|
for(String sourceFile : sourceFiles)
|
||||||
{
|
{
|
||||||
String moveOrDelete = runFunctionRequest.getValueString("moveOrDelete");
|
String moveOrDelete = runFunctionRequest.getValueString(FIELD_MOVE_OR_DELETE);
|
||||||
if("delete".equals(moveOrDelete))
|
if("delete".equals(moveOrDelete))
|
||||||
{
|
{
|
||||||
filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
|
filesystemModule.deleteFile(runFunctionRequest.getInstance(), table, sourceFile);
|
||||||
}
|
}
|
||||||
else if("move".equals(moveOrDelete))
|
else if("move".equals(moveOrDelete))
|
||||||
{
|
{
|
||||||
String destinationForMoves = runFunctionRequest.getValueString("destinationForMoves");
|
String destinationForMoves = runFunctionRequest.getValueString(FIELD_DESTINATION_FOR_MOVES);
|
||||||
if(!StringUtils.hasContent(destinationForMoves))
|
if(!StringUtils.hasContent(destinationForMoves))
|
||||||
{
|
{
|
||||||
throw (new QException("Field [destinationForMoves] is missing a value."));
|
throw (new QException("Field [" + FIELD_DESTINATION_FOR_MOVES + "] is missing a value."));
|
||||||
}
|
}
|
||||||
File file = new File(sourceFile);
|
File file = new File(sourceFile);
|
||||||
String destinationPath = destinationForMoves + File.separator + file.getName();
|
String destinationPath = destinationForMoves + File.separator + file.getName();
|
||||||
@ -85,7 +93,7 @@ public class BasicETLCleanupSourceFilesFunction implements FunctionBody
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [moveOrDelete]. Must be either [move] or [delete]."));
|
throw (new QException("Unexpected value [" + moveOrDelete + "] for field [" + FIELD_MOVE_OR_DELETE + "]. Must be either [move] or [delete]."));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
|||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
import com.kingsrook.qqq.backend.core.model.actions.delete.DeleteResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.DeleteInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -41,12 +40,13 @@ public class S3DeleteAction implements DeleteInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
public DeleteResult execute(DeleteRequest deleteRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("S3 delete not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
DeleteResult rs = new DeleteResult();
|
DeleteResult rs = new DeleteResult();
|
||||||
QTableMetaData table = deleteRequest.getTable();
|
QTableMetaData table = deleteRequest.getTable();
|
||||||
|
|
||||||
throw new NotImplementedException("S3 delete not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
@ -54,6 +54,7 @@ public class S3DeleteAction implements DeleteInterface
|
|||||||
{
|
{
|
||||||
throw new QException("Error executing delete: " + e.getMessage(), e);
|
throw new QException("Error executing delete: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,13 +22,9 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||||
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.InsertInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -44,6 +40,8 @@ public class S3InsertAction implements InsertInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public InsertResult execute(InsertRequest insertRequest) throws QException
|
public InsertResult execute(InsertRequest insertRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("S3 insert not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
InsertResult rs = new InsertResult();
|
InsertResult rs = new InsertResult();
|
||||||
@ -52,7 +50,6 @@ public class S3InsertAction implements InsertInterface
|
|||||||
List<QRecord> recordsWithStatus = new ArrayList<>();
|
List<QRecord> recordsWithStatus = new ArrayList<>();
|
||||||
rs.setRecords(recordsWithStatus);
|
rs.setRecords(recordsWithStatus);
|
||||||
|
|
||||||
throw new NotImplementedException("S3 insert not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
@ -60,6 +57,7 @@ public class S3InsertAction implements InsertInterface
|
|||||||
{
|
{
|
||||||
throw new QException("Error executing insert: " + e.getMessage(), e);
|
throw new QException("Error executing insert: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -22,13 +22,9 @@
|
|||||||
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
|
||||||
|
|
||||||
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.List;
|
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
|
||||||
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
|
||||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
|
||||||
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
import com.kingsrook.qqq.backend.core.modules.interfaces.UpdateInterface;
|
||||||
import org.apache.commons.lang.NotImplementedException;
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
@ -44,6 +40,8 @@ public class S3UpdateAction implements UpdateInterface
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
public UpdateResult execute(UpdateRequest updateRequest) throws QException
|
||||||
{
|
{
|
||||||
|
throw new NotImplementedException("S3 update not implemented");
|
||||||
|
/*
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
UpdateResult rs = new UpdateResult();
|
UpdateResult rs = new UpdateResult();
|
||||||
@ -52,14 +50,13 @@ public class S3UpdateAction implements UpdateInterface
|
|||||||
List<QRecord> records = new ArrayList<>();
|
List<QRecord> records = new ArrayList<>();
|
||||||
rs.setRecords(records);
|
rs.setRecords(records);
|
||||||
|
|
||||||
throw new NotImplementedException("S3 update not implemented");
|
|
||||||
|
|
||||||
// return rs;
|
// return rs;
|
||||||
}
|
}
|
||||||
catch(Exception e)
|
catch(Exception e)
|
||||||
{
|
{
|
||||||
throw new QException("Error executing update: " + e.getMessage(), e);
|
throw new QException("Error executing update: " + e.getMessage(), e);
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -26,10 +26,13 @@ import java.io.File;
|
|||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
import com.kingsrook.qqq.backend.core.exceptions.QInstanceValidationException;
|
||||||
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldMetaData;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
import com.kingsrook.qqq.backend.core.model.metadata.QFieldType;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||||
|
import com.kingsrook.qqq.backend.core.modules.mock.MockAuthenticationModule;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||||
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
|
||||||
@ -43,7 +46,9 @@ import org.apache.commons.io.FileUtils;
|
|||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
public class TestUtils
|
public class TestUtils
|
||||||
{
|
{
|
||||||
public static final String TABLE_NAME_PERSON = "person";
|
public static final String BACKEND_NAME_LOCAL_FS = "local-filesystem";
|
||||||
|
public static final String BACKEND_NAME_S3 = "s3";
|
||||||
|
public static final String TABLE_NAME_PERSON_LOCAL_FS = "person";
|
||||||
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
|
public static final String TABLE_NAME_PERSON_S3 = "person-s3";
|
||||||
|
|
||||||
///////////////////////////////////////////////////////////////////
|
///////////////////////////////////////////////////////////////////
|
||||||
@ -103,6 +108,7 @@ public class TestUtils
|
|||||||
public static QInstance defineInstance() throws QInstanceValidationException
|
public static QInstance defineInstance() throws QInstanceValidationException
|
||||||
{
|
{
|
||||||
QInstance qInstance = new QInstance();
|
QInstance qInstance = new QInstance();
|
||||||
|
qInstance.setAuthentication(defineAuthentication());
|
||||||
qInstance.addBackend(defineLocalFilesystemBackend());
|
qInstance.addBackend(defineLocalFilesystemBackend());
|
||||||
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
|
qInstance.addTable(defineLocalFilesystemCSVPersonTable());
|
||||||
qInstance.addBackend(defineS3Backend());
|
qInstance.addBackend(defineS3Backend());
|
||||||
@ -115,6 +121,19 @@ public class TestUtils
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Define the authentication used in standard tests - using 'mock' type.
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public static QAuthenticationMetaData defineAuthentication()
|
||||||
|
{
|
||||||
|
return new QAuthenticationMetaData()
|
||||||
|
.withName("mock")
|
||||||
|
.withType("mock");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
**
|
**
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
@ -122,7 +141,7 @@ public class TestUtils
|
|||||||
{
|
{
|
||||||
return (new FilesystemBackendMetaData()
|
return (new FilesystemBackendMetaData()
|
||||||
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
|
.withBasePath(BASE_PATH + File.separator + testInstanceCounter)
|
||||||
.withName("local-filesystem"));
|
.withName(BACKEND_NAME_LOCAL_FS));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -133,7 +152,7 @@ public class TestUtils
|
|||||||
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
|
public static QTableMetaData defineLocalFilesystemCSVPersonTable()
|
||||||
{
|
{
|
||||||
return new QTableMetaData()
|
return new QTableMetaData()
|
||||||
.withName(TABLE_NAME_PERSON)
|
.withName(TABLE_NAME_PERSON_LOCAL_FS)
|
||||||
.withLabel("Person")
|
.withLabel("Person")
|
||||||
.withBackendName(defineLocalFilesystemBackend().getName())
|
.withBackendName(defineLocalFilesystemBackend().getName())
|
||||||
.withPrimaryKeyField("id")
|
.withPrimaryKeyField("id")
|
||||||
@ -161,7 +180,7 @@ public class TestUtils
|
|||||||
return (new S3BackendMetaData()
|
return (new S3BackendMetaData()
|
||||||
.withBucketName(BaseS3Test.BUCKET_NAME)
|
.withBucketName(BaseS3Test.BUCKET_NAME)
|
||||||
.withBasePath(BaseS3Test.TEST_FOLDER)
|
.withBasePath(BaseS3Test.TEST_FOLDER)
|
||||||
.withName("s3"));
|
.withName(BACKEND_NAME_S3));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -189,4 +208,14 @@ public class TestUtils
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public static QSession getMockSession()
|
||||||
|
{
|
||||||
|
MockAuthenticationModule mockAuthenticationModule = new MockAuthenticationModule();
|
||||||
|
return (mockAuthenticationModule.createSession(null));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -70,7 +70,7 @@ public class FilesystemBackendModuleTest
|
|||||||
public void testDeleteFile() throws Exception
|
public void testDeleteFile() throws Exception
|
||||||
{
|
{
|
||||||
QInstance qInstance = TestUtils.defineInstance();
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// first list the files - then delete one, then re-list, and assert that we have one fewer //
|
// first list the files - then delete one, then re-list, and assert that we have one fewer //
|
||||||
@ -94,7 +94,7 @@ public class FilesystemBackendModuleTest
|
|||||||
public void testDeleteFileDoesNotExist() throws Exception
|
public void testDeleteFileDoesNotExist() throws Exception
|
||||||
{
|
{
|
||||||
QInstance qInstance = TestUtils.defineInstance();
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
|
||||||
|
|
||||||
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||||
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
|
// first list the files - then try to delete a fake path, then re-list, and assert that we have the same count //
|
||||||
@ -119,7 +119,7 @@ public class FilesystemBackendModuleTest
|
|||||||
public void testMoveFile() throws Exception
|
public void testMoveFile() throws Exception
|
||||||
{
|
{
|
||||||
QInstance qInstance = TestUtils.defineInstance();
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
|
||||||
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
|
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
|
||||||
String subPath = basePath + File.separator + "subdir";
|
String subPath = basePath + File.separator + "subdir";
|
||||||
|
|
||||||
@ -156,7 +156,7 @@ public class FilesystemBackendModuleTest
|
|||||||
public void testMoveFileDoesNotExit() throws Exception
|
public void testMoveFileDoesNotExit() throws Exception
|
||||||
{
|
{
|
||||||
QInstance qInstance = TestUtils.defineInstance();
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON);
|
QTableMetaData table = qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
|
||||||
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
|
String basePath = ((FilesystemBackendMetaData) qInstance.getBackendForTable(table.getName())).getBasePath();
|
||||||
String subPath = basePath + File.separator + "subdir";
|
String subPath = basePath + File.separator + "subdir";
|
||||||
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
|
List<File> filesBeforeMove = new AbstractFilesystemAction().listFiles(table, qInstance.getBackendForTable(table.getName()));
|
||||||
|
@ -0,0 +1,143 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic;
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.RunFunctionAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.callbacks.NoopCallback;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionRequest;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunFunctionResult;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Unit test for BasicETLCleanupSourceFilesFunction
|
||||||
|
*******************************************************************************/
|
||||||
|
public class BasicETLCleanupSourceFilesFunctionTest
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
public void testDelete() throws Exception
|
||||||
|
{
|
||||||
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
|
String filePath = getRandomFilePathPersonTable(qInstance);
|
||||||
|
|
||||||
|
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
|
||||||
|
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "delete",
|
||||||
|
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, "/tmp/trash"));
|
||||||
|
|
||||||
|
assertNull(runFunctionResult.getError());
|
||||||
|
assertFalse(new File(filePath).exists(), "File should have been deleted.");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
public void testMove() throws Exception
|
||||||
|
{
|
||||||
|
QInstance qInstance = TestUtils.defineInstance();
|
||||||
|
String filePath = getRandomFilePathPersonTable(qInstance);
|
||||||
|
|
||||||
|
String trashDir = File.separator + "tmp" + File.separator + "trash";
|
||||||
|
RunFunctionResult runFunctionResult = runFunction(qInstance, filePath, Map.of(
|
||||||
|
BasicETLCleanupSourceFilesFunction.FIELD_MOVE_OR_DELETE, "move",
|
||||||
|
BasicETLCleanupSourceFilesFunction.FIELD_DESTINATION_FOR_MOVES, trashDir));
|
||||||
|
|
||||||
|
assertNull(runFunctionResult.getError());
|
||||||
|
assertFalse(new File(filePath).exists(), "File should have been moved.");
|
||||||
|
|
||||||
|
String movedPath = trashDir + File.separator + (new File(filePath).getName());
|
||||||
|
assertTrue(new File(movedPath).exists(), "File should have been moved.");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private RunFunctionResult runFunction(QInstance qInstance, String filePath, Map<String, String> values) throws Exception
|
||||||
|
{
|
||||||
|
QFunctionMetaData qFunctionMetaData = new BasicETLCleanupSourceFilesFunction().defineFunctionMetaData();
|
||||||
|
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addFunction(qFunctionMetaData);
|
||||||
|
qInstance.addProcess(qProcessMetaData);
|
||||||
|
|
||||||
|
File file = new File(filePath);
|
||||||
|
FileUtils.writeStringToFile(file, "content");
|
||||||
|
|
||||||
|
List<QRecord> records = List.of(new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath));
|
||||||
|
|
||||||
|
RunFunctionRequest runFunctionRequest = new RunFunctionRequest(qInstance);
|
||||||
|
runFunctionRequest.setFunctionName(qFunctionMetaData.getName());
|
||||||
|
runFunctionRequest.setProcessName(qProcessMetaData.getName());
|
||||||
|
runFunctionRequest.setCallback(new NoopCallback());
|
||||||
|
runFunctionRequest.setRecords(records);
|
||||||
|
runFunctionRequest.setSession(TestUtils.getMockSession());
|
||||||
|
runFunctionRequest.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS);
|
||||||
|
runFunctionRequest.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
|
||||||
|
|
||||||
|
for(Map.Entry<String, String> entry : values.entrySet())
|
||||||
|
{
|
||||||
|
runFunctionRequest.addValue(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
RunFunctionAction runFunctionAction = new RunFunctionAction();
|
||||||
|
return (runFunctionAction.execute(runFunctionRequest));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
private String getRandomFilePathPersonTable(QInstance qInstance)
|
||||||
|
{
|
||||||
|
FilesystemBackendMetaData backend = (FilesystemBackendMetaData) qInstance.getBackend(TestUtils.BACKEND_NAME_LOCAL_FS);
|
||||||
|
FilesystemTableBackendDetails backendDetails = (FilesystemTableBackendDetails) qInstance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS).getBackendDetails();
|
||||||
|
String tablePath = backend.getBasePath() + File.separator + backendDetails.getPath();
|
||||||
|
String filePath = tablePath + File.separator + UUID.randomUUID();
|
||||||
|
return filePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Reference in New Issue
Block a user