Implementation of QContext everywhere, instead of passing QInstance and QSession in all ActionInputs

This commit is contained in:
2023-01-15 19:30:57 -06:00
parent 69a6104393
commit d3fa1df56f
219 changed files with 1955 additions and 1581 deletions

View File

@ -258,8 +258,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
*******************************************************************************/
public CountOutput executeCount(CountInput countInput) throws QException
{
QueryInput queryInput = new QueryInput(countInput.getInstance());
queryInput.setSession(countInput.getSession());
QueryInput queryInput = new QueryInput();
queryInput.setTableName(countInput.getTableName());
queryInput.setFilter(countInput.getFilter());
QueryOutput queryOutput = executeQuery(queryInput);

View File

@ -0,0 +1,61 @@
/*
* Copyright © 2022-2023. Nutrifresh Services <contact@nutrifreshservices.com>. All Rights Reserved.
*/
package com.kingsrook.qqq.backend.module.filesystem;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
/*******************************************************************************
**
*******************************************************************************/
public class BaseTest
{
private static final Logger LOG = LogManager.getLogger(BaseTest.class);
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
void baseBeforeEach() throws QException
{
QContext.init(TestUtils.defineInstance(), new QSession());
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void baseAfterEach()
{
QContext.clear();
}
/*******************************************************************************
**
*******************************************************************************/
protected static void reInitInstanceInContext(QInstance qInstance)
{
if(qInstance.equals(QContext.getQInstance()))
{
LOG.warn("Unexpected condition - the same qInstance that is already in the QContext was passed into reInit. You probably want a new QInstance object instance.");
}
QContext.init(qInstance, new QSession());
}
}

View File

@ -24,7 +24,10 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -39,7 +42,7 @@ import static org.junit.jupiter.api.Assertions.fail;
**
** Knows how to set up the filesystem for the tests.
*******************************************************************************/
public class FilesystemActionTest
public class FilesystemActionTest extends BaseTest
{
/*******************************************************************************
@ -49,6 +52,7 @@ public class FilesystemActionTest
public void beforeEach() throws Exception
{
primeFilesystem();
QContext.init(TestUtils.defineInstance(), new QSession());
}

View File

@ -43,7 +43,6 @@ public class FilesystemCountActionTest extends FilesystemActionTest
public void testCount1() throws QException
{
CountInput countInput = new CountInput();
countInput.setInstance(TestUtils.defineInstance());
countInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
CountOutput countOutput = new FilesystemCountAction().execute(countInput);
Assertions.assertEquals(3, countOutput.getCount(), "Unfiltered count should find all rows");

View File

@ -55,8 +55,7 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
public void testCardinalityOne() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_LOCAL_FS);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file1.txt").withValue("contents", "Hello, World")
@ -78,8 +77,7 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")

View File

@ -50,7 +50,6 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
public void testQuery1() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");
@ -72,8 +71,8 @@ public class FilesystemQueryActionTest extends FilesystemActionTest
QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
table.withCustomizer(FilesystemTableCustomizers.POST_READ_FILE.getRole(), new QCodeReference(ValueUpshifter.class, QCodeUsage.CUSTOMIZER));
reInitInstanceInContext(instance);
queryInput.setInstance(instance);
queryInput.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName());
QueryOutput queryOutput = new FilesystemQueryAction().execute(queryInput);
Assertions.assertEquals(3, queryOutput.getRecords().size(), "Unfiltered query should find all rows");

View File

@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@ -35,6 +36,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaD
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.basic.BasicETLProcess;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -48,7 +50,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCleanupSourceFilesFunction
*******************************************************************************/
public class BasicETLCleanupSourceFilesStepTest
public class BasicETLCleanupSourceFilesStepTest extends BaseTest
{
/*******************************************************************************
@ -57,7 +59,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testDelete1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath));
}
@ -70,7 +72,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testDelete2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath, filePath));
}
@ -83,7 +85,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testDelete2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testDelete(qInstance, List.of(filePath1, filePath2));
@ -97,7 +99,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testMove1Record1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath));
}
@ -110,7 +112,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testMove2Records1File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath, filePath));
}
@ -123,7 +125,7 @@ public class BasicETLCleanupSourceFilesStepTest
@Test
public void testMove2Record2File() throws Exception
{
QInstance qInstance = TestUtils.defineInstance();
QInstance qInstance = QContext.getQInstance();
String filePath1 = getRandomFilePathPersonTable(qInstance);
String filePath2 = getRandomFilePathPersonTable(qInstance);
testMove(qInstance, List.of(filePath1, filePath2));
@ -180,7 +182,7 @@ public class BasicETLCleanupSourceFilesStepTest
private RunBackendStepOutput runFunction(QInstance qInstance, List<String> filePaths, Map<String, String> values) throws Exception
{
QBackendStepMetaData backendStepMetaData = new BasicETLCleanupSourceFilesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
HashSet<String> filePathsSet = new HashSet<>(filePaths);
@ -193,11 +195,10 @@ public class BasicETLCleanupSourceFilesStepTest
// List<QRecord> records = filePaths.stream()
// .map(filePath -> new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, filePath)).toList();
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
RunBackendStepInput runBackendStepInput = new RunBackendStepInput();
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
// runFunctionRequest.setRecords(records);
runBackendStepInput.setSession(TestUtils.getMockSession());
runBackendStepInput.addValue(BasicETLProcess.FIELD_SOURCE_TABLE, TestUtils.TABLE_NAME_PERSON_LOCAL_FS_JSON);
runBackendStepInput.addValue(BasicETLProcess.FIELD_DESTINATION_TABLE, TestUtils.TABLE_NAME_PERSON_S3);
runBackendStepInput.addValue(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS, StringUtils.join(",", filePathsSet));

View File

@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields;
import org.junit.jupiter.api.Test;
@ -41,7 +42,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for BasicETLCollectSourceFileNamesFunction
*******************************************************************************/
class BasicETLCollectSourceFileNamesStepTest
class BasicETLCollectSourceFileNamesStepTest extends BaseTest
{
/*******************************************************************************
@ -97,12 +98,12 @@ class BasicETLCollectSourceFileNamesStepTest
QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesStep().defineStepMetaData();
QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData);
qInstance.addProcess(qProcessMetaData);
reInitInstanceInContext(qInstance);
List<QRecord> records = Arrays.stream(fileNames).map(fileName ->
new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList();
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepInput runBackendStepInput = new RunBackendStepInput();
runBackendStepInput.setStepName(backendStepMetaData.getName());
runBackendStepInput.setProcessName(qProcessMetaData.getName());
runBackendStepInput.setRecords(records);

View File

@ -48,12 +48,11 @@ class StreamedETLFilesystemBackendStepTest extends FilesystemActionTest
{
QInstance qInstance = TestUtils.defineInstance();
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
RunProcessInput runProcessInput = new RunProcessInput();
runProcessInput.setProcessName(TestUtils.PROCESS_NAME_STREAMED_ETL);
RunProcessOutput output = new RunProcessAction().execute(runProcessInput);
String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
RunProcessOutput output = new RunProcessAction().execute(runProcessInput);
String sourceFilePaths = ValueUtils.getValueAsString(output.getValues().get(BasicETLCollectSourceFileNamesStep.FIELD_SOURCE_FILE_PATHS));
assertThat(sourceFilePaths)
.contains("FILE-1.csv")
.contains("FILE-2.csv");

View File

@ -30,12 +30,12 @@ import com.kingsrook.qqq.backend.core.exceptions.QModuleDispatchException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.BaseS3Test;
@ -83,6 +83,7 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
reInitInstanceInContext(qInstance);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
@ -102,10 +103,9 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
//////////////////
// run the step //
//////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
RunBackendStepInput runBackendStepInput = new RunBackendStepInput();
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);
@ -143,9 +143,10 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
QTableMetaData archiveTable = defineTable(qInstance, "archive", localBackend, "archive", "*/l3/*.csv");
QTableMetaData processingTable = defineTable(qInstance, "processing", localBackend, "processing", "**/*.csv");
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
QProcessMetaData process = new FilesystemSyncProcess().defineProcessMetaData();
QBackendStepMetaData step = process.getBackendStep(FilesystemSyncStep.STEP_NAME);
qInstance.addProcess(process);
reInitInstanceInContext(qInstance);
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_SOURCE_TABLE).setDefaultValue(sourceTable.getName());
step.getInputMetaData().getFieldThrowing(FilesystemSyncProcess.FIELD_ARCHIVE_TABLE).setDefaultValue(archiveTable.getName());
@ -165,10 +166,9 @@ class FilesystemSyncProcessS3Test extends BaseS3Test
//////////////////
// run the step //
//////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
RunBackendStepInput runBackendStepInput = new RunBackendStepInput();
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);

View File

@ -27,12 +27,13 @@ import java.io.IOException;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.TestUtils;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
@ -43,7 +44,7 @@ import org.junit.jupiter.api.Test;
/*******************************************************************************
** Unit test for FilesystemSyncProcess
*******************************************************************************/
class FilesystemSyncProcessTest
class FilesystemSyncProcessTest extends BaseTest
{
/*******************************************************************************
@ -70,6 +71,7 @@ class FilesystemSyncProcessTest
qInstance.addTable(archiveTable);
qInstance.addTable(processingTable);
qInstance.addProcess(process);
reInitInstanceInContext(qInstance);
///////////////////////////
// write some test files //
@ -83,10 +85,9 @@ class FilesystemSyncProcessTest
//////////////////////
// run the step //
//////////////////////
RunBackendStepInput runBackendStepInput = new RunBackendStepInput(qInstance);
RunBackendStepInput runBackendStepInput = new RunBackendStepInput();
runBackendStepInput.setStepName(step.getName());
runBackendStepInput.setProcessName(process.getName());
runBackendStepInput.setSession(TestUtils.getMockSession());
RunBackendStepAction runFunctionAction = new RunBackendStepAction();
RunBackendStepOutput runBackendStepOutput = runFunctionAction.execute(runBackendStepInput);

View File

@ -28,6 +28,7 @@ import cloud.localstack.docker.LocalstackDockerExtension;
import cloud.localstack.docker.annotation.LocalstackDockerProperties;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3Utils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@ -39,7 +40,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
*******************************************************************************/
@ExtendWith(LocalstackDockerExtension.class)
@LocalstackDockerProperties(useSingleDockerContainer = true, services = { ServiceName.S3 }, portEdge = "2960", portElasticSearch = "2961")
public class BaseS3Test
public class BaseS3Test extends BaseTest
{
public static final String BUCKET_NAME = "localstack-test-bucket";
public static final String TEST_FOLDER = "test-files";

View File

@ -58,7 +58,6 @@ public class S3CountActionTest extends BaseS3Test
private CountInput initCountRequest() throws QException
{
CountInput countInput = new CountInput();
countInput.setInstance(TestUtils.defineInstance());
countInput.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return countInput;
}

View File

@ -57,8 +57,7 @@ public class S3InsertActionTest extends BaseS3Test
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_BLOB_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("fileName", "file2.txt").withValue("contents", "Hi, Bob.")
@ -86,8 +85,7 @@ public class S3InsertActionTest extends BaseS3Test
public void testCardinalityMany() throws QException, IOException
{
QInstance qInstance = TestUtils.defineInstance();
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
InsertInput insertInput = new InsertInput();
insertInput.setTableName(TestUtils.TABLE_NAME_PERSON_S3);
insertInput.setRecords(List.of(
new QRecord().withValue("id", "1").withValue("firstName", "Bob")

View File

@ -62,7 +62,6 @@ public class S3QueryActionTest extends BaseS3Test
private QueryInput initQueryRequest() throws QException
{
QueryInput queryInput = new QueryInput();
queryInput.setInstance(TestUtils.defineInstance());
queryInput.setTableName(TestUtils.defineS3CSVPersonTable().getName());
return queryInput;
}