QQQ-37 Redo bulk processes in streamed-etl mode

This commit is contained in:
2022-09-07 16:59:42 -05:00
parent 1c75df3a09
commit b01758879c
27 changed files with 1430 additions and 1036 deletions

View File

@ -1,74 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for BulkDeleteStoreStep
*******************************************************************************/
class BulkDeleteStoreStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithoutFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.setRecords(TestUtils.queryTable(TestUtils.defineTablePerson().getName()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testWithFilter() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue("queryFilterJSON", JsonUtils.toJson(new QQueryFilter()));
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkDeleteStoreStep().run(stepInput, stepOutput);
assertEquals(0, stepOutput.getValueInteger(BulkDeleteStoreStep.ERROR_COUNT));
}
}

View File

@ -0,0 +1,119 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.delete;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for full bulk edit process
*******************************************************************************/
class BulkDeleteTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
//////////////////////////////
// insert some test records //
//////////////////////////////
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"),
new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"),
new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com")
));
//////////////////////////////////
// set up the run-process input //
//////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkDelete");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER,
new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 3))));
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.addValue("processUUID", processUUID);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
@SuppressWarnings("unchecked")
List<ProcessSummaryLine> processSummaryLines = (List<ProcessSummaryLine>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
assertThat(processSummaryLines).hasSize(1);
assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1);
//////////////////////////////////////////////////////////////
// query for the records - assert that only id=2 exists now //
//////////////////////////////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals(2, records.get(0).getValueInteger("id"));
}
}

View File

@ -1,78 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BulkEditReceiveValuesStep
*******************************************************************************/
class BulkEditReceiveValuesStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
stepInput.addValue("firstName", "Johnny");
stepInput.addValue("email", null);
stepInput.addValue("birthDate", "1909-01-09");
List<QRecord> records = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
stepInput.setRecords(records);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkEditReceiveValuesStep().run(stepInput, stepOutput);
String valuesBeingUpdated = stepOutput.getValueString(BulkEditReceiveValuesStep.FIELD_VALUES_BEING_UPDATED);
assertThat(valuesBeingUpdated).matches("(?s).*First Name.*Johnny.*");
assertThat(valuesBeingUpdated).matches("(?s).*Email will be cleared.*");
assertThat(valuesBeingUpdated).matches("(?s).*Birth Date.*1909-01-09.*");
int count = 0;
for(QRecord record : stepOutput.getRecords())
{
assertEquals("Johnny", record.getValueString("firstName"));
assertNull(record.getValue("email"));
// todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate"));
count++;
}
assertEquals(records.size(), count);
}
}

View File

@ -1,85 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for BulkEditStoreRecordsStep
*******************************************************************************/
class BulkEditStoreRecordsStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(BulkEditReceiveValuesStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
stepInput.addValue("firstName", "Johnny");
stepInput.addValue("email", null);
stepInput.addValue("birthDate", "1909-01-09");
List<QRecord> records = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
stepInput.setRecords(records);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkEditStoreRecordsStep().run(stepInput, stepOutput);
assertRecordValues(stepOutput.getRecords());
// re-fetch the records, make sure they are updated.
// but since Mock backend doesn't actually update them, we can't do this..
// todo - implement an in-memory backend, that would do this.
// List<QRecord> updatedRecords = TestUtils.queryTable(TestUtils.defineTablePerson().getName());
// assertRecordValues(updatedRecords);
}
/*******************************************************************************
**
*******************************************************************************/
private void assertRecordValues(List<QRecord> records)
{
for(QRecord record : records)
{
assertEquals("Johnny", record.getValueString("firstName"));
assertNull(record.getValue("email"));
// todo value utils needed in getValueDate... assertEquals(LocalDate.of(1909, 1, 9), record.getValueDate("birthDate"));
}
}
}

View File

@ -0,0 +1,145 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.edit;
import java.util.List;
import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
/*******************************************************************************
** Unit test for full bulk edit process
*******************************************************************************/
class BulkEditTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
//////////////////////////////
// insert some test records //
//////////////////////////////
QInstance qInstance = TestUtils.defineInstance();
TestUtils.insertRecords(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), List.of(
new QRecord().withValue("id", 1).withValue("firstName", "Darin").withValue("lastName", "Kelkhoff").withValue("email", "darin.kelkhoff@kingsrook.com"),
new QRecord().withValue("id", 2).withValue("firstName", "Tim").withValue("lastName", "Chamberlain").withValue("email", "tim.chamberlain@kingsrook.com"),
new QRecord().withValue("id", 3).withValue("firstName", "James").withValue("lastName", "Maes").withValue("email", "james.maes@kingsrook.com")
));
//////////////////////////////////
// set up the run-process input //
//////////////////////////////////
RunProcessInput runProcessInput = new RunProcessInput(qInstance);
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkEdit");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DEFAULT_QUERY_FILTER,
new QQueryFilter().withCriteria(new QFilterCriteria("id", QCriteriaOperator.IN, List.of(1, 2))));
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.addValue(BulkEditTransformStep.FIELD_ENABLED_FIELDS, "firstName,email,birthDate");
runProcessInput.addValue("firstName", "Johnny");
runProcessInput.addValue("email", null);
runProcessInput.addValue("birthDate", "1909-01-09");
runProcessInput.setProcessUUID(processUUID);
runProcessInput.setStartAfterStep("edit");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
@SuppressWarnings("unchecked")
List<ProcessSummaryLine> processSummaryLines = (List<ProcessSummaryLine>) runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
assertThat(processSummaryLines).hasSize(4);
assertThat(processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.OK))).hasSize(1);
List<ProcessSummaryLine> infoLines = processSummaryLines.stream().filter(psl -> psl.getStatus().equals(Status.INFO)).collect(Collectors.toList());
assertThat(infoLines).hasSize(3);
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*First Name.*Johnny.*"));
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Email was cleared.*"));
assertThat(infoLines.stream().map(ProcessSummaryLine::getMessage)).anyMatch(m -> m.matches("(?s).*Birth Date.*1909-01-09.*"));
/////////////////////////////////////////////////////////////////////////////////
// query for the edited records - assert that id 1 & 2 were updated, 3 was not //
/////////////////////////////////////////////////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals("Johnny", records.get(0).getValueString("firstName"));
assertEquals("Johnny", records.get(1).getValueString("firstName"));
assertEquals("James", records.get(2).getValueString("firstName"));
assertEquals("1909-01-09", records.get(0).getValueString("birthDate"));
assertEquals("1909-01-09", records.get(1).getValueString("birthDate"));
assertNull(records.get(2).getValueString("birthDate"));
assertNull(records.get(0).getValue("email"));
assertNull(records.get(1).getValue("email"));
assertEquals("james.maes@kingsrook.com", records.get(2).getValue("email"));
}
}

View File

@ -1,114 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
/*******************************************************************************
** Unit test for BulkInsertReceiveFileStep
*******************************************************************************/
class BulkInsertReceiveFileStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
List<QRecord> records = stepOutput.getRecords();
assertEquals(2, records.size());
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNull(records.get(0).getValue("id"));
assertNull(records.get(1).getValue("id"));
assertEquals(2, stepOutput.getValueInteger("noOfFileRows"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testBadFileType() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes()); // todo - this is NOT excel content...
qUploadedFile.setFilename("test.xslx");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
assertThrows(QUserFacingException.class, () ->
{
new BulkInsertReceiveFileStep().run(stepInput, stepOutput);
});
}
}

View File

@ -1,80 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for BulkInsertStoreRecordsStep
*******************************************************************************/
class BulkInsertStoreRecordsStepTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey key = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(key, qUploadedFile);
////////////////////////////
// setup and run the step //
////////////////////////////
RunBackendStepInput stepInput = new RunBackendStepInput(TestUtils.defineInstance());
stepInput.setSession(TestUtils.getMockSession());
stepInput.setTableName(TestUtils.defineTablePerson().getName());
stepInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, key);
RunBackendStepOutput stepOutput = new RunBackendStepOutput();
new BulkInsertStoreRecordsStep().run(stepInput, stepOutput);
List<QRecord> records = stepOutput.getRecords();
assertEquals(2, records.size());
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNotNull(records.get(0).getValue("id"));
assertNotNull(records.get(1).getValue("id"));
}
}

View File

@ -0,0 +1,121 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.QUploadedFile;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
import com.kingsrook.qqq.backend.core.state.StateType;
import com.kingsrook.qqq.backend.core.state.TempFileStateProvider;
import com.kingsrook.qqq.backend.core.state.UUIDAndTypeStateKey;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/*******************************************************************************
** Unit test for full bulk insert process
*******************************************************************************/
class BulkInsertTest
{
/*******************************************************************************
**
*******************************************************************************/
@BeforeEach
@AfterEach
void beforeAndAfterEach()
{
MemoryRecordStore.getInstance().reset();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
///////////////////////////////////////
// make sure table is empty to start //
///////////////////////////////////////
assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
////////////////////////////////////////////////////////////////
// create an uploaded file, similar to how an http server may //
////////////////////////////////////////////////////////////////
QUploadedFile qUploadedFile = new QUploadedFile();
qUploadedFile.setBytes((TestUtils.getPersonCsvHeaderUsingLabels() + TestUtils.getPersonCsvRow1() + TestUtils.getPersonCsvRow2()).getBytes());
qUploadedFile.setFilename("test.csv");
UUIDAndTypeStateKey uploadedFileKey = new UUIDAndTypeStateKey(StateType.UPLOADED_FILE);
TempFileStateProvider.getInstance().put(uploadedFileKey, qUploadedFile);
RunProcessInput runProcessInput = new RunProcessInput(TestUtils.defineInstance());
runProcessInput.setSession(TestUtils.getMockSession());
runProcessInput.setProcessName(TestUtils.TABLE_NAME_PERSON_MEMORY + ".bulkInsert");
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
String processUUID = runProcessOutput.getProcessUUID();
runProcessInput.setProcessUUID(processUUID);
runProcessInput.setStartAfterStep("upload");
runProcessInput.addValue(QUploadedFile.DEFAULT_UPLOADED_FILE_FIELD_NAME, uploadedFileKey);
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
runProcessInput.addValue(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY)).isNotNull().isInstanceOf(List.class);
runProcessInput.setStartAfterStep("review");
runProcessOutput = new RunProcessAction().execute(runProcessInput);
assertThat(runProcessOutput.getRecords()).hasSize(2);
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
assertThat(runProcessOutput.getException()).isEmpty();
////////////////////////////////////
// query for the inserted records //
////////////////////////////////////
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
assertEquals("John", records.get(0).getValueString("firstName"));
assertEquals("Jane", records.get(1).getValueString("firstName"));
assertNotNull(records.get(0).getValue("id"));
assertNotNull(records.get(1).getValue("id"));
}
}

View File

@ -29,7 +29,6 @@ import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.automation.AutomationStatus;
import com.kingsrook.qqq.backend.core.actions.automation.RecordAutomationHandler;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.dashboard.PersonsByCreateDateBarChart;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.processes.person.addtopeoplesage.AddAge;
@ -164,7 +163,6 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
@ -207,6 +205,20 @@ public class TestUtils
/*******************************************************************************
**
*******************************************************************************/
public static void insertRecords(QInstance qInstance, QTableMetaData table, List<QRecord> records) throws QException
{
InsertInput insertInput = new InsertInput(qInstance);
insertInput.setSession(new QSession());
insertInput.setTableName(table.getName());
insertInput.setRecords(records);
new InsertAction().execute(insertInput);
}
/*******************************************************************************
**
*******************************************************************************/