mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-18 13:10:44 +00:00
CE-1955 Initial checkin (plus add a memory-storage table to testutils)
This commit is contained in:
@ -0,0 +1,236 @@
|
|||||||
|
/*
|
||||||
|
* QQQ - Low-code Application Framework for Engineers.
|
||||||
|
* Copyright (C) 2021-2022. Kingsrook, LLC
|
||||||
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||||
|
* contact@kingsrook.com
|
||||||
|
* https://github.com/Kingsrook/
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License as
|
||||||
|
* published by the Free Software Foundation, either version 3 of the
|
||||||
|
* License, or (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU Affero General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
|
||||||
|
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.UUID;
|
||||||
|
import java.util.function.Consumer;
|
||||||
|
import com.kingsrook.qqq.backend.core.BaseTest;
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
|
||||||
|
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||||
|
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||||
|
import com.kingsrook.qqq.backend.core.model.metadata.frontend.QFrontendFieldMetaData;
|
||||||
|
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||||
|
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadProfileField;
|
||||||
|
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
|
||||||
|
import com.kingsrook.qqq.backend.core.utils.TestUtils;
|
||||||
|
import org.junit.jupiter.api.AfterEach;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import static org.assertj.core.api.Assertions.assertThat;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Unit test for full bulk insert process
|
||||||
|
*******************************************************************************/
|
||||||
|
class BulkInsertV2Test extends BaseTest
|
||||||
|
{
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@BeforeEach
|
||||||
|
@AfterEach
|
||||||
|
void beforeAndAfterEach()
|
||||||
|
{
|
||||||
|
MemoryRecordStore.getInstance().reset();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public static String getPersonCsvRow1()
|
||||||
|
{
|
||||||
|
return ("""
|
||||||
|
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1980-01-01","john@doe.com","Missouri",42
|
||||||
|
""");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public static String getPersonCsvRow2()
|
||||||
|
{
|
||||||
|
return ("""
|
||||||
|
"0","2021-10-26 14:39:37","2021-10-26 14:39:37","Jane","Doe","1981-01-01","john@doe.com","Illinois",
|
||||||
|
""");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
public static String getPersonCsvHeaderUsingLabels()
|
||||||
|
{
|
||||||
|
return ("""
|
||||||
|
"Id","Create Date","Modify Date","First Name","Last Name","Birth Date","Email","Home State",noOfShoes
|
||||||
|
""");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
**
|
||||||
|
*******************************************************************************/
|
||||||
|
@Test
|
||||||
|
void test() throws Exception
|
||||||
|
{
|
||||||
|
String defaultEmail = "noone@kingsrook.com";
|
||||||
|
|
||||||
|
///////////////////////////////////////
|
||||||
|
// make sure table is empty to start //
|
||||||
|
///////////////////////////////////////
|
||||||
|
assertThat(TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY)).isEmpty();
|
||||||
|
|
||||||
|
QInstance qInstance = QContext.getQInstance();
|
||||||
|
String processName = "PersonBulkInsertV2";
|
||||||
|
new QInstanceEnricher(qInstance).defineTableBulkInsertV2(qInstance, qInstance.getTable(TestUtils.TABLE_NAME_PERSON_MEMORY), processName);
|
||||||
|
|
||||||
|
/////////////////////////////////////////////////////////
|
||||||
|
// start the process - expect to go to the upload step //
|
||||||
|
/////////////////////////////////////////////////////////
|
||||||
|
RunProcessInput runProcessInput = new RunProcessInput();
|
||||||
|
runProcessInput.setProcessName(processName);
|
||||||
|
RunProcessOutput runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||||
|
String processUUID = runProcessOutput.getProcessUUID();
|
||||||
|
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("upload");
|
||||||
|
|
||||||
|
//////////////////////////////
|
||||||
|
// simulate the file upload //
|
||||||
|
//////////////////////////////
|
||||||
|
String storageReference = UUID.randomUUID() + ".csv";
|
||||||
|
StorageInput storageInput = new StorageInput(TestUtils.TABLE_NAME_MEMORY_STORAGE).withReference(storageReference);
|
||||||
|
try(OutputStream outputStream = new StorageAction().createOutputStream(storageInput))
|
||||||
|
{
|
||||||
|
outputStream.write((getPersonCsvHeaderUsingLabels() + getPersonCsvRow1() + getPersonCsvRow2()).getBytes());
|
||||||
|
}
|
||||||
|
catch(IOException e)
|
||||||
|
{
|
||||||
|
throw (e);
|
||||||
|
}
|
||||||
|
|
||||||
|
//////////////////////////
|
||||||
|
// continue post-upload //
|
||||||
|
//////////////////////////
|
||||||
|
runProcessInput.setProcessUUID(processUUID);
|
||||||
|
runProcessInput.setStartAfterStep("upload");
|
||||||
|
runProcessInput.addValue("tableName", TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||||
|
runProcessInput.addValue("theFile", new ArrayList<>(List.of(storageInput)));
|
||||||
|
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||||
|
assertEquals(List.of("Id", "Create Date", "Modify Date", "First Name", "Last Name", "Birth Date", "Email", "Home State", "noOfShoes"), runProcessOutput.getValue("headerValues"));
|
||||||
|
assertEquals(List.of("A", "B", "C", "D", "E", "F", "G", "H", "I"), runProcessOutput.getValue("headerLetters"));
|
||||||
|
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("fileMapping");
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// all subsequent steps will want these data - so set up a lambda to set them //
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
Consumer<RunProcessInput> addProfileToRunProcessInput = (RunProcessInput input) ->
|
||||||
|
{
|
||||||
|
input.addValue("version", "v1");
|
||||||
|
input.addValue("layout", "FLAT");
|
||||||
|
input.addValue("hasHeaderRow", "true");
|
||||||
|
input.addValue("fieldListJSON", JsonUtils.toJson(List.of(
|
||||||
|
new BulkLoadProfileField().withFieldName("firstName").withColumnIndex(3),
|
||||||
|
new BulkLoadProfileField().withFieldName("lastName").withColumnIndex(4),
|
||||||
|
new BulkLoadProfileField().withFieldName("email").withDefaultValue(defaultEmail),
|
||||||
|
new BulkLoadProfileField().withFieldName("homeStateId").withColumnIndex(7).withDoValueMapping(true),
|
||||||
|
new BulkLoadProfileField().withFieldName("noOfShoes").withColumnIndex(8)
|
||||||
|
)));
|
||||||
|
};
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
// continue post file-mapping //
|
||||||
|
////////////////////////////////
|
||||||
|
runProcessInput.setStartAfterStep("fileMapping");
|
||||||
|
addProfileToRunProcessInput.accept(runProcessInput);
|
||||||
|
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||||
|
Serializable valueMappingField = runProcessOutput.getValue("valueMappingField");
|
||||||
|
assertThat(valueMappingField).isInstanceOf(QFrontendFieldMetaData.class);
|
||||||
|
assertEquals("homeStateId", ((QFrontendFieldMetaData) valueMappingField).getName());
|
||||||
|
assertEquals(List.of("Missouri", "Illinois"), runProcessOutput.getValue("fileValues"));
|
||||||
|
assertEquals(List.of("homeStateId"), runProcessOutput.getValue("fieldNamesToDoValueMapping"));
|
||||||
|
assertEquals(0, runProcessOutput.getValue("valueMappingFieldIndex"));
|
||||||
|
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("valueMapping");
|
||||||
|
|
||||||
|
/////////////////////////////////
|
||||||
|
// continue post value-mapping //
|
||||||
|
/////////////////////////////////
|
||||||
|
runProcessInput.setStartAfterStep("valueMapping");
|
||||||
|
runProcessInput.addValue("mappedValuesJSON", JsonUtils.toJson(Map.of("Illinois", 1, "Missouri", 2)));
|
||||||
|
addProfileToRunProcessInput.accept(runProcessInput);
|
||||||
|
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||||
|
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("review");
|
||||||
|
|
||||||
|
/////////////////////////////////
|
||||||
|
// continue post review screen //
|
||||||
|
/////////////////////////////////
|
||||||
|
runProcessInput.setStartAfterStep("review");
|
||||||
|
addProfileToRunProcessInput.accept(runProcessInput);
|
||||||
|
runProcessOutput = new RunProcessAction().execute(runProcessInput);
|
||||||
|
assertThat(runProcessOutput.getRecords()).hasSize(2);
|
||||||
|
assertThat(runProcessOutput.getProcessState().getNextStepName()).isPresent().get().isEqualTo("result");
|
||||||
|
assertThat(runProcessOutput.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY)).isNotNull().isInstanceOf(List.class);
|
||||||
|
assertThat(runProcessOutput.getException()).isEmpty();
|
||||||
|
|
||||||
|
////////////////////////////////////
|
||||||
|
// query for the inserted records //
|
||||||
|
////////////////////////////////////
|
||||||
|
List<QRecord> records = TestUtils.queryTable(TestUtils.TABLE_NAME_PERSON_MEMORY);
|
||||||
|
|
||||||
|
assertEquals("John", records.get(0).getValueString("firstName"));
|
||||||
|
assertEquals("Jane", records.get(1).getValueString("firstName"));
|
||||||
|
|
||||||
|
assertNotNull(records.get(0).getValue("id"));
|
||||||
|
assertNotNull(records.get(1).getValue("id"));
|
||||||
|
|
||||||
|
assertEquals(2, records.get(0).getValueInteger("homeStateId"));
|
||||||
|
assertEquals(1, records.get(1).getValueInteger("homeStateId"));
|
||||||
|
|
||||||
|
assertEquals(defaultEmail, records.get(0).getValueString("email"));
|
||||||
|
assertEquals(defaultEmail, records.get(1).getValueString("email"));
|
||||||
|
|
||||||
|
assertEquals(42, records.get(0).getValueInteger("noOfShoes"));
|
||||||
|
assertNull(records.get(1).getValue("noOfShoes"));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -142,6 +142,7 @@ public class TestUtils
|
|||||||
public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
|
public static final String APP_NAME_MISCELLANEOUS = "miscellaneous";
|
||||||
|
|
||||||
public static final String TABLE_NAME_TWO_KEYS = "twoKeys";
|
public static final String TABLE_NAME_TWO_KEYS = "twoKeys";
|
||||||
|
public static final String TABLE_NAME_MEMORY_STORAGE = "memoryStorage";
|
||||||
public static final String TABLE_NAME_PERSON = "person";
|
public static final String TABLE_NAME_PERSON = "person";
|
||||||
public static final String TABLE_NAME_SHAPE = "shape";
|
public static final String TABLE_NAME_SHAPE = "shape";
|
||||||
public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache";
|
public static final String TABLE_NAME_SHAPE_CACHE = "shapeCache";
|
||||||
@ -204,6 +205,7 @@ public class TestUtils
|
|||||||
|
|
||||||
qInstance.addTable(defineTablePerson());
|
qInstance.addTable(defineTablePerson());
|
||||||
qInstance.addTable(defineTableTwoKeys());
|
qInstance.addTable(defineTableTwoKeys());
|
||||||
|
qInstance.addTable(defineTableMemoryStorage());
|
||||||
qInstance.addTable(definePersonFileTable());
|
qInstance.addTable(definePersonFileTable());
|
||||||
qInstance.addTable(definePersonMemoryTable());
|
qInstance.addTable(definePersonMemoryTable());
|
||||||
qInstance.addTable(definePersonMemoryCacheTable());
|
qInstance.addTable(definePersonMemoryCacheTable());
|
||||||
@ -594,6 +596,22 @@ public class TestUtils
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/*******************************************************************************
|
||||||
|
** Define a table in the memory store that can be used for the StorageAction
|
||||||
|
*******************************************************************************/
|
||||||
|
public static QTableMetaData defineTableMemoryStorage()
|
||||||
|
{
|
||||||
|
return new QTableMetaData()
|
||||||
|
.withName(TABLE_NAME_MEMORY_STORAGE)
|
||||||
|
.withLabel("Memory Storage")
|
||||||
|
.withBackendName(MEMORY_BACKEND_NAME)
|
||||||
|
.withPrimaryKeyField("reference")
|
||||||
|
.withField(new QFieldMetaData("reference", QFieldType.STRING).withIsEditable(false))
|
||||||
|
.withField(new QFieldMetaData("contents", QFieldType.BLOB));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/*******************************************************************************
|
/*******************************************************************************
|
||||||
** Define the 'person' table used in standard tests.
|
** Define the 'person' table used in standard tests.
|
||||||
*******************************************************************************/
|
*******************************************************************************/
|
||||||
|
|||||||
Reference in New Issue
Block a user