QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Initial checkin

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-12 09:16:59 -06:00
parent 7d058530d5
commit 7ba205a262
22 changed files with 3404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java Normal file
View File

@ -0,0 +1,81 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
BulkLoadFileRow headerRow = fileToRowsInterface.next();
List<String> bodyRows = new ArrayList<>();
while(fileToRowsInterface.hasNext() && bodyRows.size() < 20)
{
bodyRows.add(fileToRowsInterface.next().toString());
}
runBackendStepOutput.addValue("header", headerRow.toString());
runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows));
System.out.println("Done receiving file");
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
}

59
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java Normal file
View File

@ -0,0 +1,59 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveMappingStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(true);
bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name"
));
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
// probably need to what, receive the mapping object, store it into state
// what, do we maybe return to a different sub-mapping screen (e.g., values)
// then at some point - cool - proceed to ETL's steps
}
}

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java Normal file
View File

@ -0,0 +1,58 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertStepUtils
{
/***************************************************************************
**
***************************************************************************/
public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException
{
@SuppressWarnings("unchecked")
ArrayList<StorageInput> storageInputs = (ArrayList<StorageInput>) input.getValue("theFile");
if(storageInputs == null)
{
throw (new QException("StorageInputs for theFile were not found in process state"));
}
if(storageInputs.isEmpty())
{
throw (new QException("StorageInputs for theFile was an empty list"));
}
StorageInput storageInput = storageInputs.get(0);
return (storageInput);
}
}

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java Normal file
View File

@ -0,0 +1,126 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
/*******************************************************************************
** Extract step for generic table bulk-insert ETL process
**
** This step does a little bit of transforming, actually - taking rows from
** an uploaded file, and potentially merging them (for child-table use-cases)
** and applying the "Mapping" - to put fully built records into the pipe for the
** Transform step.
*******************************************************************************/
public class BulkInsertV2ExtractStep extends AbstractExtractStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface();
try
(
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// open a stream to read from our file, and a FileToRows object, that knows how to read from that stream //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
///////////////////////////////////////////////////////////
// read the header row (if this file & mapping uses one) //
///////////////////////////////////////////////////////////
BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null;
////////////////////////////////////////////////////////////////////////////////////////////////////////
// while there are more rows in the file - and we're under the limit - get more records form the file //
////////////////////////////////////////////////////////////////////////////////////////////////////////
while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit)
{
int remainingLimit = originalLimit - rowsAdded;
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
int pageLimit = Math.min(remainingLimit, getMaxPageSize());
List<QRecord> page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit);
if(page.size() > remainingLimit)
{
/////////////////////////////////////////////////////////////
// in case we got back more than we asked for, sub-list it //
/////////////////////////////////////////////////////////////
page = page.subList(0, remainingLimit);
}
/////////////////////////////////////////////
// send this page of records into the pipe //
/////////////////////////////////////////////
getRecordPipe().addRecords(page);
rowsAdded += page.size();
}
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
/***************************************************************************
**
***************************************************************************/
private int getMaxPageSize()
{
return (1000);
}
}

167
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java Normal file
View File

@ -0,0 +1,167 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Collectors;
/*******************************************************************************
** A row of values, e.g., from a file, for bulk-load
*******************************************************************************/
public class BulkLoadFileRow implements Serializable
{
private Serializable[] values;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkLoadFileRow(Serializable[] values)
{
this.values = values;
}
/***************************************************************************
**
***************************************************************************/
public int size()
{
if(values == null)
{
return (0);
}
return (values.length);
}
/***************************************************************************
**
***************************************************************************/
public boolean hasIndex(int i)
{
if(values == null)
{
return (false);
}
if(i >= values.length || i < 0)
{
return (false);
}
return (true);
}
/***************************************************************************
**
***************************************************************************/
public Serializable getValue(int i)
{
if(values == null)
{
throw new IllegalStateException("Row has no values");
}
if(i >= values.length || i < 0)
{
throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length);
}
return (values[i]);
}
/***************************************************************************
**
***************************************************************************/
public Serializable getValueElseNull(int i)
{
if(!hasIndex(i))
{
return (null);
}
return (values[i]);
}
/***************************************************************************
**
***************************************************************************/
@Override
public String toString()
{
if(values == null)
{
return ("null");
}
return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(","));
}
/***************************************************************************
**
***************************************************************************/
@Override
public boolean equals(Object o)
{
if(this == o)
{
return true;
}
if(o == null || getClass() != o.getClass())
{
return false;
}
BulkLoadFileRow row = (BulkLoadFileRow) o;
return Objects.deepEquals(values, row.values);
}
/***************************************************************************
**
***************************************************************************/
@Override
public int hashCode()
{
return Arrays.hashCode(values);
}
}

125
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java Normal file
View File

@ -0,0 +1,125 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.util.Iterator;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
/*******************************************************************************
**
*******************************************************************************/
public abstract class AbstractIteratorBasedFileToRows<E> implements FileToRowsInterface
{
private Iterator<E> iterator;
private boolean useLast = false;
private BulkLoadFileRow last;
/***************************************************************************
**
***************************************************************************/
@Override
public boolean hasNext()
{
if(iterator == null)
{
throw new IllegalStateException("Object was not init'ed");
}
if(useLast)
{
return true;
}
return iterator.hasNext();
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow next()
{
if(iterator == null)
{
throw new IllegalStateException("Object was not init'ed");
}
if(useLast)
{
useLast = false;
return (this.last);
}
E e = iterator.next();
BulkLoadFileRow row = makeRow(e);
this.last = row;
return (this.last);
}
/***************************************************************************
**
***************************************************************************/
public abstract BulkLoadFileRow makeRow(E e);
/***************************************************************************
**
***************************************************************************/
@Override
public void unNext()
{
useLast = true;
}
/*******************************************************************************
** Getter for iterator
*******************************************************************************/
public Iterator<E> getIterator()
{
return (this.iterator);
}
/*******************************************************************************
** Setter for iterator
*******************************************************************************/
public void setIterator(Iterator<E> iterator)
{
this.iterator = iterator;
}
}

112
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java Normal file
View File

@ -0,0 +1,112 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
/*******************************************************************************
**
*******************************************************************************/
public class CsvFileToRows extends AbstractIteratorBasedFileToRows<CSVRecord> implements FileToRowsInterface
{
private CSVParser csvParser;
/***************************************************************************
**
***************************************************************************/
public static CsvFileToRows forString(String csv) throws QException
{
CsvFileToRows csvFileToRows = new CsvFileToRows();
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes());
csvFileToRows.init(byteArrayInputStream);
return (csvFileToRows);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void init(InputStream inputStream) throws QException
{
try
{
csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT
.withIgnoreSurroundingSpaces()
);
setIterator(csvParser.iterator());
}
catch(IOException e)
{
throw new QException("Error opening CSV Parser", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow makeRow(CSVRecord csvRecord)
{
Serializable[] values = new Serializable[csvRecord.size()];
int i = 0;
for(String s : csvRecord)
{
values[i++] = s;
}
return (new BulkLoadFileRow(values));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws Exception
{
if(csvParser != null)
{
csvParser.close();
}
}
}

74
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java Normal file
View File

@ -0,0 +1,74 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Locale;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
/*******************************************************************************
**
*******************************************************************************/
public interface FileToRowsInterface extends AutoCloseable, Iterator<BulkLoadFileRow>
{
/***************************************************************************
**
***************************************************************************/
static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException
{
FileToRowsInterface rs;
if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
{
rs = new CsvFileToRows();
}
else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx"))
{
rs = new XlsxFileToRows();
}
else
{
throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx"));
}
rs.init(inputStream);
return rs;
}
/*******************************************************************************
**
*******************************************************************************/
void init(InputStream inputStream) throws QException;
/***************************************************************************
**
***************************************************************************/
void unNext();
}

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java Normal file
View File

@ -0,0 +1,102 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet;
/*******************************************************************************
**
*******************************************************************************/
public class XlsxFileToRows extends AbstractIteratorBasedFileToRows<org.dhatim.fastexcel.reader.Row> implements FileToRowsInterface
{
private ReadableWorkbook workbook;
private Stream<org.dhatim.fastexcel.reader.Row> rows;
/***************************************************************************
**
***************************************************************************/
@Override
public void init(InputStream inputStream) throws QException
{
try
{
workbook = new ReadableWorkbook(inputStream);
Sheet sheet = workbook.getFirstSheet();
rows = sheet.openStream();
setIterator(rows.iterator());
}
catch(IOException e)
{
throw new QException("Error opening XLSX Parser", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow)
{
Serializable[] values = new Serializable[readerRow.getCellCount()];
for(int i = 0; i < readerRow.getCellCount(); i++)
{
values[i] = readerRow.getCell(i).getText();
}
return new BulkLoadFileRow(values);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws Exception
{
if(workbook != null)
{
workbook.close();
}
if(rows != null)
{
rows.close();
}
}
}

503
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java Normal file
View File

@ -0,0 +1,503 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertMapping implements Serializable
{
private String tableName;
private Boolean hasHeaderRow;
private Layout layout;
/////////////////////////////////////////////////////////////////////
// keys in here are: //
// fieldName (for the main table) //
// association.fieldName (for an associated child table) //
// association.association.fieldName (for grandchild associations) //
/////////////////////////////////////////////////////////////////////
private Map<String, String> fieldNameToHeaderNameMap = new HashMap<>();
private Map<String, Integer> fieldNameToIndexMap = new HashMap<>();
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Memoization<Pair<String, String>, Boolean> shouldProcessFieldForTable = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
public enum Layout
{
FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecord::new);
/***************************************************************************
**
***************************************************************************/
private final Supplier<? extends RowsToRecordInterface> supplier;
/***************************************************************************
**
***************************************************************************/
Layout(Supplier<? extends RowsToRecordInterface> supplier)
{
this.supplier = supplier;
}
/***************************************************************************
**
***************************************************************************/
public RowsToRecordInterface newRowsToRecordInterface()
{
return (supplier.get());
}
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException
{
if(hasHeaderRow && fieldNameToHeaderNameMap != null)
{
return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow));
}
else if(fieldNameToIndexMap != null)
{
return (fieldNameToIndexMap);
}
throw (new QException("Mapping was not properly configured."));
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Map<String, Serializable>> getFieldNameToValueMappingForTable(String associatedTableName)
{
Map<String, Map<String, Serializable>> rs = new HashMap<>();
for(Map.Entry<String, Map<String, Serializable>> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet())
{
if(shouldProcessFieldForTable(entry.getKey(), associatedTableName))
{
String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey();
rs.put(key, entry.getValue());
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain)
{
return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p ->
{
List<String> fieldNameParts = new ArrayList<>();
List<String> associationParts = new ArrayList<>();
if(StringUtils.hasContent(fieldNameWithAssociationPrefix))
{
fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\.")));
}
if(StringUtils.hasContent(associationChain))
{
associationParts.addAll(Arrays.asList(associationChain.split("\\.")));
}
if(!fieldNameParts.isEmpty())
{
fieldNameParts.remove(fieldNameParts.size() - 1);
}
return (fieldNameParts.equals(associationParts));
}).orElse(false);
}
/***************************************************************************
**
***************************************************************************/
private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow)
{
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName());
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(field.getName(), headerIndex);
}
}
}
return (rs);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public BulkInsertMapping withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for hasHeaderRow
*******************************************************************************/
public Boolean getHasHeaderRow()
{
return (this.hasHeaderRow);
}
/*******************************************************************************
** Setter for hasHeaderRow
*******************************************************************************/
public void setHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
}
/*******************************************************************************
** Fluent setter for hasHeaderRow
*******************************************************************************/
public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMap
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMap()
{
return (this.fieldNameToHeaderNameMap);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMap
*******************************************************************************/
public void setFieldNameToHeaderNameMap(Map<String, String> fieldNameToHeaderNameMap)
{
this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToHeaderNameMap(Map<String, String> fieldNameToHeaderNameMap)
{
this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToIndexMap
*******************************************************************************/
public Map<String, Integer> getFieldNameToIndexMap()
{
return (this.fieldNameToIndexMap);
}
/*******************************************************************************
** Setter for fieldNameToIndexMap
*******************************************************************************/
public void setFieldNameToIndexMap(Map<String, Integer> fieldNameToIndexMap)
{
this.fieldNameToIndexMap = fieldNameToIndexMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToIndexMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToIndexMap(Map<String, Integer> fieldNameToIndexMap)
{
this.fieldNameToIndexMap = fieldNameToIndexMap;
return (this);
}
/*******************************************************************************
** Getter for mappedAssociations
*******************************************************************************/
public List<String> getMappedAssociations()
{
return (this.mappedAssociations);
}
/*******************************************************************************
** Setter for mappedAssociations
*******************************************************************************/
public void setMappedAssociations(List<String> mappedAssociations)
{
this.mappedAssociations = mappedAssociations;
}
/*******************************************************************************
** Fluent setter for mappedAssociations
*******************************************************************************/
public BulkInsertMapping withMappedAssociations(List<String> mappedAssociations)
{
this.mappedAssociations = mappedAssociations;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToDefaultValueMap
*******************************************************************************/
public Map<String, Serializable> getFieldNameToDefaultValueMap()
{
if(this.fieldNameToDefaultValueMap == null)
{
this.fieldNameToDefaultValueMap = new HashMap<>();
}
return (this.fieldNameToDefaultValueMap);
}
/*******************************************************************************
** Setter for fieldNameToDefaultValueMap
*******************************************************************************/
public void setFieldNameToDefaultValueMap(Map<String, Serializable> fieldNameToDefaultValueMap)
{
this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToDefaultValueMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToDefaultValueMap(Map<String, Serializable> fieldNameToDefaultValueMap)
{
this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToValueMapping
*******************************************************************************/
public Map<String, Map<String, Serializable>> getFieldNameToValueMapping()
{
return (this.fieldNameToValueMapping);
}
/*******************************************************************************
** Setter for fieldNameToValueMapping
*******************************************************************************/
public void setFieldNameToValueMapping(Map<String, Map<String, Serializable>> fieldNameToValueMapping)
{
this.fieldNameToValueMapping = fieldNameToValueMapping;
}
/*******************************************************************************
** Fluent setter for fieldNameToValueMapping
*******************************************************************************/
public BulkInsertMapping withFieldNameToValueMapping(Map<String, Map<String, Serializable>> fieldNameToValueMapping)
{
this.fieldNameToValueMapping = fieldNameToValueMapping;
return (this);
}
/*******************************************************************************
** Getter for layout
*******************************************************************************/
public Layout getLayout()
{
return (this.layout);
}
/*******************************************************************************
** Setter for layout
*******************************************************************************/
public void setLayout(Layout layout)
{
this.layout = layout;
}
/*******************************************************************************
** Fluent setter for layout
*******************************************************************************/
public BulkInsertMapping withLayout(Layout layout)
{
this.layout = layout;
return (this);
}
/*******************************************************************************
** Getter for tallLayoutGroupByIndexMap
*******************************************************************************/
public Map<String, List<Integer>> getTallLayoutGroupByIndexMap()
{
return (this.tallLayoutGroupByIndexMap);
}
/*******************************************************************************
** Setter for tallLayoutGroupByIndexMap
*******************************************************************************/
public void setTallLayoutGroupByIndexMap(Map<String, List<Integer>> tallLayoutGroupByIndexMap)
{
this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap;
}
/*******************************************************************************
** Fluent setter for tallLayoutGroupByIndexMap
*******************************************************************************/
public BulkInsertMapping withTallLayoutGroupByIndexMap(Map<String, List<Integer>> tallLayoutGroupByIndexMap)
{
this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap;
return (this);
}
}

75
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
/*******************************************************************************
**
*******************************************************************************/
public class FlatRowsToRecord implements RowsToRecordInterface
{
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
}
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
}

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public interface RowsToRecordInterface
{
/***************************************************************************
**
***************************************************************************/
List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException;
/***************************************************************************
**
***************************************************************************/
default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
{
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
Serializable value = null;
if(index != null && row != null)
{
value = row.getValueElseNull(index);
}
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix))
{
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix);
}
if(value != null)
{
record.setValue(fieldName, value);
}
}
}

316
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java Normal file
View File

@ -0,0 +1,316 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class TallRowsToRecord implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
List<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
String associationNameChain = "";
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
continue;
}
if(rowsForCurrentRecord.isEmpty())
{
///////////////////////////////////
// this is first - so it's a yes //
///////////////////////////////////
recordGroupByValues = rowGroupByValues;
rowsForCurrentRecord.add(row);
}
else if(Objects.equals(recordGroupByValues, rowGroupByValues))
{
/////////////////////////////
// a match - so keep going //
/////////////////////////////
rowsForCurrentRecord.add(row);
}
else
{
//////////////////////////////////////////////////////////////
// not first, and not a match, so we can finish this record //
//////////////////////////////////////////////////////////////
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
////////////////////////////////////////
// reset these record-specific values //
////////////////////////////////////////
rowsForCurrentRecord = new ArrayList<>();
recordGroupByValues = null;
//////////////////////////////////////////////////////////////////////////////////////////////////////
// we need to push this row back onto the fileToRows object, so it'll be handled in the next record //
//////////////////////////////////////////////////////////////////////////////////////////////////////
fileToRowsInterface.unNext();
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// i wrote this condition in here: && rs.size() < limit //
// but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... //
/////////////////////////////////////////////////////////////////////////////////////////////////
if(!rowsForCurrentRecord.isEmpty())
{
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List<BulkLoadFileRow> rows) throws QException
{
QRecord record = new QRecord();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);
//////////////////////////////////////////////////////
// get all rows for the main table from the 0th row //
//////////////////////////////////////////////////////
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
}
/////////////////////////////
// associations (children) //
/////////////////////////////
for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations()))
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
List<QRecord> associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
return record;
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List<BulkLoadFileRow> rows) throws QException
{
List<QRecord> rs = new ArrayList<>();
QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName());
String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName;
List<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
for(BulkLoadFileRow row : rows)
{
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls);
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
continue;
}
if(rowsForCurrentRecord.isEmpty())
{
///////////////////////////////////
// this is first - so it's a yes //
///////////////////////////////////
recordGroupByValues = rowGroupByValues;
rowsForCurrentRecord.add(row);
}
else if(Objects.equals(recordGroupByValues, rowGroupByValues))
{
/////////////////////////////
// a match - so keep going //
/////////////////////////////
rowsForCurrentRecord.add(row);
}
else
{
//////////////////////////////////////////////////////////////
// not first, and not a match, so we can finish this record //
//////////////////////////////////////////////////////////////
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord));
////////////////////////////////////////
// reset these record-specific values //
////////////////////////////////////////
rowsForCurrentRecord = new ArrayList<>();
//////////////////////////////////////////////////
// use the current row to start the next record //
//////////////////////////////////////////////////
rowsForCurrentRecord.add(row);
recordGroupByValues = rowGroupByValues;
}
}
///////////
// final //
///////////
if(!rowsForCurrentRecord.isEmpty())
{
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord));
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getGroupByValues(BulkLoadFileRow row, List<Integer> indexes)
{
List<Serializable> rowGroupByValues = new ArrayList<>();
boolean haveAnyGroupByValues = false;
for(Integer index : indexes)
{
Serializable value = row.getValueElseNull(index);
rowGroupByValues.add(value);
if(value != null && !"".equals(value))
{
haveAnyGroupByValues = true;
}
}
if(!haveAnyGroupByValues)
{
return (null);
}
return (rowGroupByValues);
}
}

79
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java Normal file
View File

@ -0,0 +1,79 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public class ValueMapper
{
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping)
{
valueMapping(records, mapping, null);
}
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, String associationNameChain)
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
return;
}
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Map<String, Serializable>> entry : mappingForTable.entrySet())
{
String fieldName = entry.getKey();
Map<String, Serializable> map = entry.getValue();
String value = record.getValueString(fieldName);
if(value != null && map.containsKey(value))
{
record.setValue(fieldName, map.get(value));
}
}
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey());
}
}
}
}

333
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java Normal file
View File

@ -0,0 +1,333 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecord implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size());
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException
{
for(String associationName : mapping.getMappedAssociations())
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
// List<QRecord> associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record);
List<QRecord> associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
{
List<QRecord> rs = new ArrayList<>();
Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
{
if(entry.getKey().startsWith(associationName + "."))
{
String fieldName = entry.getKey().substring(associationName.length() + 1);
//////////////////////////////////////////////////////////////////////////
// make sure the name here is for this table - not a sub-table under it //
//////////////////////////////////////////////////////////////////////////
if(!fieldName.contains("."))
{
fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
}
}
}
/////////////////////////////////////////////////////////////////////
// loop over the length of the record, building associated records //
/////////////////////////////////////////////////////////////////////
QRecord associatedRecord = new QRecord();
Set<String> processedFieldNames = new HashSet<>();
boolean gotAnyValues = false;
int subStartIndex = -1;
for(int i = startIndex; i < endIndex; i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
{
if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
{
///////////////////////////////////////////////
// ok - this is a value for this association //
///////////////////////////////////////////////
if(subStartIndex == -1)
{
subStartIndex = i;
}
String fieldName = entry.getKey();
if(processedFieldNames.contains(fieldName))
{
/////////////////////////////////////////////////
// this means we're starting a new sub-record! //
/////////////////////////////////////////////////
if(gotAnyValues)
{
addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
rs.add(associatedRecord);
}
associatedRecord = new QRecord();
processedFieldNames = new HashSet<>();
gotAnyValues = false;
subStartIndex = i + 1;
}
processedFieldNames.add(fieldName);
Serializable value = row.getValueElseNull(i);
if(value != null && !"".equals(value))
{
gotAnyValues = true;
}
setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
}
}
}
////////////////////////
// handle final value //
////////////////////////
if(gotAnyValues)
{
addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
rs.add(associatedRecord);
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null);
}
}
}
/***************************************************************************
**
***************************************************************************/
// private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// String associationNameChainForRecursiveCalls = associationName;
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + "."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue());
// }
// }
// Map<String, List<Integer>> indexes = new HashMap<>();
// for(int i = 0; i < headerRow.size(); i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i);
// }
// }
// }
// int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0);
// //////////////////////////////////////////////////////
// // figure out how many sub-rows we'll be processing //
// //////////////////////////////////////////////////////
// for(int i = 0; i < maxIndex; i++)
// {
// QRecord associatedRecord = new QRecord();
// boolean gotAnyValues = false;
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// String fieldName = entry.getKey();
// if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i)
// {
// Integer index = indexes.get(fieldName).get(i);
// Serializable value = row.getValueElseNull(index);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, mapping, row, index);
// }
// }
// if(gotAnyValues)
// {
// processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size());
// rs.add(associatedRecord);
// }
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
}

60
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java Normal file
View File

@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.ByteArrayInputStream;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
/*******************************************************************************
** Unit test for CsvFileToRows
*******************************************************************************/
class CsvFileToRowsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
byte[] csvBytes = """
one,two,three
1,2,3,4
""".getBytes();
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes));
BulkLoadFileRow headerRow = fileToRowsInterface.next();
BulkLoadFileRow bodyRow = fileToRowsInterface.next();
assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }), headerRow);
assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }), bodyRow);
assertFalse(fileToRowsInterface.hasNext());
}
}

86
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java Normal file
View File

@ -0,0 +1,86 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.InputStream;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
/***************************************************************************
**
***************************************************************************/
public class TestFileToRows extends AbstractIteratorBasedFileToRows<Serializable[]> implements FileToRowsInterface
{
private final List<Serializable[]> rows;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public TestFileToRows(List<Serializable[]> rows)
{
this.rows = rows;
setIterator(this.rows.iterator());
}
/***************************************************************************
**
***************************************************************************/
@Override
public void init(InputStream inputStream) throws QException
{
///////////
// noop! //
///////////
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws Exception
{
///////////
// noop! //
///////////
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow makeRow(Serializable[] values)
{
return (new BulkLoadFileRow(values));
}
}

106
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java Normal file
View File

@ -0,0 +1,106 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.time.LocalDate;
import java.time.Month;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction;
import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest;
import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
/*******************************************************************************
** Unit test for XlsxFileToRows
*******************************************************************************/
class XlsxFileToRowsTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test() throws QException
{
byte[] byteArray = writeExcelBytes();
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray));
BulkLoadFileRow headerRow = fileToRowsInterface.next();
BulkLoadFileRow bodyRow = fileToRowsInterface.next();
assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow);
assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow);
///////////////////////////////////////////////////////////////////////////////////////
// make sure there's at least a limit (less than 20) to how many more rows there are //
///////////////////////////////////////////////////////////////////////////////////////
int otherRowCount = 0;
while(fileToRowsInterface.hasNext() && otherRowCount < 20)
{
fileToRowsInterface.next();
otherRowCount++;
}
assertFalse(fileToRowsInterface.hasNext());
}
/***************************************************************************
**
***************************************************************************/
private static byte[] writeExcelBytes() throws QException
{
ReportFormat format = ReportFormat.XLSX;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
QInstance qInstance = QContext.getQInstance();
qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport());
GenerateReportActionTest.insertPersonRecords(qInstance);
ReportInput reportInput = new ReportInput();
reportInput.setReportName(REPORT_NAME);
reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos));
reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now()));
reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new);
new GenerateReportAction().execute(reportInput);
byte[] byteArray = baos.toByteArray();
return byteArray;
}
}

150
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java Normal file
View File

@ -0,0 +1,150 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for FlatRowsToRecord
*******************************************************************************/
class FlatRowsToRecordTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldNameToHeaderNameMapping() throws QException
{
TestFileToRows fileToRows = new TestFileToRows(List.of(
new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" },
new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" },
new Serializable[] { 2, "Marge", "Simpson", false, "" },
new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" },
new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" }
));
BulkLoadFileRow header = fileToRows.next();
FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name",
"cost", "cost"
))
.withFieldNameToDefaultValueMap(Map.of(
"noOfShoes", 2
))
.withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00"))))
.withTableName(TestUtils.TABLE_NAME_PERSON)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 1);
assertEquals(List.of("Homer"), getValues(records, "firstName"));
assertEquals(List.of("Simpson"), getValues(records, "lastName"));
assertEquals(List.of(2), getValues(records, "noOfShoes"));
assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost"));
assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set
records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost"));
records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(List.of("Ned"), getValues(records, "firstName"));
assertEquals(List.of(2), getValues(records, "noOfShoes"));
assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testFieldNameToIndexMapping() throws QException
{
TestFileToRows fileToRows = new TestFileToRows(List.of(
new Serializable[] { 1, "Homer", "Simpson", true },
new Serializable[] { 2, "Marge", "Simpson", false },
new Serializable[] { 3, "Bart", "Simpson", "A" },
new Serializable[] { 4, "Ned", "Flanders", 3.1 }
));
BulkLoadFileRow header = null;
FlatRowsToRecord rowsToRecord = new FlatRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToIndexMap(Map.of(
"firstName", 1,
"lastName", 2
))
.withFieldNameToDefaultValueMap(Map.of(
"noOfShoes", 2
))
.withTableName(TestUtils.TABLE_NAME_PERSON)
.withHasHeaderRow(false);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, 1);
assertEquals(List.of("Homer"), getValues(records, "firstName"));
assertEquals(List.of("Simpson"), getValues(records, "lastName"));
assertEquals(List.of(2), getValues(records, "noOfShoes"));
assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set
records = rowsToRecord.nextPage(fileToRows, header, mapping, 2);
assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName"));
assertEquals(List.of(2, 2), getValues(records, "noOfShoes"));
records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(List.of("Ned"), getValues(records, "firstName"));
assertEquals(List.of(2), getValues(records, "noOfShoes"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

284
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java Normal file
View File

@ -0,0 +1,284 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
** Unit test for TallRowsToRecord
*******************************************************************************/
class TallRowsToRecordTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLines() throws QException
{
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName, SKU, Quantity
1, Homer, Simpson, DONUT, 12
, Homer, Simpson, BEER, 500
, Homer, Simpson, COUCH, 1
2, Ned, Flanders, BIBLE, 7
, Ned, Flanders, LAWNMOWER, 1
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity"
))
.withTallLayoutGroupByIndexMap(Map.of(
TestUtils.TABLE_NAME_ORDER, List.of(1, 2),
"orderLine", List.of(3)
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsic() throws QException
{
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value
1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart
1, , , BEER, 500, Coupon Code, 10QOff
1, , , COUCH, 1
2, Ned, Flanders, BIBLE, 7
2, , , LAWNMOWER, 1
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value"
))
.withTallLayoutGroupByIndexMap(Map.of(
TestUtils.TABLE_NAME_ORDER, List.of(0),
"orderLine", List.of(3),
"extrinsics", List.of(5)
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(3, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(2, order.getAssociatedRecords().get("extrinsics").size());
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(2, order.getAssociatedRecords().get("orderLine").size());
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException
{
Integer DEFAULT_STORE_ID = 101;
Integer DEFAULT_LINE_NO = 102;
String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file";
CsvFileToRows fileToRows = CsvFileToRows.forString("""
orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value
1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate
1, , , DONUT, , Coupon Code, 10QOff, Size, Large
1, , , BEER, 500, , , Flavor, Hops
1, , , COUCH, 1
2, Ned, Flanders, BIBLE, 7, , , Flavor, King James
2, , , LAWNMOWER, 1
""");
BulkLoadFileRow header = fileToRows.next();
TallRowsToRecord rowsToRecord = new TallRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value",
"orderLine.extrinsics.key", "Line Extrinsic Key",
"orderLine.extrinsics.value", "Line Extrinsic Value"
))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", DEFAULT_STORE_ID,
"orderLine.lineNumber", DEFAULT_LINE_NO,
"orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE
))
.withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT")))
.withTallLayoutGroupByIndexMap(Map.of(
TestUtils.TABLE_NAME_ORDER, List.of(0),
"orderLine", List.of(3),
"extrinsics", List.of(5),
"orderLine.extrinsics", List.of(7)
))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.TALL)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(DEFAULT_STORE_ID, order.getValue("storeId"));
assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(DEFAULT_STORE_ID, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testShouldProcessAssociation()
{
TallRowsToRecord tallRowsToRecord = new TallRowsToRecord();
assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo"));
assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo"));
assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar"));
assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz"));
assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar"));
assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar"));
assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar"));
assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar"));
assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}

134
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java Normal file
View File

@ -0,0 +1,134 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Unit test for ValueMapper
*******************************************************************************/
class ValueMapperTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void test()
{
BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of(
"storeId", Map.of("QQQMart", 1, "Q'R'Us", 2),
"shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"),
"lineItem.sku", Map.of("ABC", "Alphabet"),
"lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"),
"extrinsics.key", Map.of("1", "one", "2", "two")
));
QRecord inputRecord = new QRecord()
.withValue("storeId", "QQQMart")
.withValue("shipToName", "HoJu")
.withAssociatedRecord("lineItem", new QRecord()
.withValue("sku", "ABC")
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", "myKey")
.withValue("value", "foo")
)
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", "yourKey")
.withValue("value", "bar")
)
)
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", 1)
.withValue("value", "foo")
);
JSONObject beforeJson = recordToJson(inputRecord);
QRecord expectedRecord = new QRecord()
.withValue("storeId", 1)
.withValue("shipToName", "Homer")
.withAssociatedRecord("lineItem", new QRecord()
.withValue("sku", "Alphabet")
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", "myKey")
.withValue("value", "bar")
)
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", "yourKey")
.withValue("value", "baz")
)
)
.withAssociatedRecord("extrinsics", new QRecord()
.withValue("key", "one")
.withValue("value", "foo")
);
JSONObject expectedJson = recordToJson(expectedRecord);
ValueMapper.valueMapping(List.of(inputRecord), mapping);
JSONObject actualJson = recordToJson(inputRecord);
System.out.println("Before");
System.out.println(beforeJson.toString(3));
System.out.println("Actual");
System.out.println(actualJson.toString(3));
System.out.println("Expected");
System.out.println(expectedJson.toString(3));
assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson);
}
/***************************************************************************
**
***************************************************************************/
public static JSONObject recordToJson(QRecord record)
{
JSONObject jsonObject = new JSONObject();
for(Map.Entry<String, Serializable> valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet())
{
jsonObject.put(valueEntry.getKey(), valueEntry.getValue());
}
for(Map.Entry<String, List<QRecord>> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet())
{
JSONArray jsonArray = new JSONArray();
for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue()))
{
jsonArray.put(recordToJson(associationRecord));
}
jsonObject.put(associationEntry.getKey(), jsonArray);
}
return (jsonObject);
}
}

305
qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java Normal file
View File

@ -0,0 +1,305 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
/*******************************************************************************
** Unit test for WideRowsToRecord
*******************************************************************************/
class WideRowsToRecordTest extends BaseTest
{
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithoutDupes() throws QException
{
testOrderAndLines("""
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderAndLinesWithDupes() throws QException
{
testOrderAndLines("""
orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""");
}
/***************************************************************************
**
***************************************************************************/
private void testOrderAndLines(String csv) throws QException
{
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity"
))
.withMappedAssociations(List.of("orderLine"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException
{
testOrderLinesAndOrderExtrinsic("""
orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesAndOrderExtrinsicWithDupes() throws QException
{
testOrderLinesAndOrderExtrinsic("""
orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value
1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff
2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1
""");
}
/*******************************************************************************
**
*******************************************************************************/
private void testOrderLinesAndOrderExtrinsic(String csv) throws QException
{
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value"
))
.withMappedAssociations(List.of("orderLine", "extrinsics"))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException
{
testOrderLinesWithLineExtrinsicsAndOrderExtrinsic("""
orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2, Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo,
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
""");
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException
{
testOrderLinesWithLineExtrinsicsAndOrderExtrinsic("""
orderNo, Ship To, lastName, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key
1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo
2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1
""");
}
/***************************************************************************
**
***************************************************************************/
private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException
{
Integer DEFAULT_STORE_ID = 42;
Integer DEFAULT_LINE_NO = 47;
String DEFAULT_LINE_EXTRA_VALUE = "bar";
CsvFileToRows fileToRows = CsvFileToRows.forString(csv);
BulkLoadFileRow header = fileToRows.next();
WideRowsToRecord rowsToRecord = new WideRowsToRecord();
BulkInsertMapping mapping = new BulkInsertMapping()
.withFieldNameToHeaderNameMap(Map.of(
"orderNo", "orderNo",
"shipToName", "Ship To",
"orderLine.sku", "SKU",
"orderLine.quantity", "Quantity",
"extrinsics.key", "Extrinsic Key",
"extrinsics.value", "Extrinsic Value",
"orderLine.extrinsics.key", "Line Extrinsic Key",
"orderLine.extrinsics.value", "Line Extrinsic Value"
))
.withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics"))
.withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL")))
.withFieldNameToDefaultValueMap(Map.of(
"storeId", DEFAULT_STORE_ID,
"orderLine.lineNumber", DEFAULT_LINE_NO,
"orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE
))
.withTableName(TestUtils.TABLE_NAME_ORDER)
.withLayout(BulkInsertMapping.Layout.WIDE)
.withHasHeaderRow(true);
List<QRecord> records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE);
assertEquals(2, records.size());
QRecord order = records.get(0);
assertEquals(1, order.getValueInteger("orderNo"));
assertEquals("Homer", order.getValueString("shipToName"));
assertEquals(DEFAULT_STORE_ID, order.getValue("storeId"));
assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value"));
QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(1);
assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
lineItem = order.getAssociatedRecords().get("orderLine").get(2);
assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
order = records.get(1);
assertEquals(2, order.getValueInteger("orderNo"));
assertEquals("Ned", order.getValueString("shipToName"));
assertEquals(DEFAULT_STORE_ID, order.getValue("storeId"));
assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku"));
assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity"));
assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber"));
assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty();
lineItem = order.getAssociatedRecords().get("orderLine").get(0);
assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key"));
assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value"));
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getValues(List<QRecord> records, String fieldName)
{
return (records.stream().map(r -> r.getValue(fieldName)).toList());
}
}