QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-20 06:00:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

CE-1955 Initial checkin

Browse Source
This commit is contained in:
darin.kelkhoff
2024-11-12 09:16:59 -06:00
parent 7d058530d5
commit 7ba205a262
22 changed files with 3404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java Normal file
View File

@ -0,0 +1,81 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveFileStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
try
(
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
BulkLoadFileRow headerRow = fileToRowsInterface.next();
List<String> bodyRows = new ArrayList<>();
while(fileToRowsInterface.hasNext() && bodyRows.size() < 20)
{
bodyRows.add(fileToRowsInterface.next().toString());
}
runBackendStepOutput.addValue("header", headerRow.toString());
runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows));
System.out.println("Done receiving file");
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
}

59
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java Normal file
View File

@ -0,0 +1,59 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertReceiveMappingStep implements BackendStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
BulkInsertMapping bulkInsertMapping = new BulkInsertMapping();
bulkInsertMapping.setTableName(runBackendStepInput.getTableName());
bulkInsertMapping.setHasHeaderRow(true);
bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of(
"firstName", "firstName",
"lastName", "Last Name"
));
runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping);
// probably need to what, receive the mapping object, store it into state
// what, do we maybe return to a different sub-mapping screen (e.g., values)
// then at some point - cool - proceed to ETL's steps
}
}

58
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java Normal file
View File

@ -0,0 +1,58 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.util.ArrayList;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertStepUtils
{
/***************************************************************************
**
***************************************************************************/
public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException
{
@SuppressWarnings("unchecked")
ArrayList<StorageInput> storageInputs = (ArrayList<StorageInput>) input.getValue("theFile");
if(storageInputs == null)
{
throw (new QException("StorageInputs for theFile were not found in process state"));
}
if(storageInputs.isEmpty())
{
throw (new QException("StorageInputs for theFile was an empty list"));
}
StorageInput storageInput = storageInputs.get(0);
return (storageInput);
}
}

126
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java Normal file
View File

@ -0,0 +1,126 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.InputStream;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
/*******************************************************************************
** Extract step for generic table bulk-insert ETL process
**
** This step does a little bit of transforming, actually - taking rows from
** an uploaded file, and potentially merging them (for child-table use-cases)
** and applying the "Mapping" - to put fully built records into the pipe for the
** Transform step.
*******************************************************************************/
public class BulkInsertV2ExtractStep extends AbstractExtractStep
{
/***************************************************************************
**
***************************************************************************/
@Override
public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
{
int rowsAdded = 0;
int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE);
StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput);
BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping");
RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface();
try
(
///////////////////////////////////////////////////////////////////////////////////////////////////////////
// open a stream to read from our file, and a FileToRows object, that knows how to read from that stream //
///////////////////////////////////////////////////////////////////////////////////////////////////////////
InputStream inputStream = new StorageAction().getInputStream(storageInput);
FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream);
)
{
///////////////////////////////////////////////////////////
// read the header row (if this file & mapping uses one) //
///////////////////////////////////////////////////////////
BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null;
////////////////////////////////////////////////////////////////////////////////////////////////////////
// while there are more rows in the file - and we're under the limit - get more records form the file //
////////////////////////////////////////////////////////////////////////////////////////////////////////
while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit)
{
int remainingLimit = originalLimit - rowsAdded;
////////////////////////////////////////////////////////////////////////////////////////////////////////////
// put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once //
////////////////////////////////////////////////////////////////////////////////////////////////////////////
int pageLimit = Math.min(remainingLimit, getMaxPageSize());
List<QRecord> page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit);
if(page.size() > remainingLimit)
{
/////////////////////////////////////////////////////////////
// in case we got back more than we asked for, sub-list it //
/////////////////////////////////////////////////////////////
page = page.subList(0, remainingLimit);
}
/////////////////////////////////////////////
// send this page of records into the pipe //
/////////////////////////////////////////////
getRecordPipe().addRecords(page);
rowsAdded += page.size();
}
}
catch(QException qe)
{
throw qe;
}
catch(Exception e)
{
throw new QException("Unhandled error in bulk insert extract step", e);
}
}
/***************************************************************************
**
***************************************************************************/
private int getMaxPageSize()
{
return (1000);
}
}

167
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java Normal file
View File

@ -0,0 +1,167 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Objects;
import java.util.stream.Collectors;
/*******************************************************************************
** A row of values, e.g., from a file, for bulk-load
*******************************************************************************/
public class BulkLoadFileRow implements Serializable
{
private Serializable[] values;
/*******************************************************************************
** Constructor
**
*******************************************************************************/
public BulkLoadFileRow(Serializable[] values)
{
this.values = values;
}
/***************************************************************************
**
***************************************************************************/
public int size()
{
if(values == null)
{
return (0);
}
return (values.length);
}
/***************************************************************************
**
***************************************************************************/
public boolean hasIndex(int i)
{
if(values == null)
{
return (false);
}
if(i >= values.length || i < 0)
{
return (false);
}
return (true);
}
/***************************************************************************
**
***************************************************************************/
public Serializable getValue(int i)
{
if(values == null)
{
throw new IllegalStateException("Row has no values");
}
if(i >= values.length || i < 0)
{
throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length);
}
return (values[i]);
}
/***************************************************************************
**
***************************************************************************/
public Serializable getValueElseNull(int i)
{
if(!hasIndex(i))
{
return (null);
}
return (values[i]);
}
/***************************************************************************
**
***************************************************************************/
@Override
public String toString()
{
if(values == null)
{
return ("null");
}
return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(","));
}
/***************************************************************************
**
***************************************************************************/
@Override
public boolean equals(Object o)
{
if(this == o)
{
return true;
}
if(o == null || getClass() != o.getClass())
{
return false;
}
BulkLoadFileRow row = (BulkLoadFileRow) o;
return Objects.deepEquals(values, row.values);
}
/***************************************************************************
**
***************************************************************************/
@Override
public int hashCode()
{
return Arrays.hashCode(values);
}
}

125
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java Normal file
View File

@ -0,0 +1,125 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.util.Iterator;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
/*******************************************************************************
**
*******************************************************************************/
public abstract class AbstractIteratorBasedFileToRows<E> implements FileToRowsInterface
{
private Iterator<E> iterator;
private boolean useLast = false;
private BulkLoadFileRow last;
/***************************************************************************
**
***************************************************************************/
@Override
public boolean hasNext()
{
if(iterator == null)
{
throw new IllegalStateException("Object was not init'ed");
}
if(useLast)
{
return true;
}
return iterator.hasNext();
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow next()
{
if(iterator == null)
{
throw new IllegalStateException("Object was not init'ed");
}
if(useLast)
{
useLast = false;
return (this.last);
}
E e = iterator.next();
BulkLoadFileRow row = makeRow(e);
this.last = row;
return (this.last);
}
/***************************************************************************
**
***************************************************************************/
public abstract BulkLoadFileRow makeRow(E e);
/***************************************************************************
**
***************************************************************************/
@Override
public void unNext()
{
useLast = true;
}
/*******************************************************************************
** Getter for iterator
*******************************************************************************/
public Iterator<E> getIterator()
{
return (this.iterator);
}
/*******************************************************************************
** Setter for iterator
*******************************************************************************/
public void setIterator(Iterator<E> iterator)
{
this.iterator = iterator;
}
}

112
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java Normal file
View File

@ -0,0 +1,112 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Serializable;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
/*******************************************************************************
**
*******************************************************************************/
public class CsvFileToRows extends AbstractIteratorBasedFileToRows<CSVRecord> implements FileToRowsInterface
{
private CSVParser csvParser;
/***************************************************************************
**
***************************************************************************/
public static CsvFileToRows forString(String csv) throws QException
{
CsvFileToRows csvFileToRows = new CsvFileToRows();
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes());
csvFileToRows.init(byteArrayInputStream);
return (csvFileToRows);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void init(InputStream inputStream) throws QException
{
try
{
csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT
.withIgnoreSurroundingSpaces()
);
setIterator(csvParser.iterator());
}
catch(IOException e)
{
throw new QException("Error opening CSV Parser", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow makeRow(CSVRecord csvRecord)
{
Serializable[] values = new Serializable[csvRecord.size()];
int i = 0;
for(String s : csvRecord)
{
values[i++] = s;
}
return (new BulkLoadFileRow(values));
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws Exception
{
if(csvParser != null)
{
csvParser.close();
}
}
}

74
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java Normal file
View File

@ -0,0 +1,74 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.InputStream;
import java.util.Iterator;
import java.util.Locale;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
/*******************************************************************************
**
*******************************************************************************/
public interface FileToRowsInterface extends AutoCloseable, Iterator<BulkLoadFileRow>
{
/***************************************************************************
**
***************************************************************************/
static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException
{
FileToRowsInterface rs;
if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv"))
{
rs = new CsvFileToRows();
}
else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx"))
{
rs = new XlsxFileToRows();
}
else
{
throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx"));
}
rs.init(inputStream);
return rs;
}
/*******************************************************************************
**
*******************************************************************************/
void init(InputStream inputStream) throws QException;
/***************************************************************************
**
***************************************************************************/
void unNext();
}

102
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java Normal file
View File

@ -0,0 +1,102 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling;
import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Sheet;
/*******************************************************************************
**
*******************************************************************************/
public class XlsxFileToRows extends AbstractIteratorBasedFileToRows<org.dhatim.fastexcel.reader.Row> implements FileToRowsInterface
{
private ReadableWorkbook workbook;
private Stream<org.dhatim.fastexcel.reader.Row> rows;
/***************************************************************************
**
***************************************************************************/
@Override
public void init(InputStream inputStream) throws QException
{
try
{
workbook = new ReadableWorkbook(inputStream);
Sheet sheet = workbook.getFirstSheet();
rows = sheet.openStream();
setIterator(rows.iterator());
}
catch(IOException e)
{
throw new QException("Error opening XLSX Parser", e);
}
}
/***************************************************************************
**
***************************************************************************/
@Override
public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow)
{
Serializable[] values = new Serializable[readerRow.getCellCount()];
for(int i = 0; i < readerRow.getCellCount(); i++)
{
values[i] = readerRow.getCell(i).getText();
}
return new BulkLoadFileRow(values);
}
/***************************************************************************
**
***************************************************************************/
@Override
public void close() throws Exception
{
if(workbook != null)
{
workbook.close();
}
if(rows != null)
{
rows.close();
}
}
}

503
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java Normal file
View File

@ -0,0 +1,503 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class BulkInsertMapping implements Serializable
{
private String tableName;
private Boolean hasHeaderRow;
private Layout layout;
/////////////////////////////////////////////////////////////////////
// keys in here are: //
// fieldName (for the main table) //
// association.fieldName (for an associated child table) //
// association.association.fieldName (for grandchild associations) //
/////////////////////////////////////////////////////////////////////
private Map<String, String> fieldNameToHeaderNameMap = new HashMap<>();
private Map<String, Integer> fieldNameToIndexMap = new HashMap<>();
private Map<String, Serializable> fieldNameToDefaultValueMap = new HashMap<>();
private Map<String, Map<String, Serializable>> fieldNameToValueMapping = new HashMap<>();
private Map<String, List<Integer>> tallLayoutGroupByIndexMap = new HashMap<>();
private List<String> mappedAssociations = new ArrayList<>();
private Memoization<Pair<String, String>, Boolean> shouldProcessFieldForTable = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
public enum Layout
{
FLAT(FlatRowsToRecord::new),
TALL(TallRowsToRecord::new),
WIDE(WideRowsToRecord::new);
/***************************************************************************
**
***************************************************************************/
private final Supplier<? extends RowsToRecordInterface> supplier;
/***************************************************************************
**
***************************************************************************/
Layout(Supplier<? extends RowsToRecordInterface> supplier)
{
this.supplier = supplier;
}
/***************************************************************************
**
***************************************************************************/
public RowsToRecordInterface newRowsToRecordInterface()
{
return (supplier.get());
}
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Integer> getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException
{
if(hasHeaderRow && fieldNameToHeaderNameMap != null)
{
return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow));
}
else if(fieldNameToIndexMap != null)
{
return (fieldNameToIndexMap);
}
throw (new QException("Mapping was not properly configured."));
}
/***************************************************************************
**
***************************************************************************/
@JsonIgnore
public Map<String, Map<String, Serializable>> getFieldNameToValueMappingForTable(String associatedTableName)
{
Map<String, Map<String, Serializable>> rs = new HashMap<>();
for(Map.Entry<String, Map<String, Serializable>> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet())
{
if(shouldProcessFieldForTable(entry.getKey(), associatedTableName))
{
String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey();
rs.put(key, entry.getValue());
}
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain)
{
return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p ->
{
List<String> fieldNameParts = new ArrayList<>();
List<String> associationParts = new ArrayList<>();
if(StringUtils.hasContent(fieldNameWithAssociationPrefix))
{
fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\.")));
}
if(StringUtils.hasContent(associationChain))
{
associationParts.addAll(Arrays.asList(associationChain.split("\\.")));
}
if(!fieldNameParts.isEmpty())
{
fieldNameParts.remove(fieldNameParts.size() - 1);
}
return (fieldNameParts.equals(associationParts));
}).orElse(false);
}
/***************************************************************************
**
***************************************************************************/
private Map<String, Integer> getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow)
{
Map<String, Integer> rs = new HashMap<>();
////////////////////////////////////////////////////////
// for the current file, map header values to indexes //
////////////////////////////////////////////////////////
Map<String, Integer> headerToIndexMap = new HashMap<>();
for(int i = 0; i < headerRow.size(); i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
headerToIndexMap.put(headerValue, i);
}
/////////////////////////////////////////////////////////////////////////////////////////////////////////
// loop over fields - finding what header name they are mapped to - then what index that header is at. //
/////////////////////////////////////////////////////////////////////////////////////////////////////////
String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + ".";
for(QFieldMetaData field : table.getFields().values())
{
String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName());
if(headerName != null)
{
Integer headerIndex = headerToIndexMap.get(headerName);
if(headerIndex != null)
{
rs.put(field.getName(), headerIndex);
}
}
}
return (rs);
}
/*******************************************************************************
** Getter for tableName
*******************************************************************************/
public String getTableName()
{
return (this.tableName);
}
/*******************************************************************************
** Setter for tableName
*******************************************************************************/
public void setTableName(String tableName)
{
this.tableName = tableName;
}
/*******************************************************************************
** Fluent setter for tableName
*******************************************************************************/
public BulkInsertMapping withTableName(String tableName)
{
this.tableName = tableName;
return (this);
}
/*******************************************************************************
** Getter for hasHeaderRow
*******************************************************************************/
public Boolean getHasHeaderRow()
{
return (this.hasHeaderRow);
}
/*******************************************************************************
** Setter for hasHeaderRow
*******************************************************************************/
public void setHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
}
/*******************************************************************************
** Fluent setter for hasHeaderRow
*******************************************************************************/
public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow)
{
this.hasHeaderRow = hasHeaderRow;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToHeaderNameMap
*******************************************************************************/
public Map<String, String> getFieldNameToHeaderNameMap()
{
return (this.fieldNameToHeaderNameMap);
}
/*******************************************************************************
** Setter for fieldNameToHeaderNameMap
*******************************************************************************/
public void setFieldNameToHeaderNameMap(Map<String, String> fieldNameToHeaderNameMap)
{
this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToHeaderNameMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToHeaderNameMap(Map<String, String> fieldNameToHeaderNameMap)
{
this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToIndexMap
*******************************************************************************/
public Map<String, Integer> getFieldNameToIndexMap()
{
return (this.fieldNameToIndexMap);
}
/*******************************************************************************
** Setter for fieldNameToIndexMap
*******************************************************************************/
public void setFieldNameToIndexMap(Map<String, Integer> fieldNameToIndexMap)
{
this.fieldNameToIndexMap = fieldNameToIndexMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToIndexMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToIndexMap(Map<String, Integer> fieldNameToIndexMap)
{
this.fieldNameToIndexMap = fieldNameToIndexMap;
return (this);
}
/*******************************************************************************
** Getter for mappedAssociations
*******************************************************************************/
public List<String> getMappedAssociations()
{
return (this.mappedAssociations);
}
/*******************************************************************************
** Setter for mappedAssociations
*******************************************************************************/
public void setMappedAssociations(List<String> mappedAssociations)
{
this.mappedAssociations = mappedAssociations;
}
/*******************************************************************************
** Fluent setter for mappedAssociations
*******************************************************************************/
public BulkInsertMapping withMappedAssociations(List<String> mappedAssociations)
{
this.mappedAssociations = mappedAssociations;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToDefaultValueMap
*******************************************************************************/
public Map<String, Serializable> getFieldNameToDefaultValueMap()
{
if(this.fieldNameToDefaultValueMap == null)
{
this.fieldNameToDefaultValueMap = new HashMap<>();
}
return (this.fieldNameToDefaultValueMap);
}
/*******************************************************************************
** Setter for fieldNameToDefaultValueMap
*******************************************************************************/
public void setFieldNameToDefaultValueMap(Map<String, Serializable> fieldNameToDefaultValueMap)
{
this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap;
}
/*******************************************************************************
** Fluent setter for fieldNameToDefaultValueMap
*******************************************************************************/
public BulkInsertMapping withFieldNameToDefaultValueMap(Map<String, Serializable> fieldNameToDefaultValueMap)
{
this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap;
return (this);
}
/*******************************************************************************
** Getter for fieldNameToValueMapping
*******************************************************************************/
public Map<String, Map<String, Serializable>> getFieldNameToValueMapping()
{
return (this.fieldNameToValueMapping);
}
/*******************************************************************************
** Setter for fieldNameToValueMapping
*******************************************************************************/
public void setFieldNameToValueMapping(Map<String, Map<String, Serializable>> fieldNameToValueMapping)
{
this.fieldNameToValueMapping = fieldNameToValueMapping;
}
/*******************************************************************************
** Fluent setter for fieldNameToValueMapping
*******************************************************************************/
public BulkInsertMapping withFieldNameToValueMapping(Map<String, Map<String, Serializable>> fieldNameToValueMapping)
{
this.fieldNameToValueMapping = fieldNameToValueMapping;
return (this);
}
/*******************************************************************************
** Getter for layout
*******************************************************************************/
public Layout getLayout()
{
return (this.layout);
}
/*******************************************************************************
** Setter for layout
*******************************************************************************/
public void setLayout(Layout layout)
{
this.layout = layout;
}
/*******************************************************************************
** Fluent setter for layout
*******************************************************************************/
public BulkInsertMapping withLayout(Layout layout)
{
this.layout = layout;
return (this);
}
/*******************************************************************************
** Getter for tallLayoutGroupByIndexMap
*******************************************************************************/
public Map<String, List<Integer>> getTallLayoutGroupByIndexMap()
{
return (this.tallLayoutGroupByIndexMap);
}
/*******************************************************************************
** Setter for tallLayoutGroupByIndexMap
*******************************************************************************/
public void setTallLayoutGroupByIndexMap(Map<String, List<Integer>> tallLayoutGroupByIndexMap)
{
this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap;
}
/*******************************************************************************
** Fluent setter for tallLayoutGroupByIndexMap
*******************************************************************************/
public BulkInsertMapping withTallLayoutGroupByIndexMap(Map<String, List<Integer>> tallLayoutGroupByIndexMap)
{
this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap;
return (this);
}
}

75
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java Normal file
View File

@ -0,0 +1,75 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
/*******************************************************************************
**
*******************************************************************************/
public class FlatRowsToRecord implements RowsToRecordInterface
{
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
}
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
}

69
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java Normal file
View File

@ -0,0 +1,69 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public interface RowsToRecordInterface
{
/***************************************************************************
**
***************************************************************************/
List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException;
/***************************************************************************
**
***************************************************************************/
default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index)
{
String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName;
Serializable value = null;
if(index != null && row != null)
{
value = row.getValueElseNull(index);
}
else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix))
{
value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix);
}
if(value != null)
{
record.setValue(fieldName, value);
}
}
}

316
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java Normal file
View File

@ -0,0 +1,316 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class TallRowsToRecord implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
List<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
String associationNameChain = "";
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName());
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
continue;
}
if(rowsForCurrentRecord.isEmpty())
{
///////////////////////////////////
// this is first - so it's a yes //
///////////////////////////////////
recordGroupByValues = rowGroupByValues;
rowsForCurrentRecord.add(row);
}
else if(Objects.equals(recordGroupByValues, rowGroupByValues))
{
/////////////////////////////
// a match - so keep going //
/////////////////////////////
rowsForCurrentRecord.add(row);
}
else
{
//////////////////////////////////////////////////////////////
// not first, and not a match, so we can finish this record //
//////////////////////////////////////////////////////////////
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
////////////////////////////////////////
// reset these record-specific values //
////////////////////////////////////////
rowsForCurrentRecord = new ArrayList<>();
recordGroupByValues = null;
//////////////////////////////////////////////////////////////////////////////////////////////////////
// we need to push this row back onto the fileToRows object, so it'll be handled in the next record //
//////////////////////////////////////////////////////////////////////////////////////////////////////
fileToRowsInterface.unNext();
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// i wrote this condition in here: && rs.size() < limit //
// but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... //
/////////////////////////////////////////////////////////////////////////////////////////////////
if(!rowsForCurrentRecord.isEmpty())
{
rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord));
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List<BulkLoadFileRow> rows) throws QException
{
QRecord record = new QRecord();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow);
//////////////////////////////////////////////////////
// get all rows for the main table from the 0th row //
//////////////////////////////////////////////////////
BulkLoadFileRow row = rows.get(0);
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName()));
}
/////////////////////////////
// associations (children) //
/////////////////////////////
for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations()))
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
List<QRecord> associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
return record;
}
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List<BulkLoadFileRow> rows) throws QException
{
List<QRecord> rs = new ArrayList<>();
QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName());
String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName;
List<BulkLoadFileRow> rowsForCurrentRecord = new ArrayList<>();
List<Serializable> recordGroupByValues = null;
for(BulkLoadFileRow row : rows)
{
List<Integer> groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls);
if(CollectionUtils.nullSafeIsEmpty(groupByIndexes))
{
throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls));
}
List<Serializable> rowGroupByValues = getGroupByValues(row, groupByIndexes);
if(rowGroupByValues == null)
{
continue;
}
if(rowsForCurrentRecord.isEmpty())
{
///////////////////////////////////
// this is first - so it's a yes //
///////////////////////////////////
recordGroupByValues = rowGroupByValues;
rowsForCurrentRecord.add(row);
}
else if(Objects.equals(recordGroupByValues, rowGroupByValues))
{
/////////////////////////////
// a match - so keep going //
/////////////////////////////
rowsForCurrentRecord.add(row);
}
else
{
//////////////////////////////////////////////////////////////
// not first, and not a match, so we can finish this record //
//////////////////////////////////////////////////////////////
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord));
////////////////////////////////////////
// reset these record-specific values //
////////////////////////////////////////
rowsForCurrentRecord = new ArrayList<>();
//////////////////////////////////////////////////
// use the current row to start the next record //
//////////////////////////////////////////////////
rowsForCurrentRecord.add(row);
recordGroupByValues = rowGroupByValues;
}
}
///////////
// final //
///////////
if(!rowsForCurrentRecord.isEmpty())
{
rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord));
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private List<Serializable> getGroupByValues(BulkLoadFileRow row, List<Integer> indexes)
{
List<Serializable> rowGroupByValues = new ArrayList<>();
boolean haveAnyGroupByValues = false;
for(Integer index : indexes)
{
Serializable value = row.getValueElseNull(index);
rowGroupByValues.add(value);
if(value != null && !"".equals(value))
{
haveAnyGroupByValues = true;
}
}
if(!haveAnyGroupByValues)
{
return (null);
}
return (rowGroupByValues);
}
}

79
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java Normal file
View File

@ -0,0 +1,79 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
**
*******************************************************************************/
public class ValueMapper
{
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping)
{
valueMapping(records, mapping, null);
}
/***************************************************************************
**
***************************************************************************/
public static void valueMapping(List<QRecord> records, BulkInsertMapping mapping, String associationNameChain)
{
if(CollectionUtils.nullSafeIsEmpty(records))
{
return;
}
Map<String, Map<String, Serializable>> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain);
for(QRecord record : records)
{
for(Map.Entry<String, Map<String, Serializable>> entry : mappingForTable.entrySet())
{
String fieldName = entry.getKey();
Map<String, Serializable> map = entry.getValue();
String value = record.getValueString(fieldName);
if(value != null && map.containsKey(value))
{
record.setValue(fieldName, map.get(value));
}
}
for(Map.Entry<String, List<QRecord>> entry : record.getAssociatedRecords().entrySet())
{
valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey());
}
}
}
}

333
qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java Normal file
View File

@ -0,0 +1,333 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2024. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface;
import com.kingsrook.qqq.backend.core.utils.Pair;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.memoization.Memoization;
/*******************************************************************************
**
*******************************************************************************/
public class WideRowsToRecord implements RowsToRecordInterface
{
private Memoization<Pair<String, String>, Boolean> shouldProcesssAssociationMemoization = new Memoization<>();
/***************************************************************************
**
***************************************************************************/
@Override
public List<QRecord> nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException
{
QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName());
if(table == null)
{
throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance"));
}
List<QRecord> rs = new ArrayList<>();
Map<String, Integer> fieldIndexes = mapping.getFieldIndexes(table, null, headerRow);
while(fileToRowsInterface.hasNext() && rs.size() < limit)
{
BulkLoadFileRow row = fileToRowsInterface.next();
QRecord record = new QRecord();
for(QFieldMetaData field : table.getFields().values())
{
setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName()));
}
processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size());
rs.add(record);
}
ValueMapper.valueMapping(rs, mapping);
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException
{
for(String associationName : mapping.getMappedAssociations())
{
boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName);
if(processAssociation)
{
String associationNameMinusChain = StringUtils.hasContent(associationNameChain)
? associationName.substring(associationNameChain.length() + 1)
: associationName;
Optional<Association> association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst();
if(association.isEmpty())
{
throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName()));
}
QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName());
// List<QRecord> associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record);
List<QRecord> associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex);
record.withAssociatedRecords(associationNameMinusChain, associatedRecords);
}
}
}
/***************************************************************************
**
***************************************************************************/
private List<QRecord> processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException
{
List<QRecord> rs = new ArrayList<>();
Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
{
if(entry.getKey().startsWith(associationName + "."))
{
String fieldName = entry.getKey().substring(associationName.length() + 1);
//////////////////////////////////////////////////////////////////////////
// make sure the name here is for this table - not a sub-table under it //
//////////////////////////////////////////////////////////////////////////
if(!fieldName.contains("."))
{
fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue());
}
}
}
/////////////////////////////////////////////////////////////////////
// loop over the length of the record, building associated records //
/////////////////////////////////////////////////////////////////////
QRecord associatedRecord = new QRecord();
Set<String> processedFieldNames = new HashSet<>();
boolean gotAnyValues = false;
int subStartIndex = -1;
for(int i = startIndex; i < endIndex; i++)
{
String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
{
if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
{
///////////////////////////////////////////////
// ok - this is a value for this association //
///////////////////////////////////////////////
if(subStartIndex == -1)
{
subStartIndex = i;
}
String fieldName = entry.getKey();
if(processedFieldNames.contains(fieldName))
{
/////////////////////////////////////////////////
// this means we're starting a new sub-record! //
/////////////////////////////////////////////////
if(gotAnyValues)
{
addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i);
rs.add(associatedRecord);
}
associatedRecord = new QRecord();
processedFieldNames = new HashSet<>();
gotAnyValues = false;
subStartIndex = i + 1;
}
processedFieldNames.add(fieldName);
Serializable value = row.getValueElseNull(i);
if(value != null && !"".equals(value))
{
gotAnyValues = true;
}
setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i);
}
}
}
////////////////////////
// handle final value //
////////////////////////
if(gotAnyValues)
{
addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName);
processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex);
rs.add(associatedRecord);
}
return (rs);
}
/***************************************************************************
**
***************************************************************************/
private void addDefaultValuesToAssociatedRecord(Set<String> processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain)
{
for(QFieldMetaData field : table.getFields().values())
{
if(!processedFieldNames.contains(field.getName()))
{
setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null);
}
}
}
/***************************************************************************
**
***************************************************************************/
// private List<QRecord> processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException
// {
// List<QRecord> rs = new ArrayList<>();
// String associationNameChainForRecursiveCalls = associationName;
// Map<String, String> fieldNameToHeaderNameMapForThisAssociation = new HashMap<>();
// for(Map.Entry<String, String> entry : mapping.getFieldNameToHeaderNameMap().entrySet())
// {
// if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + "."))
// {
// fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue());
// }
// }
// Map<String, List<Integer>> indexes = new HashMap<>();
// for(int i = 0; i < headerRow.size(); i++)
// {
// String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i));
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+"))
// {
// indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i);
// }
// }
// }
// int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0);
// //////////////////////////////////////////////////////
// // figure out how many sub-rows we'll be processing //
// //////////////////////////////////////////////////////
// for(int i = 0; i < maxIndex; i++)
// {
// QRecord associatedRecord = new QRecord();
// boolean gotAnyValues = false;
// for(Map.Entry<String, String> entry : fieldNameToHeaderNameMapForThisAssociation.entrySet())
// {
// String fieldName = entry.getKey();
// if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i)
// {
// Integer index = indexes.get(fieldName).get(i);
// Serializable value = row.getValueElseNull(index);
// if(value != null && !"".equals(value))
// {
// gotAnyValues = true;
// }
// setValueOrDefault(associatedRecord, fieldName, mapping, row, index);
// }
// }
// if(gotAnyValues)
// {
// processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size());
// rs.add(associatedRecord);
// }
// }
// return (rs);
// }
/***************************************************************************
**
***************************************************************************/
boolean shouldProcessAssociation(String associationNameChain, String associationName)
{
return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p ->
{
List<String> chainParts = new ArrayList<>();
List<String> nameParts = new ArrayList<>();
if(StringUtils.hasContent(associationNameChain))
{
chainParts.addAll(Arrays.asList(associationNameChain.split("\\.")));
}
if(StringUtils.hasContent(associationName))
{
nameParts.addAll(Arrays.asList(associationName.split("\\.")));
}
if(!nameParts.isEmpty())
{
nameParts.remove(nameParts.size() - 1);
}
return (chainParts.equals(nameParts));
}).orElse(false);
}
}