diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java new file mode 100644 index 00000000..6ebcf3a8 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveFileStep.java @@ -0,0 +1,81 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.JsonUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveFileStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + + try + ( + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + + List bodyRows = new ArrayList<>(); + while(fileToRowsInterface.hasNext() && bodyRows.size() < 20) + { + bodyRows.add(fileToRowsInterface.next().toString()); + } + + runBackendStepOutput.addValue("header", headerRow.toString()); + runBackendStepOutput.addValue("body", JsonUtils.toPrettyJson(bodyRows)); + System.out.println("Done receiving file"); + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java new file mode 100644 index 00000000..0071aa49 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertReceiveMappingStep.java @@ -0,0 +1,59 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertReceiveMappingStep implements BackendStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + BulkInsertMapping bulkInsertMapping = new BulkInsertMapping(); + bulkInsertMapping.setTableName(runBackendStepInput.getTableName()); + bulkInsertMapping.setHasHeaderRow(true); + bulkInsertMapping.setFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name" + )); + runBackendStepOutput.addValue("bulkInsertMapping", bulkInsertMapping); + + // probably need to what, receive the mapping object, store it into state + // what, do we maybe return to a different sub-mapping screen (e.g., values) + // then at some point - cool - proceed to ETL's steps + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java new file mode 100644 index 00000000..e3dac892 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertStepUtils.java @@ -0,0 +1,58 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.util.ArrayList; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertStepUtils +{ + + /*************************************************************************** + ** + ***************************************************************************/ + public static StorageInput getStorageInputForTheFile(RunBackendStepInput input) throws QException + { + @SuppressWarnings("unchecked") + ArrayList storageInputs = (ArrayList) input.getValue("theFile"); + if(storageInputs == null) + { + throw (new QException("StorageInputs for theFile were not found in process state")); + } + + if(storageInputs.isEmpty()) + { + throw (new QException("StorageInputs for theFile was an empty list")); + } + + StorageInput storageInput = storageInputs.get(0); + return (storageInput); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java new file mode 100644 index 00000000..2e49e48a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkInsertV2ExtractStep.java @@ -0,0 +1,126 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.InputStream; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.StorageAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.BulkInsertMapping; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping.RowsToRecordInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; + + +/******************************************************************************* + ** Extract step for generic table bulk-insert ETL process + ** + ** This step does a little bit of transforming, actually - taking rows from + ** an uploaded file, and potentially merging them (for child-table use-cases) + ** and applying the "Mapping" - to put fully built records into the pipe for the + ** Transform step. + *******************************************************************************/ +public class BulkInsertV2ExtractStep extends AbstractExtractStep +{ + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + int rowsAdded = 0; + int originalLimit = Objects.requireNonNullElse(getLimit(), Integer.MAX_VALUE); + + StorageInput storageInput = BulkInsertStepUtils.getStorageInputForTheFile(runBackendStepInput); + BulkInsertMapping bulkInsertMapping = (BulkInsertMapping) runBackendStepOutput.getValue("bulkInsertMapping"); + RowsToRecordInterface rowsToRecord = bulkInsertMapping.getLayout().newRowsToRecordInterface(); + + try + ( + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // open a stream to read from our file, and a FileToRows object, that knows how to read from that stream // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + InputStream inputStream = new StorageAction().getInputStream(storageInput); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile(storageInput.getReference(), inputStream); + ) + { + /////////////////////////////////////////////////////////// + // read the header row (if this file & mapping uses one) // + /////////////////////////////////////////////////////////// + BulkLoadFileRow headerRow = bulkInsertMapping.getHasHeaderRow() ? fileToRowsInterface.next() : null; + + //////////////////////////////////////////////////////////////////////////////////////////////////////// + // while there are more rows in the file - and we're under the limit - get more records form the file // + //////////////////////////////////////////////////////////////////////////////////////////////////////// + while(fileToRowsInterface.hasNext() && rowsAdded < originalLimit) + { + int remainingLimit = originalLimit - rowsAdded; + + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + // put a page-size limit on the rows-to-record class, so it won't be tempted to do whole file all at once // + //////////////////////////////////////////////////////////////////////////////////////////////////////////// + int pageLimit = Math.min(remainingLimit, getMaxPageSize()); + List page = rowsToRecord.nextPage(fileToRowsInterface, headerRow, bulkInsertMapping, pageLimit); + + if(page.size() > remainingLimit) + { + ///////////////////////////////////////////////////////////// + // in case we got back more than we asked for, sub-list it // + ///////////////////////////////////////////////////////////// + page = page.subList(0, remainingLimit); + } + + ///////////////////////////////////////////// + // send this page of records into the pipe // + ///////////////////////////////////////////// + getRecordPipe().addRecords(page); + rowsAdded += page.size(); + } + } + catch(QException qe) + { + throw qe; + } + catch(Exception e) + { + throw new QException("Unhandled error in bulk insert extract step", e); + } + + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private int getMaxPageSize() + { + return (1000); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java new file mode 100644 index 00000000..487f606a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/BulkLoadFileRow.java @@ -0,0 +1,167 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert; + + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + + +/******************************************************************************* + ** A row of values, e.g., from a file, for bulk-load + *******************************************************************************/ +public class BulkLoadFileRow implements Serializable +{ + private Serializable[] values; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public BulkLoadFileRow(Serializable[] values) + { + this.values = values; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public int size() + { + if(values == null) + { + return (0); + } + + return (values.length); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public boolean hasIndex(int i) + { + if(values == null) + { + return (false); + } + + if(i >= values.length || i < 0) + { + return (false); + } + + return (true); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValue(int i) + { + if(values == null) + { + throw new IllegalStateException("Row has no values"); + } + + if(i >= values.length || i < 0) + { + throw new IllegalArgumentException("Index out of bounds: Requested index " + i + "; values.length: " + values.length); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public Serializable getValueElseNull(int i) + { + if(!hasIndex(i)) + { + return (null); + } + + return (values[i]); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public String toString() + { + if(values == null) + { + return ("null"); + } + + return Arrays.stream(values).map(String::valueOf).collect(Collectors.joining(",")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean equals(Object o) + { + if(this == o) + { + return true; + } + + if(o == null || getClass() != o.getClass()) + { + return false; + } + + BulkLoadFileRow row = (BulkLoadFileRow) o; + return Objects.deepEquals(values, row.values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public int hashCode() + { + return Arrays.hashCode(values); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java new file mode 100644 index 00000000..383dfa21 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/AbstractIteratorBasedFileToRows.java @@ -0,0 +1,125 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.util.Iterator; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public abstract class AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private Iterator iterator; + + private boolean useLast = false; + private BulkLoadFileRow last; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public boolean hasNext() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + return true; + } + + return iterator.hasNext(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow next() + { + if(iterator == null) + { + throw new IllegalStateException("Object was not init'ed"); + } + + if(useLast) + { + useLast = false; + return (this.last); + } + + E e = iterator.next(); + + BulkLoadFileRow row = makeRow(e); + + this.last = row; + return (this.last); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public abstract BulkLoadFileRow makeRow(E e); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void unNext() + { + useLast = true; + } + + + + /******************************************************************************* + ** Getter for iterator + *******************************************************************************/ + public Iterator getIterator() + { + return (this.iterator); + } + + + + /******************************************************************************* + ** Setter for iterator + *******************************************************************************/ + public void setIterator(Iterator iterator) + { + this.iterator = iterator; + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java new file mode 100644 index 00000000..f39f4d45 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRows.java @@ -0,0 +1,112 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Serializable; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class CsvFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private CSVParser csvParser; + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static CsvFileToRows forString(String csv) throws QException + { + CsvFileToRows csvFileToRows = new CsvFileToRows(); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(csv.getBytes()); + csvFileToRows.init(byteArrayInputStream); + + return (csvFileToRows); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + csvParser = new CSVParser(new InputStreamReader(inputStream), CSVFormat.DEFAULT + .withIgnoreSurroundingSpaces() + ); + setIterator(csvParser.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening CSV Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(CSVRecord csvRecord) + { + Serializable[] values = new Serializable[csvRecord.size()]; + int i = 0; + for(String s : csvRecord) + { + values[i++] = s; + } + + return (new BulkLoadFileRow(values)); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(csvParser != null) + { + csvParser.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java new file mode 100644 index 00000000..d2d6c78a --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/FileToRowsInterface.java @@ -0,0 +1,74 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.util.Iterator; +import java.util.Locale; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface FileToRowsInterface extends AutoCloseable, Iterator +{ + + /*************************************************************************** + ** + ***************************************************************************/ + static FileToRowsInterface forFile(String fileName, InputStream inputStream) throws QException + { + FileToRowsInterface rs; + if(fileName.toLowerCase(Locale.ROOT).endsWith(".csv")) + { + rs = new CsvFileToRows(); + } + else if(fileName.toLowerCase(Locale.ROOT).endsWith(".xlsx")) + { + rs = new XlsxFileToRows(); + } + else + { + throw (new QUserFacingException("Unrecognized file extension - expecting .csv or .xlsx")); + } + + rs.init(inputStream); + return rs; + } + + + /******************************************************************************* + ** + *******************************************************************************/ + void init(InputStream inputStream) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + void unNext(); + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java new file mode 100644 index 00000000..21c9d928 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java @@ -0,0 +1,102 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.util.stream.Stream; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.dhatim.fastexcel.reader.ReadableWorkbook; +import org.dhatim.fastexcel.reader.Sheet; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private ReadableWorkbook workbook; + private Stream rows; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + try + { + workbook = new ReadableWorkbook(inputStream); + Sheet sheet = workbook.getFirstSheet(); + + rows = sheet.openStream(); + setIterator(rows.iterator()); + } + catch(IOException e) + { + throw new QException("Error opening XLSX Parser", e); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(org.dhatim.fastexcel.reader.Row readerRow) + { + Serializable[] values = new Serializable[readerRow.getCellCount()]; + + for(int i = 0; i < readerRow.getCellCount(); i++) + { + values[i] = readerRow.getCell(i).getText(); + } + + return new BulkLoadFileRow(values); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + if(workbook != null) + { + workbook.close(); + } + + if(rows != null) + { + rows.close(); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java new file mode 100644 index 00000000..d95d48e4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/BulkInsertMapping.java @@ -0,0 +1,503 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class BulkInsertMapping implements Serializable +{ + private String tableName; + private Boolean hasHeaderRow; + + private Layout layout; + + ///////////////////////////////////////////////////////////////////// + // keys in here are: // + // fieldName (for the main table) // + // association.fieldName (for an associated child table) // + // association.association.fieldName (for grandchild associations) // + ///////////////////////////////////////////////////////////////////// + private Map fieldNameToHeaderNameMap = new HashMap<>(); + private Map fieldNameToIndexMap = new HashMap<>(); + private Map fieldNameToDefaultValueMap = new HashMap<>(); + private Map> fieldNameToValueMapping = new HashMap<>(); + + private Map> tallLayoutGroupByIndexMap = new HashMap<>(); + private List mappedAssociations = new ArrayList<>(); + + private Memoization, Boolean> shouldProcessFieldForTable = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + public enum Layout + { + FLAT(FlatRowsToRecord::new), + TALL(TallRowsToRecord::new), + WIDE(WideRowsToRecord::new); + + + /*************************************************************************** + ** + ***************************************************************************/ + private final Supplier supplier; + + + + /*************************************************************************** + ** + ***************************************************************************/ + Layout(Supplier supplier) + { + this.supplier = supplier; + } + + + /*************************************************************************** + ** + ***************************************************************************/ + public RowsToRecordInterface newRowsToRecordInterface() + { + return (supplier.get()); + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map getFieldIndexes(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) throws QException + { + if(hasHeaderRow && fieldNameToHeaderNameMap != null) + { + return (getFieldIndexesForHeaderMappedUseCase(table, associationNameChain, headerRow)); + } + else if(fieldNameToIndexMap != null) + { + return (fieldNameToIndexMap); + } + + throw (new QException("Mapping was not properly configured.")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @JsonIgnore + public Map> getFieldNameToValueMappingForTable(String associatedTableName) + { + Map> rs = new HashMap<>(); + + for(Map.Entry> entry : CollectionUtils.nonNullMap(fieldNameToValueMapping).entrySet()) + { + if(shouldProcessFieldForTable(entry.getKey(), associatedTableName)) + { + String key = StringUtils.hasContent(associatedTableName) ? entry.getKey().substring(associatedTableName.length() + 1) : entry.getKey(); + rs.put(key, entry.getValue()); + } + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessFieldForTable(String fieldNameWithAssociationPrefix, String associationChain) + { + return shouldProcessFieldForTable.getResult(Pair.of(fieldNameWithAssociationPrefix, associationChain), p -> + { + List fieldNameParts = new ArrayList<>(); + List associationParts = new ArrayList<>(); + + if(StringUtils.hasContent(fieldNameWithAssociationPrefix)) + { + fieldNameParts.addAll(Arrays.asList(fieldNameWithAssociationPrefix.split("\\."))); + } + + if(StringUtils.hasContent(associationChain)) + { + associationParts.addAll(Arrays.asList(associationChain.split("\\."))); + } + + if(!fieldNameParts.isEmpty()) + { + fieldNameParts.remove(fieldNameParts.size() - 1); + } + + return (fieldNameParts.equals(associationParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private Map getFieldIndexesForHeaderMappedUseCase(QTableMetaData table, String associationNameChain, BulkLoadFileRow headerRow) + { + Map rs = new HashMap<>(); + + //////////////////////////////////////////////////////// + // for the current file, map header values to indexes // + //////////////////////////////////////////////////////// + Map headerToIndexMap = new HashMap<>(); + for(int i = 0; i < headerRow.size(); i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + headerToIndexMap.put(headerValue, i); + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + // loop over fields - finding what header name they are mapped to - then what index that header is at. // + ///////////////////////////////////////////////////////////////////////////////////////////////////////// + String fieldNamePrefix = !StringUtils.hasContent(associationNameChain) ? "" : associationNameChain + "."; + for(QFieldMetaData field : table.getFields().values()) + { + String headerName = fieldNameToHeaderNameMap.get(fieldNamePrefix + field.getName()); + if(headerName != null) + { + Integer headerIndex = headerToIndexMap.get(headerName); + if(headerIndex != null) + { + rs.put(field.getName(), headerIndex); + } + } + } + + return (rs); + } + + + + /******************************************************************************* + ** Getter for tableName + *******************************************************************************/ + public String getTableName() + { + return (this.tableName); + } + + + + /******************************************************************************* + ** Setter for tableName + *******************************************************************************/ + public void setTableName(String tableName) + { + this.tableName = tableName; + } + + + + /******************************************************************************* + ** Fluent setter for tableName + *******************************************************************************/ + public BulkInsertMapping withTableName(String tableName) + { + this.tableName = tableName; + return (this); + } + + + + /******************************************************************************* + ** Getter for hasHeaderRow + *******************************************************************************/ + public Boolean getHasHeaderRow() + { + return (this.hasHeaderRow); + } + + + + /******************************************************************************* + ** Setter for hasHeaderRow + *******************************************************************************/ + public void setHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + } + + + + /******************************************************************************* + ** Fluent setter for hasHeaderRow + *******************************************************************************/ + public BulkInsertMapping withHasHeaderRow(Boolean hasHeaderRow) + { + this.hasHeaderRow = hasHeaderRow; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToHeaderNameMap + *******************************************************************************/ + public Map getFieldNameToHeaderNameMap() + { + return (this.fieldNameToHeaderNameMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public void setFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToHeaderNameMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToHeaderNameMap(Map fieldNameToHeaderNameMap) + { + this.fieldNameToHeaderNameMap = fieldNameToHeaderNameMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToIndexMap + *******************************************************************************/ + public Map getFieldNameToIndexMap() + { + return (this.fieldNameToIndexMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToIndexMap + *******************************************************************************/ + public void setFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToIndexMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToIndexMap(Map fieldNameToIndexMap) + { + this.fieldNameToIndexMap = fieldNameToIndexMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for mappedAssociations + *******************************************************************************/ + public List getMappedAssociations() + { + return (this.mappedAssociations); + } + + + + /******************************************************************************* + ** Setter for mappedAssociations + *******************************************************************************/ + public void setMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + } + + + + /******************************************************************************* + ** Fluent setter for mappedAssociations + *******************************************************************************/ + public BulkInsertMapping withMappedAssociations(List mappedAssociations) + { + this.mappedAssociations = mappedAssociations; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToDefaultValueMap + *******************************************************************************/ + public Map getFieldNameToDefaultValueMap() + { + if(this.fieldNameToDefaultValueMap == null) + { + this.fieldNameToDefaultValueMap = new HashMap<>(); + } + + return (this.fieldNameToDefaultValueMap); + } + + + + /******************************************************************************* + ** Setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public void setFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToDefaultValueMap + *******************************************************************************/ + public BulkInsertMapping withFieldNameToDefaultValueMap(Map fieldNameToDefaultValueMap) + { + this.fieldNameToDefaultValueMap = fieldNameToDefaultValueMap; + return (this); + } + + + + /******************************************************************************* + ** Getter for fieldNameToValueMapping + *******************************************************************************/ + public Map> getFieldNameToValueMapping() + { + return (this.fieldNameToValueMapping); + } + + + + /******************************************************************************* + ** Setter for fieldNameToValueMapping + *******************************************************************************/ + public void setFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + } + + + + /******************************************************************************* + ** Fluent setter for fieldNameToValueMapping + *******************************************************************************/ + public BulkInsertMapping withFieldNameToValueMapping(Map> fieldNameToValueMapping) + { + this.fieldNameToValueMapping = fieldNameToValueMapping; + return (this); + } + + + + /******************************************************************************* + ** Getter for layout + *******************************************************************************/ + public Layout getLayout() + { + return (this.layout); + } + + + + /******************************************************************************* + ** Setter for layout + *******************************************************************************/ + public void setLayout(Layout layout) + { + this.layout = layout; + } + + + + /******************************************************************************* + ** Fluent setter for layout + *******************************************************************************/ + public BulkInsertMapping withLayout(Layout layout) + { + this.layout = layout; + return (this); + } + + + + /******************************************************************************* + ** Getter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public Map> getTallLayoutGroupByIndexMap() + { + return (this.tallLayoutGroupByIndexMap); + } + + + + /******************************************************************************* + ** Setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public void setTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + } + + + + /******************************************************************************* + ** Fluent setter for tallLayoutGroupByIndexMap + *******************************************************************************/ + public BulkInsertMapping withTallLayoutGroupByIndexMap(Map> tallLayoutGroupByIndexMap) + { + this.tallLayoutGroupByIndexMap = tallLayoutGroupByIndexMap; + return (this); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java new file mode 100644 index 00000000..ac98adf4 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecord.java @@ -0,0 +1,75 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class FlatRowsToRecord implements RowsToRecordInterface +{ + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java new file mode 100644 index 00000000..91481980 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/RowsToRecordInterface.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public interface RowsToRecordInterface +{ + + /*************************************************************************** + ** + ***************************************************************************/ + List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException; + + + /*************************************************************************** + ** + ***************************************************************************/ + default void setValueOrDefault(QRecord record, String fieldName, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow row, Integer index) + { + String fieldNameWithAssociationPrefix = StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + fieldName : fieldName; + + Serializable value = null; + if(index != null && row != null) + { + value = row.getValueElseNull(index); + } + else if(mapping.getFieldNameToDefaultValueMap().containsKey(fieldNameWithAssociationPrefix)) + { + value = mapping.getFieldNameToDefaultValueMap().get(fieldNameWithAssociationPrefix); + } + + if(value != null) + { + record.setValue(fieldName, value); + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java new file mode 100644 index 00000000..1dbccd92 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecord.java @@ -0,0 +1,316 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class TallRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + + String associationNameChain = ""; + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(table.getName()); + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + recordGroupByValues = null; + + ////////////////////////////////////////////////////////////////////////////////////////////////////// + // we need to push this row back onto the fileToRows object, so it'll be handled in the next record // + ////////////////////////////////////////////////////////////////////////////////////////////////////// + fileToRowsInterface.unNext(); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////// + // i wrote this condition in here: && rs.size() < limit // + // but IJ is saying it's always true... I can't quite see it, but, trusting static analysis... // + ///////////////////////////////////////////////////////////////////////////////////////////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChain, mapping, headerRow, rowsForCurrentRecord)); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QRecord makeRecordFromRows(QTableMetaData table, String associationNameChain, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + QRecord record = new QRecord(); + + Map fieldIndexes = mapping.getFieldIndexes(table, associationNameChain, headerRow); + + ////////////////////////////////////////////////////// + // get all rows for the main table from the 0th row // + ////////////////////////////////////////////////////// + BulkLoadFileRow row = rows.get(0); + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), associationNameChain, mapping, row, fieldIndexes.get(field.getName())); + } + + ///////////////////////////// + // associations (children) // + ///////////////////////////// + for(String associationName : CollectionUtils.nonNullList(mapping.getMappedAssociations())) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + List associatedRecords = processAssociation(associationNameMinusChain, associationNameChain, associatedTable, mapping, headerRow, rows); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + + return record; + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociation(String associationName, String associationNameChain, QTableMetaData associatedTable, BulkInsertMapping mapping, BulkLoadFileRow headerRow, List rows) throws QException + { + List rs = new ArrayList<>(); + + QTableMetaData table = QContext.getQInstance().getTable(associatedTable.getName()); + String associationNameChainForRecursiveCalls = "".equals(associationNameChain) ? associationName : associationNameChain + "." + associationName; + + List rowsForCurrentRecord = new ArrayList<>(); + List recordGroupByValues = null; + for(BulkLoadFileRow row : rows) + { + List groupByIndexes = mapping.getTallLayoutGroupByIndexMap().get(associationNameChainForRecursiveCalls); + if(CollectionUtils.nullSafeIsEmpty(groupByIndexes)) + { + throw (new QException("Missing group-by-index(es) for association: " + associationNameChainForRecursiveCalls)); + } + + List rowGroupByValues = getGroupByValues(row, groupByIndexes); + if(rowGroupByValues == null) + { + continue; + } + + if(rowsForCurrentRecord.isEmpty()) + { + /////////////////////////////////// + // this is first - so it's a yes // + /////////////////////////////////// + recordGroupByValues = rowGroupByValues; + rowsForCurrentRecord.add(row); + } + else if(Objects.equals(recordGroupByValues, rowGroupByValues)) + { + ///////////////////////////// + // a match - so keep going // + ///////////////////////////// + rowsForCurrentRecord.add(row); + } + else + { + ////////////////////////////////////////////////////////////// + // not first, and not a match, so we can finish this record // + ////////////////////////////////////////////////////////////// + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + + //////////////////////////////////////// + // reset these record-specific values // + //////////////////////////////////////// + rowsForCurrentRecord = new ArrayList<>(); + + ////////////////////////////////////////////////// + // use the current row to start the next record // + ////////////////////////////////////////////////// + rowsForCurrentRecord.add(row); + recordGroupByValues = rowGroupByValues; + } + } + + /////////// + // final // + /////////// + if(!rowsForCurrentRecord.isEmpty()) + { + rs.add(makeRecordFromRows(table, associationNameChainForRecursiveCalls, mapping, headerRow, rowsForCurrentRecord)); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getGroupByValues(BulkLoadFileRow row, List indexes) + { + List rowGroupByValues = new ArrayList<>(); + boolean haveAnyGroupByValues = false; + for(Integer index : indexes) + { + Serializable value = row.getValueElseNull(index); + rowGroupByValues.add(value); + + if(value != null && !"".equals(value)) + { + haveAnyGroupByValues = true; + } + } + + if(!haveAnyGroupByValues) + { + return (null); + } + + return (rowGroupByValues); + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java new file mode 100644 index 00000000..09e074d3 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapper.java @@ -0,0 +1,79 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class ValueMapper +{ + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping) + { + valueMapping(records, mapping, null); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static void valueMapping(List records, BulkInsertMapping mapping, String associationNameChain) + { + if(CollectionUtils.nullSafeIsEmpty(records)) + { + return; + } + + Map> mappingForTable = mapping.getFieldNameToValueMappingForTable(associationNameChain); + for(QRecord record : records) + { + for(Map.Entry> entry : mappingForTable.entrySet()) + { + String fieldName = entry.getKey(); + Map map = entry.getValue(); + String value = record.getValueString(fieldName); + if(value != null && map.containsKey(value)) + { + record.setValue(fieldName, map.get(value)); + } + } + + for(Map.Entry> entry : record.getAssociatedRecords().entrySet()) + { + valueMapping(entry.getValue(), mapping, StringUtils.hasContent(associationNameChain) ? associationNameChain + "." + entry.getKey() : entry.getKey()); + } + } + } + +} diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java new file mode 100644 index 00000000..40e37071 --- /dev/null +++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecord.java @@ -0,0 +1,333 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Association; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.FileToRowsInterface; +import com.kingsrook.qqq.backend.core.utils.Pair; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; +import com.kingsrook.qqq.backend.core.utils.memoization.Memoization; + + +/******************************************************************************* + ** + *******************************************************************************/ +public class WideRowsToRecord implements RowsToRecordInterface +{ + private Memoization, Boolean> shouldProcesssAssociationMemoization = new Memoization<>(); + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public List nextPage(FileToRowsInterface fileToRowsInterface, BulkLoadFileRow headerRow, BulkInsertMapping mapping, Integer limit) throws QException + { + QTableMetaData table = QContext.getQInstance().getTable(mapping.getTableName()); + if(table == null) + { + throw (new QException("Table [" + mapping.getTableName() + "] was not found in the Instance")); + } + + List rs = new ArrayList<>(); + + Map fieldIndexes = mapping.getFieldIndexes(table, null, headerRow); + + while(fileToRowsInterface.hasNext() && rs.size() < limit) + { + BulkLoadFileRow row = fileToRowsInterface.next(); + QRecord record = new QRecord(); + + for(QFieldMetaData field : table.getFields().values()) + { + setValueOrDefault(record, field.getName(), null, mapping, row, fieldIndexes.get(field.getName())); + } + + processAssociations("", headerRow, mapping, table, row, record, 0, headerRow.size()); + + rs.add(record); + } + + ValueMapper.valueMapping(rs, mapping); + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void processAssociations(String associationNameChain, BulkLoadFileRow headerRow, BulkInsertMapping mapping, QTableMetaData table, BulkLoadFileRow row, QRecord record, int startIndex, int endIndex) throws QException + { + for(String associationName : mapping.getMappedAssociations()) + { + boolean processAssociation = shouldProcessAssociation(associationNameChain, associationName); + + if(processAssociation) + { + String associationNameMinusChain = StringUtils.hasContent(associationNameChain) + ? associationName.substring(associationNameChain.length() + 1) + : associationName; + + Optional association = table.getAssociations().stream().filter(a -> a.getName().equals(associationNameMinusChain)).findFirst(); + if(association.isEmpty()) + { + throw (new QException("Couldn't find association: " + associationNameMinusChain + " under table: " + table.getName())); + } + + QTableMetaData associatedTable = QContext.getQInstance().getTable(association.get().getAssociatedTableName()); + + // List associatedRecords = processAssociation(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record); + List associatedRecords = processAssociationV2(associationName, associationNameChain, associatedTable, mapping, row, headerRow, record, startIndex, endIndex); + record.withAssociatedRecords(associationNameMinusChain, associatedRecords); + } + } + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List processAssociationV2(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, BulkLoadFileRow row, BulkLoadFileRow headerRow, QRecord record, int startIndex, int endIndex) throws QException + { + List rs = new ArrayList<>(); + + Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + { + if(entry.getKey().startsWith(associationName + ".")) + { + String fieldName = entry.getKey().substring(associationName.length() + 1); + + ////////////////////////////////////////////////////////////////////////// + // make sure the name here is for this table - not a sub-table under it // + ////////////////////////////////////////////////////////////////////////// + if(!fieldName.contains(".")) + { + fieldNameToHeaderNameMapForThisAssociation.put(fieldName, entry.getValue()); + } + } + } + + ///////////////////////////////////////////////////////////////////// + // loop over the length of the record, building associated records // + ///////////////////////////////////////////////////////////////////// + QRecord associatedRecord = new QRecord(); + Set processedFieldNames = new HashSet<>(); + boolean gotAnyValues = false; + int subStartIndex = -1; + + for(int i = startIndex; i < endIndex; i++) + { + String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + + for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + { + if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + { + /////////////////////////////////////////////// + // ok - this is a value for this association // + /////////////////////////////////////////////// + if(subStartIndex == -1) + { + subStartIndex = i; + } + + String fieldName = entry.getKey(); + if(processedFieldNames.contains(fieldName)) + { + ///////////////////////////////////////////////// + // this means we're starting a new sub-record! // + ///////////////////////////////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, i); + rs.add(associatedRecord); + } + + associatedRecord = new QRecord(); + processedFieldNames = new HashSet<>(); + gotAnyValues = false; + subStartIndex = i + 1; + } + + processedFieldNames.add(fieldName); + + Serializable value = row.getValueElseNull(i); + if(value != null && !"".equals(value)) + { + gotAnyValues = true; + } + + setValueOrDefault(associatedRecord, fieldName, associationName, mapping, row, i); + } + } + } + + //////////////////////// + // handle final value // + //////////////////////// + if(gotAnyValues) + { + addDefaultValuesToAssociatedRecord(processedFieldNames, table, associatedRecord, mapping, associationName); + processAssociations(associationName, headerRow, mapping, table, row, associatedRecord, subStartIndex, endIndex); + rs.add(associatedRecord); + } + + return (rs); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void addDefaultValuesToAssociatedRecord(Set processedFieldNames, QTableMetaData table, QRecord associatedRecord, BulkInsertMapping mapping, String associationNameChain) + { + for(QFieldMetaData field : table.getFields().values()) + { + if(!processedFieldNames.contains(field.getName())) + { + setValueOrDefault(associatedRecord, field.getName(), associationNameChain, mapping, null, null); + } + } + } + + /*************************************************************************** + ** + ***************************************************************************/ + // private List processAssociation(String associationName, String associationNameChain, QTableMetaData table, BulkInsertMapping mapping, Row row, Row headerRow, QRecord record) throws QException + // { + // List rs = new ArrayList<>(); + // String associationNameChainForRecursiveCalls = associationName; + + // Map fieldNameToHeaderNameMapForThisAssociation = new HashMap<>(); + // for(Map.Entry entry : mapping.getFieldNameToHeaderNameMap().entrySet()) + // { + // if(entry.getKey().startsWith(associationNameChainForRecursiveCalls + ".")) + // { + // fieldNameToHeaderNameMapForThisAssociation.put(entry.getKey().substring(associationNameChainForRecursiveCalls.length() + 1), entry.getValue()); + // } + // } + + // Map> indexes = new HashMap<>(); + // for(int i = 0; i < headerRow.size(); i++) + // { + // String headerValue = ValueUtils.getValueAsString(headerRow.getValue(i)); + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // if(headerValue.equals(entry.getValue()) || headerValue.matches(entry.getValue() + " ?\\d+")) + // { + // indexes.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(i); + // } + // } + // } + + // int maxIndex = indexes.values().stream().map(l -> l.size()).max(Integer::compareTo).orElse(0); + + // ////////////////////////////////////////////////////// + // // figure out how many sub-rows we'll be processing // + // ////////////////////////////////////////////////////// + // for(int i = 0; i < maxIndex; i++) + // { + // QRecord associatedRecord = new QRecord(); + // boolean gotAnyValues = false; + + // for(Map.Entry entry : fieldNameToHeaderNameMapForThisAssociation.entrySet()) + // { + // String fieldName = entry.getKey(); + // if(indexes.containsKey(fieldName) && indexes.get(fieldName).size() > i) + // { + // Integer index = indexes.get(fieldName).get(i); + // Serializable value = row.getValueElseNull(index); + // if(value != null && !"".equals(value)) + // { + // gotAnyValues = true; + // } + + // setValueOrDefault(associatedRecord, fieldName, mapping, row, index); + // } + // } + + // if(gotAnyValues) + // { + // processAssociations(associationNameChainForRecursiveCalls, headerRow, mapping, table, row, associatedRecord, 0, headerRow.size()); + // rs.add(associatedRecord); + // } + // } + + // return (rs); + // } + + + + /*************************************************************************** + ** + ***************************************************************************/ + boolean shouldProcessAssociation(String associationNameChain, String associationName) + { + return shouldProcesssAssociationMemoization.getResult(Pair.of(associationNameChain, associationName), p -> + { + List chainParts = new ArrayList<>(); + List nameParts = new ArrayList<>(); + + if(StringUtils.hasContent(associationNameChain)) + { + chainParts.addAll(Arrays.asList(associationNameChain.split("\\."))); + } + + if(StringUtils.hasContent(associationName)) + { + nameParts.addAll(Arrays.asList(associationName.split("\\."))); + } + + if(!nameParts.isEmpty()) + { + nameParts.remove(nameParts.size() - 1); + } + + return (chainParts.equals(nameParts)); + }).orElse(false); + } + +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java new file mode 100644 index 00000000..84bdce2f --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/CsvFileToRowsTest.java @@ -0,0 +1,60 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for CsvFileToRows + *******************************************************************************/ +class CsvFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] csvBytes = """ + one,two,three + 1,2,3,4 + """.getBytes(); + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.csv", new ByteArrayInputStream(csvBytes)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] { "one", "two", "three" }), headerRow); + assertEquals(new BulkLoadFileRow(new String[] { "1", "2", "3", "4" }), bodyRow); + assertFalse(fileToRowsInterface.hasNext()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java new file mode 100644 index 00000000..e1142e89 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/TestFileToRows.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.InputStream; +import java.io.Serializable; +import java.util.List; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; + + +/*************************************************************************** + ** + ***************************************************************************/ +public class TestFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface +{ + private final List rows; + + + + /******************************************************************************* + ** Constructor + ** + *******************************************************************************/ + public TestFileToRows(List rows) + { + this.rows = rows; + setIterator(this.rows.iterator()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void init(InputStream inputStream) throws QException + { + /////////// + // noop! // + /////////// + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public void close() throws Exception + { + /////////// + // noop! // + /////////// + } + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public BulkLoadFileRow makeRow(Serializable[] values) + { + return (new BulkLoadFileRow(values)); + } +} diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java new file mode 100644 index 00000000..8a381ad1 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java @@ -0,0 +1,106 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.time.LocalDate; +import java.time.Month; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportAction; +import com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest; +import com.kingsrook.qqq.backend.core.actions.reporting.excel.fastexcel.ExcelFastexcelExportStreamer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat; +import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** Unit test for XlsxFileToRows + *******************************************************************************/ +class XlsxFileToRowsTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + byte[] byteArray = writeExcelBytes(); + + FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray)); + + BulkLoadFileRow headerRow = fileToRowsInterface.next(); + BulkLoadFileRow bodyRow = fileToRowsInterface.next(); + + assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name"}), headerRow); + assertEquals(new BulkLoadFileRow(new String[] {"1", "Darin", "Jonson"}), bodyRow); + + /////////////////////////////////////////////////////////////////////////////////////// + // make sure there's at least a limit (less than 20) to how many more rows there are // + /////////////////////////////////////////////////////////////////////////////////////// + int otherRowCount = 0; + while(fileToRowsInterface.hasNext() && otherRowCount < 20) + { + fileToRowsInterface.next(); + otherRowCount++; + } + assertFalse(fileToRowsInterface.hasNext()); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private static byte[] writeExcelBytes() throws QException + { + ReportFormat format = ReportFormat.XLSX; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + QInstance qInstance = QContext.getQInstance(); + qInstance.addReport(GenerateReportActionTest.defineTableOnlyReport()); + GenerateReportActionTest.insertPersonRecords(qInstance); + + ReportInput reportInput = new ReportInput(); + reportInput.setReportName(REPORT_NAME); + reportInput.setReportDestination(new ReportDestination().withReportFormat(format).withReportOutputStream(baos)); + reportInput.setInputValues(Map.of("startDate", LocalDate.of(1970, Month.MAY, 15), "endDate", LocalDate.now())); + reportInput.setOverrideExportStreamerSupplier(ExcelFastexcelExportStreamer::new); + new GenerateReportAction().execute(reportInput); + + byte[] byteArray = baos.toByteArray(); + return byteArray; + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java new file mode 100644 index 00000000..e5474032 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/FlatRowsToRecordTest.java @@ -0,0 +1,150 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.TestFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import com.kingsrook.qqq.backend.core.utils.collections.ListBuilder; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for FlatRowsToRecord + *******************************************************************************/ +class FlatRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToHeaderNameMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { "id", "firstName", "Last Name", "Ignore", "cost" }, + new Serializable[] { 1, "Homer", "Simpson", true, "three fifty" }, + new Serializable[] { 2, "Marge", "Simpson", false, "" }, + new Serializable[] { 3, "Bart", "Simpson", "A", "99.95" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1, "one$" } + )); + + BulkLoadFileRow header = fileToRows.next(); + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "firstName", "firstName", + "lastName", "Last Name", + "cost", "cost" + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withFieldNameToValueMapping(Map.of("cost", Map.of("three fifty", new BigDecimal("3.50"), "one$", new BigDecimal("1.00")))) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(List.of(new BigDecimal("3.50")), getValues(records, "cost")); + assertEquals(4, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of("", "99.95"), getValues(records, "cost")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(ListBuilder.of(new BigDecimal("1.00")), getValues(records, "cost")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testFieldNameToIndexMapping() throws QException + { + TestFileToRows fileToRows = new TestFileToRows(List.of( + new Serializable[] { 1, "Homer", "Simpson", true }, + new Serializable[] { 2, "Marge", "Simpson", false }, + new Serializable[] { 3, "Bart", "Simpson", "A" }, + new Serializable[] { 4, "Ned", "Flanders", 3.1 } + )); + + BulkLoadFileRow header = null; + + FlatRowsToRecord rowsToRecord = new FlatRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToIndexMap(Map.of( + "firstName", 1, + "lastName", 2 + )) + .withFieldNameToDefaultValueMap(Map.of( + "noOfShoes", 2 + )) + .withTableName(TestUtils.TABLE_NAME_PERSON) + .withHasHeaderRow(false); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, 1); + assertEquals(List.of("Homer"), getValues(records, "firstName")); + assertEquals(List.of("Simpson"), getValues(records, "lastName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + assertEquals(3, records.get(0).getValues().size()); // make sure no additional values were set + + records = rowsToRecord.nextPage(fileToRows, header, mapping, 2); + assertEquals(List.of("Marge", "Bart"), getValues(records, "firstName")); + assertEquals(List.of(2, 2), getValues(records, "noOfShoes")); + + records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(List.of("Ned"), getValues(records, "firstName")); + assertEquals(List.of(2), getValues(records, "noOfShoes")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java new file mode 100644 index 00000000..dfd872f5 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/TallRowsToRecordTest.java @@ -0,0 +1,284 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for TallRowsToRecord + *******************************************************************************/ +class TallRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLines() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity + 1, Homer, Simpson, DONUT, 12 + , Homer, Simpson, BEER, 500 + , Homer, Simpson, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + , Ned, Flanders, LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(1, 2), + "orderLine", List.of(3) + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsic() throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart + 1, , , BEER, 500, Coupon Code, 10QOff + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7 + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(3, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(2, order.getAssociatedRecords().get("extrinsics").size()); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(2, order.getAssociatedRecords().get("orderLine").size()); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic() throws QException + { + Integer DEFAULT_STORE_ID = 101; + Integer DEFAULT_LINE_NO = 102; + String DEFAULT_ORDER_LINE_EXTRA_SOURCE = "file"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(""" + orderNo, Ship To, lastName, SKU, Quantity, Extrinsic Key, Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value + 1, Homer, Simpson, DONUT, 12, Store Name, QQQ Mart, Flavor, Chocolate + 1, , , DONUT, , Coupon Code, 10QOff, Size, Large + 1, , , BEER, 500, , , Flavor, Hops + 1, , , COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, , , Flavor, King James + 2, , , LAWNMOWER, 1 + """); + + BulkLoadFileRow header = fileToRows.next(); + + TallRowsToRecord rowsToRecord = new TallRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.source", DEFAULT_ORDER_LINE_EXTRA_SOURCE + )) + .withFieldNameToValueMapping(Map.of("orderLine.sku", Map.of("DONUT", "D'OH-NUT"))) + .withTallLayoutGroupByIndexMap(Map.of( + TestUtils.TABLE_NAME_ORDER, List.of(0), + "orderLine", List.of(3), + "extrinsics", List.of(5), + "orderLine.extrinsics", List.of(7) + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.TALL) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("D'OH-NUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "Large"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE, DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + assertEquals(List.of(DEFAULT_ORDER_LINE_EXTRA_SOURCE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "source")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testShouldProcessAssociation() + { + TallRowsToRecord tallRowsToRecord = new TallRowsToRecord(); + assertTrue(tallRowsToRecord.shouldProcessAssociation(null, "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("", "foo")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar")); + assertTrue(tallRowsToRecord.shouldProcessAssociation("foo.bar", "foo.bar.baz")); + + assertFalse(tallRowsToRecord.shouldProcessAssociation(null, "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("fiz.biz", "foo.bar")); + assertFalse(tallRowsToRecord.shouldProcessAssociation("foo", "foo.bar.baz")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java new file mode 100644 index 00000000..9e4108e9 --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/ValueMapperTest.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Unit test for ValueMapper + *******************************************************************************/ +class ValueMapperTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() + { + BulkInsertMapping mapping = new BulkInsertMapping().withFieldNameToValueMapping(Map.of( + "storeId", Map.of("QQQMart", 1, "Q'R'Us", 2), + "shipToName", Map.of("HoJu", "Homer", "Bart", "Bartholomew"), + "lineItem.sku", Map.of("ABC", "Alphabet"), + "lineItem.extrinsics.value", Map.of("foo", "bar", "bar", "baz"), + "extrinsics.key", Map.of("1", "one", "2", "two") + )); + + QRecord inputRecord = new QRecord() + .withValue("storeId", "QQQMart") + .withValue("shipToName", "HoJu") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "ABC") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "foo") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "bar") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", 1) + .withValue("value", "foo") + ); + JSONObject beforeJson = recordToJson(inputRecord); + + QRecord expectedRecord = new QRecord() + .withValue("storeId", 1) + .withValue("shipToName", "Homer") + .withAssociatedRecord("lineItem", new QRecord() + .withValue("sku", "Alphabet") + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "myKey") + .withValue("value", "bar") + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "yourKey") + .withValue("value", "baz") + ) + ) + .withAssociatedRecord("extrinsics", new QRecord() + .withValue("key", "one") + .withValue("value", "foo") + ); + JSONObject expectedJson = recordToJson(expectedRecord); + + ValueMapper.valueMapping(List.of(inputRecord), mapping); + JSONObject actualJson = recordToJson(inputRecord); + + System.out.println("Before"); + System.out.println(beforeJson.toString(3)); + System.out.println("Actual"); + System.out.println(actualJson.toString(3)); + System.out.println("Expected"); + System.out.println(expectedJson.toString(3)); + + assertThat(actualJson).usingRecursiveComparison().isEqualTo(expectedJson); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + public static JSONObject recordToJson(QRecord record) + { + JSONObject jsonObject = new JSONObject(); + for(Map.Entry valueEntry : CollectionUtils.nonNullMap(record.getValues()).entrySet()) + { + jsonObject.put(valueEntry.getKey(), valueEntry.getValue()); + } + for(Map.Entry> associationEntry : CollectionUtils.nonNullMap(record.getAssociatedRecords()).entrySet()) + { + JSONArray jsonArray = new JSONArray(); + for(QRecord associationRecord : CollectionUtils.nonNullList(associationEntry.getValue())) + { + jsonArray.put(recordToJson(associationRecord)); + } + jsonObject.put(associationEntry.getKey(), jsonArray); + } + return (jsonObject); + } + +} \ No newline at end of file diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java new file mode 100644 index 00000000..990ad1ef --- /dev/null +++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/mapping/WideRowsToRecordTest.java @@ -0,0 +1,305 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2024. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.mapping; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qqq.backend.core.BaseTest; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.BulkLoadFileRow; +import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.filehandling.CsvFileToRows; +import com.kingsrook.qqq.backend.core.utils.TestUtils; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for WideRowsToRecord + *******************************************************************************/ +class WideRowsToRecordTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithoutDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderAndLinesWithDupes() throws QException + { + testOrderAndLines(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1 + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderAndLines(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity" + )) + .withMappedAssociations(List.of("orderLine")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU 1, Quantity 1, SKU 2, Quantity 2, SKU 3, Quantity 3, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2 + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesAndOrderExtrinsic(""" + orderNo, Ship To, lastName, SKU, Quantity, SKU, Quantity, SKU, Quantity, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value + 1, Homer, Simpson, DONUT, 12, BEER, 500, COUCH, 1, Store Name, QQQ Mart, Coupon Code, 10QOff + 2, Ned, Flanders, BIBLE, 7, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private void testOrderLinesAndOrderExtrinsic(String csv) throws QException + { + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics")) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithoutDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key 1, Extrinsic Value 1, Extrinsic Key 2, Extrinsic Value 2, SKU 1, Quantity 1, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2, Line Extrinsic Value 2, SKU 2, Quantity 2, Line Extrinsic Key 1, Line Extrinsic Value 1, SKU 3, Quantity 3, Line Extrinsic Key 1, Line Extrinsic Value 1, Line Extrinsic Key 2 + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo, + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testOrderLinesWithLineExtrinsicsAndOrderExtrinsicWithDupes() throws QException + { + testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(""" + orderNo, Ship To, lastName, Extrinsic Key, Extrinsic Value, Extrinsic Key, Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, SKU, Quantity, Line Extrinsic Key, Line Extrinsic Value, Line Extrinsic Key + 1, Homer, Simpson, Store Name, QQQ Mart, Coupon Code, 10QOff, DONUT, 12, Flavor, Chocolate, Size, Large, BEER, 500, Flavor, Hops, COUCH, 1, Color, Brown, foo + 2, Ned, Flanders, , , , , BIBLE, 7, Flavor, King James, Size, X-Large, LAWNMOWER, 1 + """); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private void testOrderLinesWithLineExtrinsicsAndOrderExtrinsic(String csv) throws QException + { + Integer DEFAULT_STORE_ID = 42; + Integer DEFAULT_LINE_NO = 47; + String DEFAULT_LINE_EXTRA_VALUE = "bar"; + + CsvFileToRows fileToRows = CsvFileToRows.forString(csv); + BulkLoadFileRow header = fileToRows.next(); + + WideRowsToRecord rowsToRecord = new WideRowsToRecord(); + + BulkInsertMapping mapping = new BulkInsertMapping() + .withFieldNameToHeaderNameMap(Map.of( + "orderNo", "orderNo", + "shipToName", "Ship To", + "orderLine.sku", "SKU", + "orderLine.quantity", "Quantity", + "extrinsics.key", "Extrinsic Key", + "extrinsics.value", "Extrinsic Value", + "orderLine.extrinsics.key", "Line Extrinsic Key", + "orderLine.extrinsics.value", "Line Extrinsic Value" + )) + .withMappedAssociations(List.of("orderLine", "extrinsics", "orderLine.extrinsics")) + .withFieldNameToValueMapping(Map.of("orderLine.extrinsics.value", Map.of("Large", "L", "X-Large", "XL"))) + .withFieldNameToDefaultValueMap(Map.of( + "storeId", DEFAULT_STORE_ID, + "orderLine.lineNumber", DEFAULT_LINE_NO, + "orderLine.extrinsics.value", DEFAULT_LINE_EXTRA_VALUE + )) + .withTableName(TestUtils.TABLE_NAME_ORDER) + .withLayout(BulkInsertMapping.Layout.WIDE) + .withHasHeaderRow(true); + + List records = rowsToRecord.nextPage(fileToRows, header, mapping, Integer.MAX_VALUE); + assertEquals(2, records.size()); + + QRecord order = records.get(0); + assertEquals(1, order.getValueInteger("orderNo")); + assertEquals("Homer", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("DONUT", "BEER", "COUCH"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("12", "500", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertEquals(List.of("Store Name", "Coupon Code"), getValues(order.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("QQQ Mart", "10QOff"), getValues(order.getAssociatedRecords().get("extrinsics"), "value")); + + QRecord lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Chocolate", "L"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(1); + assertEquals(List.of("Flavor"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Hops"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + lineItem = order.getAssociatedRecords().get("orderLine").get(2); + assertEquals(List.of("Color", "foo"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("Brown", DEFAULT_LINE_EXTRA_VALUE), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + + order = records.get(1); + assertEquals(2, order.getValueInteger("orderNo")); + assertEquals("Ned", order.getValueString("shipToName")); + assertEquals(DEFAULT_STORE_ID, order.getValue("storeId")); + assertEquals(List.of("BIBLE", "LAWNMOWER"), getValues(order.getAssociatedRecords().get("orderLine"), "sku")); + assertEquals(List.of("7", "1"), getValues(order.getAssociatedRecords().get("orderLine"), "quantity")); + assertEquals(List.of(DEFAULT_LINE_NO, DEFAULT_LINE_NO), getValues(order.getAssociatedRecords().get("orderLine"), "lineNumber")); + assertThat(order.getAssociatedRecords().get("extrinsics")).isNullOrEmpty(); + + lineItem = order.getAssociatedRecords().get("orderLine").get(0); + assertEquals(List.of("Flavor", "Size"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "key")); + assertEquals(List.of("King James", "XL"), getValues(lineItem.getAssociatedRecords().get("extrinsics"), "value")); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private List getValues(List records, String fieldName) + { + return (records.stream().map(r -> r.getValue(fieldName)).toList()); + } + +} \ No newline at end of file