QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-21 - Adding ETL Update Function for use in ONE-18

Browse Source
This commit is contained in:
t-samples
2022-07-12 10:59:57 -05:00
parent 86adccddd4
commit f7f7a92842
1 changed files with 100 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/basic/BasicETLLoadAsUpdateFunction.java Normal file
View File

@ -0,0 +1,100 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.processes.implementations.etl.basic;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.interfaces.BackendStep;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateRequest;
import com.kingsrook.qqq.backend.core.model.actions.update.UpdateResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
** Function body for performing the Load step of a basic ETL process using update.
*******************************************************************************/
public class BasicETLLoadAsUpdateFunction implements BackendStep
{
private static final Logger LOG = LogManager.getLogger(BasicETLLoadAsUpdateFunction.class);
/*******************************************************************************
**
*******************************************************************************/
@Override
public void run(RunBackendStepRequest runBackendStepRequest, RunBackendStepResult runBackendStepResult) throws QException
{
//////////////////////////////////////////////////////
// exit early with no-op if no records made it here //
//////////////////////////////////////////////////////
List<QRecord> inputRecords = runBackendStepRequest.getRecords();
LOG.info("Received [" + inputRecords.size() + "] records to load using update");
if(CollectionUtils.nullSafeIsEmpty(inputRecords))
{
runBackendStepResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, 0);
return;
}
/////////////////////////////////////////////////////////////////
// put the destination table name in all records being updated //
/////////////////////////////////////////////////////////////////
String table = runBackendStepRequest.getValueString(BasicETLProcess.FIELD_DESTINATION_TABLE);
for(QRecord record : inputRecords)
{
record.setTableName(table);
}
//////////////////////////////////////////
// run an update request on the records //
//////////////////////////////////////////
int recordsUpdated = 0;
List<QRecord> outputRecords = new ArrayList<>();
int pageSize = 1000; // todo - make this a field?
for(List<QRecord> page : CollectionUtils.getPages(inputRecords, pageSize))
{
LOG.info("Updating a page of [" + page.size() + "] records. Progress: " + recordsUpdated + " loaded out of " + inputRecords.size() + " total");
UpdateRequest updateRequest = new UpdateRequest(runBackendStepRequest.getInstance());
updateRequest.setSession(runBackendStepRequest.getSession());
updateRequest.setTableName(table);
updateRequest.setRecords(page);
UpdateAction updateAction = new UpdateAction();
UpdateResult updateResult = updateAction.execute(updateRequest);
outputRecords.addAll(updateResult.getRecords());
recordsUpdated += updateResult.getRecords().size();
}
runBackendStepResult.setRecords(outputRecords);
runBackendStepResult.addValue(BasicETLProcess.FIELD_RECORD_COUNT, recordsUpdated);
}
}