QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

QQQ-16 Deleting "direct" implementation of ETL

Browse Source
This commit is contained in:
darin.kelkhoff
2022-06-23 14:29:12 -05:00
parent 62f875b254
commit 2cd100aee0
6 changed files with 0 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/main/java/com/kingsrook/qqq/backend/core/actions/etl/RunETL.java
View File

@ -1,75 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.etl;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest;
import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.etl.QDataBatch;
import com.kingsrook.qqq.backend.core.model.etl.QDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/*******************************************************************************
**
*******************************************************************************/
public class RunETL
{
private static final Logger LOG = LogManager.getLogger(RunETL.class);
/*******************************************************************************
**
*******************************************************************************/
public void run(QInstance instance, QSession session, QDataSource source, QTableMetaData destination) throws QException
{
List<String> batchIdentifiers = source.listAvailableBatches();
if(CollectionUtils.nullSafeHasContents(batchIdentifiers))
{
for(String identifier : batchIdentifiers)
{
QDataBatch batch = source.getBatch(identifier, destination);
InsertRequest insertRequest = new InsertRequest(instance);
insertRequest.setTableName(destination.getName());
insertRequest.setSession(session);
insertRequest.setRecords(batch.getRecords());
InsertAction insertAction = new InsertAction();
InsertResult insertResult = insertAction.execute(insertRequest);
System.out.println("** Inserted [" + insertResult.getRecords().size() + "] records into table [" + destination.getName() + "].");
for(QRecord record : insertRequest.getRecords())
{
System.out.println(" Inserted [" + record.getValueString("firstName") + "][" + record.getValueString("lastName") + "].");
}
source.discardBatch(batch);
}
}
}
}

80
src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithStatus.java
View File

@ -1,80 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.data;
import java.util.List;
/*******************************************************************************
** Wrapper on a QRecord, to add status information after an action took place.
** e.g., any errors that occurred.
**
** TODO - expand?
**
*******************************************************************************/
public class QRecordWithStatus extends QRecord
{
private List<Exception> errors;
/*******************************************************************************
**
*******************************************************************************/
public QRecordWithStatus()
{
}
/*******************************************************************************
**
*******************************************************************************/
public QRecordWithStatus(QRecord record)
{
super.setTableName(record.getTableName());
super.setValues(record.getValues());
}
/*******************************************************************************
** Getter for errors
**
*******************************************************************************/
public List<Exception> getErrors()
{
return errors;
}
/*******************************************************************************
** Setter for errors
**
*******************************************************************************/
public void setErrors(List<Exception> errors)
{
this.errors = errors;
}
}

105
src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataBatch.java
View File

@ -1,105 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.etl;
import java.util.List;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
**
*******************************************************************************/
public class QDataBatch
{
private String identity; // e.g., a full path to a file
private List<QRecord> records;
/*******************************************************************************
** Getter for identity
**
*******************************************************************************/
public String getIdentity()
{
return identity;
}
/*******************************************************************************
** Setter for identity
**
*******************************************************************************/
public void setIdentity(String identity)
{
this.identity = identity;
}
/*******************************************************************************
** Fluent setter for identity
**
*******************************************************************************/
public QDataBatch withIdentity(String identity)
{
this.identity = identity;
return (this);
}
/*******************************************************************************
** Getter for records
**
*******************************************************************************/
public List<QRecord> getRecords()
{
return records;
}
/*******************************************************************************
** Setter for records
**
*******************************************************************************/
public void setRecords(List<QRecord> records)
{
this.records = records;
}
/*******************************************************************************
** Setter for records
**
*******************************************************************************/
public QDataBatch withRecords(List<QRecord> records)
{
this.records = records;
return (this);
}
}

53
src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataSource.java
View File

@ -1,53 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.etl;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
/*******************************************************************************
**
*******************************************************************************/
public interface QDataSource
{
/*******************************************************************************
** listAvailableBatches
**
*******************************************************************************/
List<String> listAvailableBatches();
/*******************************************************************************
** getBatch
**
*******************************************************************************/
QDataBatch getBatch(String identity, QTableMetaData destination) throws QException;
/*******************************************************************************
** discardBatch
**
*******************************************************************************/
void discardBatch(QDataBatch batch);
}

182
src/main/java/com/kingsrook/qqq/backend/core/model/etl/QFileSystemDataSource.java
View File

@ -1,182 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.model.etl;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.oro.io.GlobFilenameFilter;
/*******************************************************************************
**
*******************************************************************************/
public class QFileSystemDataSource implements QDataSource
{
private static final Logger LOG = LogManager.getLogger(QFileSystemDataSource.class);
private String path;
private String glob;
/*******************************************************************************
** Getter for path
**
*******************************************************************************/
public String getPath()
{
return path;
}
/*******************************************************************************
** Setter for path
**
*******************************************************************************/
public void setPath(String path)
{
this.path = path;
}
/*******************************************************************************
** Fluent setter for path
**
*******************************************************************************/
public QFileSystemDataSource withPath(String path)
{
this.path = path;
return this;
}
/*******************************************************************************
** Getter for glob
**
*******************************************************************************/
public String getGlob()
{
return glob;
}
/*******************************************************************************
** Setter for glob
**
*******************************************************************************/
public void setGlob(String glob)
{
this.glob = glob;
}
/*******************************************************************************
** Fluent setter for glob
**
*******************************************************************************/
public QFileSystemDataSource withGlob(String glob)
{
this.glob = glob;
return this;
}
@Override
public List<String> listAvailableBatches()
{
List<String> rs = new ArrayList<>();
File directory = new File(path);
System.out.println("Listing available batches at [" + path + "].");
for(String entry : Objects.requireNonNull(directory.list(new GlobFilenameFilter(glob))))
{
String entryPath = directory + File.separator + entry;
if(new File(entryPath).isFile())
{
rs.add(entryPath);
}
}
System.out.println("Found [" + rs.size() + "] batches.");
return (rs);
}
@Override
public QDataBatch getBatch(String identity, QTableMetaData table) throws QException
{
File file = new File(identity);
if(!file.exists())
{
throw new QException("File [" + identity + "] was not found.");
}
if(!file.isFile())
{
throw new QException("File [" + identity + "] is not a regular file.");
}
try
{
System.out.println("Reading batch file [" + identity + "].");
String contents = FileUtils.readFileToString(file);
List<QRecord> qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(contents, table, null);// todo!!
System.out.println("Read [" + qRecords.size() + "] records from batch file.");
return (new QDataBatch().withIdentity(identity).withRecords(qRecords));
}
catch(IOException e)
{
throw new QException("Error reading file", e);
}
}
@Override
public void discardBatch(QDataBatch batch)
{
File file = new File(batch.getIdentity());
File trashFile = new File("/tmp/" + UUID.randomUUID());
if(file.renameTo(trashFile))
{
System.out.println("Discard batch file [" + batch.getIdentity() + "] to trash file [" + trashFile + "].");
}
}
}

50
src/test/java/com/kingsrook/qqq/backend/core/actions/etl/RunETLTest.java
View File

@ -1,50 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2022. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package com.kingsrook.qqq.backend.core.actions.etl;
import com.kingsrook.qqq.backend.core.model.etl.QDataSource;
import com.kingsrook.qqq.backend.core.model.etl.QFileSystemDataSource;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.utils.TestUtils;
import org.junit.jupiter.api.Test;
/*******************************************************************************
**
*******************************************************************************/
class RunETLTest
{
//TODO disucss with DK
// @Test
// public void testRun() throws Exception
// {
// RunETL runETL = new RunETL();
//
// QDataSource dataSource = new QFileSystemDataSource()
// .withPath("/tmp/etl-source")
// .withGlob("*.csv");
//
// QInstance qInstance = TestUtils.defineInstance();
// runETL.run(qInstance, TestUtils.getMockSession(), dataSource, qInstance.getTable("person"));
// }
}