diff --git a/src/main/java/com/kingsrook/qqq/backend/core/actions/etl/RunETL.java b/src/main/java/com/kingsrook/qqq/backend/core/actions/etl/RunETL.java deleted file mode 100644 index 9b015d25..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/actions/etl/RunETL.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.actions.etl; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.actions.InsertAction; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.actions.insert.InsertRequest; -import com.kingsrook.qqq.backend.core.model.actions.insert.InsertResult; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.etl.QDataBatch; -import com.kingsrook.qqq.backend.core.model.etl.QDataSource; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import com.kingsrook.qqq.backend.core.model.session.QSession; -import com.kingsrook.qqq.backend.core.utils.CollectionUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class RunETL -{ - private static final Logger LOG = LogManager.getLogger(RunETL.class); - - /******************************************************************************* - ** - *******************************************************************************/ - public void run(QInstance instance, QSession session, QDataSource source, QTableMetaData destination) throws QException - { - List batchIdentifiers = source.listAvailableBatches(); - if(CollectionUtils.nullSafeHasContents(batchIdentifiers)) - { - for(String identifier : batchIdentifiers) - { - QDataBatch batch = source.getBatch(identifier, destination); - InsertRequest insertRequest = new InsertRequest(instance); - insertRequest.setTableName(destination.getName()); - insertRequest.setSession(session); - insertRequest.setRecords(batch.getRecords()); - - InsertAction insertAction = new InsertAction(); - InsertResult insertResult = insertAction.execute(insertRequest); - System.out.println("** Inserted [" + insertResult.getRecords().size() + "] records into table [" + destination.getName() + "]."); - for(QRecord record : insertRequest.getRecords()) - { - System.out.println(" Inserted [" + record.getValueString("firstName") + "][" + record.getValueString("lastName") + "]."); - } - source.discardBatch(batch); - } - } - } -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithStatus.java b/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithStatus.java deleted file mode 100644 index 8ac50db9..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecordWithStatus.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.model.data; - - -import java.util.List; - - -/******************************************************************************* - ** Wrapper on a QRecord, to add status information after an action took place. - ** e.g., any errors that occurred. - ** - ** TODO - expand? - ** - *******************************************************************************/ -public class QRecordWithStatus extends QRecord -{ - private List errors; - - - - /******************************************************************************* - ** - *******************************************************************************/ - public QRecordWithStatus() - { - } - - - - /******************************************************************************* - ** - *******************************************************************************/ - public QRecordWithStatus(QRecord record) - { - super.setTableName(record.getTableName()); - super.setValues(record.getValues()); - } - - - - /******************************************************************************* - ** Getter for errors - ** - *******************************************************************************/ - public List getErrors() - { - return errors; - } - - - - /******************************************************************************* - ** Setter for errors - ** - *******************************************************************************/ - public void setErrors(List errors) - { - this.errors = errors; - } -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataBatch.java b/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataBatch.java deleted file mode 100644 index dcf7ad57..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataBatch.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.model.etl; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.model.data.QRecord; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class QDataBatch -{ - private String identity; // e.g., a full path to a file - private List records; - - - - /******************************************************************************* - ** Getter for identity - ** - *******************************************************************************/ - public String getIdentity() - { - return identity; - } - - - - /******************************************************************************* - ** Setter for identity - ** - *******************************************************************************/ - public void setIdentity(String identity) - { - this.identity = identity; - } - - - - /******************************************************************************* - ** Fluent setter for identity - ** - *******************************************************************************/ - public QDataBatch withIdentity(String identity) - { - this.identity = identity; - return (this); - } - - - - /******************************************************************************* - ** Getter for records - ** - *******************************************************************************/ - public List getRecords() - { - return records; - } - - - - /******************************************************************************* - ** Setter for records - ** - *******************************************************************************/ - public void setRecords(List records) - { - this.records = records; - } - - - - /******************************************************************************* - ** Setter for records - ** - *******************************************************************************/ - public QDataBatch withRecords(List records) - { - this.records = records; - return (this); - } - -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataSource.java b/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataSource.java deleted file mode 100644 index bc6e1ccf..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QDataSource.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.model.etl; - - -import java.util.List; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; - - -/******************************************************************************* - ** - *******************************************************************************/ -public interface QDataSource - -{ - /******************************************************************************* - ** listAvailableBatches - ** - *******************************************************************************/ - List listAvailableBatches(); - - /******************************************************************************* - ** getBatch - ** - *******************************************************************************/ - QDataBatch getBatch(String identity, QTableMetaData destination) throws QException; - - /******************************************************************************* - ** discardBatch - ** - *******************************************************************************/ - void discardBatch(QDataBatch batch); -} diff --git a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QFileSystemDataSource.java b/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QFileSystemDataSource.java deleted file mode 100644 index 9ceac9f2..00000000 --- a/src/main/java/com/kingsrook/qqq/backend/core/model/etl/QFileSystemDataSource.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.model.etl; - - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.UUID; -import com.kingsrook.qqq.backend.core.adapters.CsvToQRecordAdapter; -import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.data.QRecord; -import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; -import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.oro.io.GlobFilenameFilter; - - -/******************************************************************************* - ** - *******************************************************************************/ -public class QFileSystemDataSource implements QDataSource -{ - private static final Logger LOG = LogManager.getLogger(QFileSystemDataSource.class); - - private String path; - private String glob; - - - - /******************************************************************************* - ** Getter for path - ** - *******************************************************************************/ - public String getPath() - { - return path; - } - - - - /******************************************************************************* - ** Setter for path - ** - *******************************************************************************/ - public void setPath(String path) - { - this.path = path; - } - - - - /******************************************************************************* - ** Fluent setter for path - ** - *******************************************************************************/ - public QFileSystemDataSource withPath(String path) - { - this.path = path; - return this; - } - - - - /******************************************************************************* - ** Getter for glob - ** - *******************************************************************************/ - public String getGlob() - { - return glob; - } - - - - /******************************************************************************* - ** Setter for glob - ** - *******************************************************************************/ - public void setGlob(String glob) - { - this.glob = glob; - } - - - - /******************************************************************************* - ** Fluent setter for glob - ** - *******************************************************************************/ - public QFileSystemDataSource withGlob(String glob) - { - this.glob = glob; - return this; - } - - - - @Override - public List listAvailableBatches() - { - List rs = new ArrayList<>(); - File directory = new File(path); - System.out.println("Listing available batches at [" + path + "]."); - for(String entry : Objects.requireNonNull(directory.list(new GlobFilenameFilter(glob)))) - { - String entryPath = directory + File.separator + entry; - if(new File(entryPath).isFile()) - { - rs.add(entryPath); - } - } - System.out.println("Found [" + rs.size() + "] batches."); - - return (rs); - } - - - - @Override - public QDataBatch getBatch(String identity, QTableMetaData table) throws QException - { - File file = new File(identity); - if(!file.exists()) - { - throw new QException("File [" + identity + "] was not found."); - } - if(!file.isFile()) - { - throw new QException("File [" + identity + "] is not a regular file."); - } - - try - { - System.out.println("Reading batch file [" + identity + "]."); - String contents = FileUtils.readFileToString(file); - List qRecords = new CsvToQRecordAdapter().buildRecordsFromCsv(contents, table, null);// todo!! - System.out.println("Read [" + qRecords.size() + "] records from batch file."); - - return (new QDataBatch().withIdentity(identity).withRecords(qRecords)); - } - catch(IOException e) - { - throw new QException("Error reading file", e); - } - } - - - - @Override - public void discardBatch(QDataBatch batch) - { - File file = new File(batch.getIdentity()); - File trashFile = new File("/tmp/" + UUID.randomUUID()); - if(file.renameTo(trashFile)) - { - System.out.println("Discard batch file [" + batch.getIdentity() + "] to trash file [" + trashFile + "]."); - } - } -} diff --git a/src/test/java/com/kingsrook/qqq/backend/core/actions/etl/RunETLTest.java b/src/test/java/com/kingsrook/qqq/backend/core/actions/etl/RunETLTest.java deleted file mode 100644 index 9feb2bc3..00000000 --- a/src/test/java/com/kingsrook/qqq/backend/core/actions/etl/RunETLTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * QQQ - Low-code Application Framework for Engineers. - * Copyright (C) 2021-2022. Kingsrook, LLC - * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States - * contact@kingsrook.com - * https://github.com/Kingsrook/ - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package com.kingsrook.qqq.backend.core.actions.etl; - - -import com.kingsrook.qqq.backend.core.model.etl.QDataSource; -import com.kingsrook.qqq.backend.core.model.etl.QFileSystemDataSource; -import com.kingsrook.qqq.backend.core.model.metadata.QInstance; -import com.kingsrook.qqq.backend.core.utils.TestUtils; -import org.junit.jupiter.api.Test; - - -/******************************************************************************* - ** - *******************************************************************************/ -class RunETLTest -{ - //TODO disucss with DK - // @Test - // public void testRun() throws Exception - // { - // RunETL runETL = new RunETL(); - // - // QDataSource dataSource = new QFileSystemDataSource() - // .withPath("/tmp/etl-source") - // .withGlob("*.csv"); - // - // QInstance qInstance = TestUtils.defineInstance(); - // runETL.run(qInstance, TestUtils.getMockSession(), dataSource, qInstance.getTable("person")); - // } -} \ No newline at end of file