diff --git a/.circleci/config.yml b/.circleci/config.yml index 40a1a443..573814e9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ jobs: - localstack/startup - install_java17 - run_maven: - maven_subcommand: test + maven_subcommand: verify - slack/notify: event: fail diff --git a/pom.xml b/pom.xml index 1d51bc31..d2714df8 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,8 @@ 17 true true + true + 0.80 @@ -115,6 +117,9 @@ org.apache.maven.plugins maven-surefire-plugin 3.0.0-M5 + + @{jaCoCoArgLine} + org.apache.maven.plugins @@ -164,7 +169,84 @@ 1 - + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + pre-unit-test + + prepare-agent + + + jaCoCoArgLine + + + + unit-test-check + + check + + + + ${coverage.haltOnFailure} + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + ${coverage.instructionCoveredRatioMinimum} + + + + + + + + post-unit-test + verify + + report + + + + + + exec-maven-plugin + org.codehaus.mojo + 3.0.0 + + + test-coverage-summary + verify + + exec + + + sh + + -c + + /tmp/$$.headers +xpath -q -e '/html/body/table/tfoot/tr[1]/td/text()' target/site/jacoco/index.html > /tmp/$$.values +echo +echo "Jacoco coverage summary report:" +echo " See also target/site/jacoco/index.html" +echo " and https://www.jacoco.org/jacoco/trunk/doc/counters.html" +echo "------------------------------------------------------------" +paste /tmp/$$.headers /tmp/$$.values | tail +2 | awk -v FS='\t' '{printf("%-20s %s\n",$1,$2)}' +rm /tmp/$$.headers /tmp/$$.values + ]]> + + + + + + diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java index 3051e18a..23c38c46 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/FilesystemBackendModuleInterface.java @@ -31,6 +31,8 @@ import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFile *******************************************************************************/ public interface FilesystemBackendModuleInterface { + String CUSTOMIZER_FILE_POST_FILE_READ = "postFileRead"; + /******************************************************************************* ** For filesystem backends, get the module-specific action base-class, that helps ** with functions like listing and deleting files. diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java index 9a04f289..8b211f7f 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java @@ -41,6 +41,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.QTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemBackendModuleInterface; import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemBackendMetaData; import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails; @@ -269,7 +270,7 @@ public abstract class AbstractBaseFilesystemAction *******************************************************************************/ private String customizeFileContentsAfterReading(QTableMetaData table, String fileContents) throws QException { - Optional optionalCustomizer = table.getCustomizer("postFileRead"); + Optional optionalCustomizer = table.getCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ); if(optionalCustomizer.isEmpty()) { return (fileContents); diff --git a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java index 21958ab7..b7fc6ca7 100644 --- a/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java +++ b/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3Action.java @@ -27,6 +27,7 @@ import java.io.InputStream; import java.util.List; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; @@ -67,6 +68,17 @@ public class AbstractS3Action extends AbstractBaseFilesystemAction record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH)), @@ -57,12 +65,36 @@ public class FilesystemQueryActionTest extends FilesystemActionTest /******************************************************************************* ** *******************************************************************************/ - private QueryRequest initQueryRequest() throws QInstanceValidationException + @Test + public void testQueryWithFileCustomizer() throws QException { QueryRequest queryRequest = new QueryRequest(); - queryRequest.setInstance(TestUtils.defineInstance()); + QInstance instance = TestUtils.defineInstance(); + + QTableMetaData table = instance.getTable(TestUtils.TABLE_NAME_PERSON_LOCAL_FS); + table.withCustomizer(FilesystemBackendModuleInterface.CUSTOMIZER_FILE_POST_FILE_READ, new QCodeReference() + .withName(ValueUpshifter.class.getName()) + .withCodeType(QCodeType.JAVA) + .withCodeUsage(QCodeUsage.CUSTOMIZER)); + + queryRequest.setInstance(instance); queryRequest.setTableName(TestUtils.defineLocalFilesystemJSONPersonTable().getName()); - return queryRequest; + QueryResult queryResult = new FilesystemQueryAction().execute(queryRequest); + Assertions.assertEquals(3, queryResult.getRecords().size(), "Unfiltered query should find all rows"); + Assertions.assertTrue( + queryResult.getRecords().stream().allMatch(record -> record.getValueString("email").matches(".*KINGSROOK.COM")), + "All records should have their email addresses up-shifted."); + } + + + + public static class ValueUpshifter implements Function + { + @Override + public String apply(String s) + { + return (s.replaceAll("kingsrook.com", "KINGSROOK.COM")); + } } } \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java new file mode 100644 index 00000000..eeff6da6 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCollectSourceFileNamesFunctionTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.etl.basic; + + +import java.util.Arrays; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.RunBackendStepAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepRequest; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepResult; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.module.filesystem.TestUtils; +import com.kingsrook.qqq.backend.module.filesystem.base.FilesystemRecordBackendDetailFields; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Unit test for BasicETLCollectSourceFileNamesFunction + *******************************************************************************/ +class BasicETLCollectSourceFileNamesFunctionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testOneFile() throws Exception + { + String file = "/tmp/test1.csv"; + String result = runTest(file); + assertEquals(file, result); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testTwoFiles() throws Exception + { + String file1 = "/tmp/test1.csv"; + String file2 = "/tmp/test2.csv"; + String result = runTest(file1, file2); + + ////////////////////////////////////////////////////////////////////// + // the names go into a set, so they can come out in either order... // + ////////////////////////////////////////////////////////////////////// + assertTrue(result.equals(file1 + "," + file2) || result.equals((file2 + "," + file1))); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + public void testDuplicatedFile() throws Exception + { + String file = "/tmp/test1.csv"; + String result = runTest(file, file); + assertEquals(file, result); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private String runTest(String... fileNames) throws Exception + { + QInstance qInstance = TestUtils.defineInstance(); + QBackendStepMetaData backendStepMetaData = new BasicETLCollectSourceFileNamesFunction().defineStepMetaData(); + QProcessMetaData qProcessMetaData = new QProcessMetaData().withName("testScaffold").addStep(backendStepMetaData); + qInstance.addProcess(qProcessMetaData); + + List records = Arrays.stream(fileNames).map(fileName -> + new QRecord().withBackendDetail(FilesystemRecordBackendDetailFields.FULL_PATH, fileName)).toList(); + + RunBackendStepRequest runBackendStepRequest = new RunBackendStepRequest(qInstance); + runBackendStepRequest.setSession(TestUtils.getMockSession()); + runBackendStepRequest.setStepName(backendStepMetaData.getName()); + runBackendStepRequest.setProcessName(qProcessMetaData.getName()); + runBackendStepRequest.setRecords(records); + + RunBackendStepAction runBackendStepAction = new RunBackendStepAction(); + RunBackendStepResult result = runBackendStepAction.execute(runBackendStepRequest); + + return ((String) result.getValues().get(BasicETLCollectSourceFileNamesFunction.FIELD_SOURCE_FILE_PATHS)); + } +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java new file mode 100644 index 00000000..5ffe69b6 --- /dev/null +++ b/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/AbstractS3ActionTest.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2022. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qqq.backend.module.filesystem.s3.actions; + + +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + + +/******************************************************************************* + ** Unit test for AbstractS3Action + *******************************************************************************/ +class AbstractS3ActionTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildAmazonS3ClientFromBackendMetaData() + { + String regionName = Regions.AP_SOUTHEAST_3.getName(); + S3BackendMetaData s3BackendMetaData = new S3BackendMetaData() + .withAccessKey("Not a real access key") + .withSecretKey("Also not a real key") + .withRegion(regionName); + AmazonS3 amazonS3 = new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(s3BackendMetaData); + assertEquals(regionName, amazonS3.getRegionName()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testBuildAmazonS3ClientFromBackendMetaDataWrongType() + { + assertThrows(IllegalArgumentException.class, () -> + { + new AbstractS3Action().buildAmazonS3ClientFromBackendMetaData(new QBackendMetaData()); + }); + } +} \ No newline at end of file