QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 05:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Eliminated all warnings.

Browse Source
This commit is contained in:
darin.kelkhoff
2024-07-19 12:38:02 -05:00
parent f9af2ba983
commit 912e40fe0b
87 changed files with 257 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/base/actions/AbstractBaseFilesystemAction.java
View File

@ -25,6 +25,7 @@ package com.kingsrook.qqq.backend.module.filesystem.base.actions;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@ -222,7 +223,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
{
case CSV:
{
String fileContents = IOUtils.toString(inputStream);
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
if(queryInput.getRecordPipe() != null)
@ -245,7 +246,7 @@ public abstract class AbstractBaseFilesystemAction<FILE>
}
case JSON:
{
String fileContents = IOUtils.toString(inputStream);
String fileContents = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
fileContents = customizeFileContentsAfterReading(table, fileContents);
// todo - pipe support!!

4
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStep.java
View File

@ -129,7 +129,7 @@ public class BasicETLCleanupSourceFilesStep implements BackendStep
.withName(this.getClass().getName())
.withCodeType(QCodeType.JAVA))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.addField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
.withField(new QFieldMetaData("moveOrDelete", QFieldType.STRING))
.withField(new QFieldMetaData("destinationForMoves", QFieldType.STRING))));
}
}

8
qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcess.java
View File

@ -69,10 +69,10 @@ public class FilesystemSyncProcess
.withName(FilesystemSyncStep.class.getName())
.withCodeType(QCodeType.JAVA))
.withInputData(new QFunctionInputMetaData()
.addField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
.addField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE))
.addField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
.withField(new QFieldMetaData(FIELD_SOURCE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_ARCHIVE_TABLE, QFieldType.STRING))
.withField(new QFieldMetaData(FIELD_MAX_FILES_TO_ARCHIVE, QFieldType.INTEGER).withDefaultValue(Integer.MAX_VALUE))
.withField(new QFieldMetaData(FIELD_PROCESSING_TABLE, QFieldType.STRING)));
return new QProcessMetaData()
.withName(PROCESS_NAME)

15
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemActionTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@ -115,14 +116,14 @@ public class FilesystemActionTest extends BaseTest
{"id":2,"createDate":"2022-06-17 14:52:59","modifyDate":"2022-06-17 14:52:59","firstName":"Jane","lastName":"Smith","birthDate":"1982-02-02","email":"jane@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1);
FileUtils.writeStringToFile(new File(fullPath + "DATA-1.json"), jsonData1, StandardCharsets.UTF_8);
String jsonData2 = """
[
{"id":3,"createDate":"2021-11-27 15:40:38","modifyDate":"2021-11-27 15:40:38","firstName":"Homer","lastName":"S","birthDate":"1983-03-03","email":"homer.s@kingsrook.com"}
]
""";
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2);
FileUtils.writeStringToFile(new File(fullPath + "DATA-2.json"), jsonData2, StandardCharsets.UTF_8);
}
@ -147,14 +148,14 @@ public class FilesystemActionTest extends BaseTest
"1","2021-10-26 14:39:37","2021-10-26 14:39:37","John","Doe","1981-01-01","john@kingsrook.com"
"2","2022-06-17 14:52:59","2022-06-17 14:52:59","Jane","Smith","1982-02-02","jane@kingsrook.com"
""";
FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1);
FileUtils.writeStringToFile(new File(fullPath + "FILE-1.csv"), csvData1, StandardCharsets.UTF_8);
String csvData2 = """
"id","createDate","modifyDate","firstName","lastName","birthDate","email"
"3","2021-11-27 15:40:38","2021-11-27 15:40:38","Homer","S","1983-03-03","homer.s@kingsrook.com"
"4","2022-07-18 15:53:00","2022-07-18 15:53:00","Marge","S","1984-04-04","marge.s@kingsrook.com"
"5","2022-11-11 12:00:00","2022-11-12 13:00:00","Bart","S","1985-05-05","bart.s@kingsrook.com\""""; // intentionally no \n at EOL here
FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2);
FileUtils.writeStringToFile(new File(fullPath + "FILE-2.csv"), csvData2, StandardCharsets.UTF_8);
}
@ -177,15 +178,15 @@ public class FilesystemActionTest extends BaseTest
String data1 = """
Hello, Blob
""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1);
FileUtils.writeStringToFile(new File(fullPath + "BLOB-1.txt"), data1, StandardCharsets.UTF_8);
String data2 = """
Hi Bob""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2);
FileUtils.writeStringToFile(new File(fullPath + "BLOB-2.txt"), data2, StandardCharsets.UTF_8);
String data3 = """
# Hi MD...""";
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3);
FileUtils.writeStringToFile(new File(fullPath + "BLOB-3.md"), data3, StandardCharsets.UTF_8);
}

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/local/actions/FilesystemInsertActionTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.local.actions;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -64,7 +65,7 @@ public class FilesystemInsertActionTest extends FilesystemActionTest
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains(TestUtils.BASE_PATH))
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH))));
assertEquals("Hello, World", FileUtils.readFileToString(new File(insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH)), StandardCharsets.UTF_8));
}

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/etl/basic/BasicETLCleanupSourceFilesStepTest.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.et
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@ -189,7 +190,7 @@ public class BasicETLCleanupSourceFilesStepTest extends BaseTest
for(String filePath : filePathsSet)
{
File file = new File(filePath);
FileUtils.writeStringToFile(file, "content");
FileUtils.writeStringToFile(file, "content", StandardCharsets.UTF_8);
}
// List<QRecord> records = filePaths.stream()

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/processes/implementations/filesystem/sync/FilesystemSyncProcessTest.java
View File

@ -24,6 +24,7 @@ package com.kingsrook.qqq.backend.module.filesystem.processes.implementations.fi
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import com.kingsrook.qqq.backend.core.actions.processes.RunBackendStepAction;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
@ -105,7 +106,7 @@ class FilesystemSyncProcessTest extends BaseTest
{
String path = ((FilesystemTableBackendDetails) table.getBackendDetails()).getBasePath();
File file = new File(basePath + "/" + path + "/" + name);
FileUtils.writeStringToFile(file, content);
FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8);
}

7
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3InsertActionTest.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.actions;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.amazonaws.services.s3.model.S3Object;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -68,9 +69,9 @@ public class S3InsertActionTest extends BaseS3Test
assertThat(insertOutput.getRecords())
.allMatch(record -> record.getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH).contains("blobs"));
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List lines = IOUtils.readLines(object.getObjectContent());
String fullPath = insertOutput.getRecords().get(0).getBackendDetailString(FilesystemRecordBackendDetailFields.FULL_PATH);
S3Object object = getAmazonS3().getObject(BUCKET_NAME, fullPath);
List<String> lines = IOUtils.readLines(object.getObjectContent(), StandardCharsets.UTF_8);
assertEquals("Hi, Bob.", lines.get(0));
}

3
qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UtilsTest.java
View File

@ -23,6 +23,7 @@ package com.kingsrook.qqq.backend.module.filesystem.s3.utils;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@ -76,7 +77,7 @@ public class S3UtilsTest extends BaseS3Test
List<S3ObjectSummary> s3ObjectSummaries = s3Utils.listObjectsInBucketMatchingGlob(BUCKET_NAME, "test-files", "");
S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.stream().filter(o -> o.getKey().contains("1.csv")).findAny().get();
InputStream inputStream = s3Utils.getObjectAsInputStream(s3ObjectSummary);
String csvFromS3 = IOUtils.toString(inputStream);
String csvFromS3 = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
assertEquals(getCSVData1(), csvFromS3, "File from S3 should match expected content");
}