mirror of
https://github.com/Kingsrook/qqq.git
synced 2025-07-17 20:50:44 +00:00
Move makeConnection to its own method (for use by test process); add postAction to try to close the things; add looking for 'path' criteria and adding it to readDir call
This commit is contained in:
@ -23,21 +23,26 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
|
||||
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
|
||||
@ -81,9 +86,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
{
|
||||
SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData);
|
||||
|
||||
this.sshClient = SshClient.setUpDefaultClient();
|
||||
sshClient.start();
|
||||
|
||||
String username = sftpBackendMetaData.getUsername();
|
||||
String password = sftpBackendMetaData.getPassword();
|
||||
String hostName = sftpBackendMetaData.getHostName();
|
||||
@ -91,7 +93,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
|
||||
if(backendMetaData.getUsesVariants())
|
||||
{
|
||||
QRecord variantRecord = getVariantRecord(backendMetaData);
|
||||
QRecord variantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData);
|
||||
LOG.debug("Getting SFTP connection credentials from variant record",
|
||||
logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()),
|
||||
logPair("id", variantRecord.getValue("id")),
|
||||
@ -119,11 +121,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
}
|
||||
}
|
||||
|
||||
this.clientSession = sshClient.connect(username, hostName, port).verify().getSession();
|
||||
clientSession.addPasswordIdentity(password);
|
||||
clientSession.auth().verify();
|
||||
|
||||
this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
|
||||
makeConnection(username, hostName, port, password);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
@ -133,6 +131,52 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public void postAction()
|
||||
{
|
||||
Consumer<AutoCloseable> closer = closable ->
|
||||
{
|
||||
if(closable != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
closable.close();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName()));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
closer.accept(sshClient);
|
||||
closer.accept(clientSession);
|
||||
closer.accept(sftpClient);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException
|
||||
{
|
||||
this.sshClient = SshClient.setUpDefaultClient();
|
||||
sshClient.start();
|
||||
|
||||
this.clientSession = sshClient.connect(username, hostName, port).verify().getSession();
|
||||
clientSession.addPasswordIdentity(password);
|
||||
clientSession.auth().verify();
|
||||
|
||||
this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
|
||||
return (this.sftpClient);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
@ -195,8 +239,22 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
{
|
||||
try
|
||||
{
|
||||
String fullPath = getFullBasePath(table, backendBase);
|
||||
List<SFTPDirEntryWithPath> rs = new ArrayList<>();
|
||||
String fullPath = getFullBasePath(table, backendBase);
|
||||
|
||||
// todo - move somewhere shared
|
||||
// todo - should all do this?
|
||||
if(filter != null)
|
||||
{
|
||||
for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
|
||||
{
|
||||
if(isPathEqualsCriteria(criteria))
|
||||
{
|
||||
fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<SFTPDirEntryWithPath> rs = new ArrayList<>();
|
||||
|
||||
for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
|
||||
{
|
||||
@ -211,10 +269,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
|
||||
continue;
|
||||
}
|
||||
|
||||
// todo filter/glob
|
||||
// todo skip
|
||||
// todo limit
|
||||
// todo order by
|
||||
rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
|
||||
}
|
||||
|
||||
|
||||
@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.testcontainers.containers.Container;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
@ -63,7 +62,7 @@ public class BaseSFTPTest extends BaseTest
|
||||
|
||||
for(int i = 0; i < 5; i++)
|
||||
{
|
||||
sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt");
|
||||
copyFileToContainer("files/testfile.txt", REMOTE_DIR + "/testfile-" + i + ".txt");
|
||||
}
|
||||
|
||||
grantUploadFilesDirWritePermission();
|
||||
@ -72,6 +71,23 @@ public class BaseSFTPTest extends BaseTest
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
protected static void copyFileToContainer(String sourceFileClasspathResourceName, String fullRemotePath)
|
||||
{
|
||||
sftpContainer.copyFileToContainer(MountableFile.forClasspathResource(sourceFileClasspathResourceName), fullRemotePath);
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
***************************************************************************/
|
||||
protected static void rmrfInContainer(String fullRemotePath) throws Exception
|
||||
{
|
||||
sftpContainer.execInContainer("rm", "-rf", fullRemotePath);
|
||||
}
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
**
|
||||
@ -133,7 +149,6 @@ public class BaseSFTPTest extends BaseTest
|
||||
***************************************************************************/
|
||||
protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception
|
||||
{
|
||||
Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
|
||||
System.out.println(mkdir.getExitCode());
|
||||
sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,11 +23,15 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
@ -49,7 +53,7 @@ class SFTPQueryActionTest extends BaseSFTPTest
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
public void testQuery1() throws QException
|
||||
public void testSimpleQuery() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE);
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
@ -57,6 +61,70 @@ class SFTPQueryActionTest extends BaseSFTPTest
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueryWithPath() throws Exception
|
||||
{
|
||||
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
|
||||
try
|
||||
{
|
||||
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
|
||||
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
|
||||
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")));
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query");
|
||||
}
|
||||
finally
|
||||
{
|
||||
rmrfInContainer(subfolderPath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void testQueryWithPathAndNameLike() throws Exception
|
||||
{
|
||||
String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
|
||||
try
|
||||
{
|
||||
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
|
||||
copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
|
||||
copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt");
|
||||
|
||||
Map<String, Integer> patternExpectedCountMap = Map.of(
|
||||
"%.txt", 3,
|
||||
"sub%", 2,
|
||||
"%1%", 1,
|
||||
"%", 3,
|
||||
"*", 0
|
||||
);
|
||||
|
||||
for(Map.Entry<String, Integer> entry : patternExpectedCountMap.entrySet())
|
||||
{
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))
|
||||
.withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey())));
|
||||
QueryOutput queryOutput = new QueryAction().execute(queryInput);
|
||||
Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey());
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
rmrfInContainer(subfolderPath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@ -71,7 +139,7 @@ class SFTPQueryActionTest extends BaseSFTPTest
|
||||
|
||||
mkdirInSftpContainerUnderHomeTestuser("empty-folder/files");
|
||||
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
|
||||
QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
|
||||
assertThatThrownBy(() -> new QueryAction().execute(queryInput))
|
||||
.hasMessageContaining("Could not find Backend Variant information for Backend");
|
||||
|
||||
@ -91,15 +159,4 @@ class SFTPQueryActionTest extends BaseSFTPTest
|
||||
// Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private QueryInput initQueryRequest() throws QException
|
||||
{
|
||||
QueryInput queryInput = new QueryInput();
|
||||
return queryInput;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user