diff --git a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java index 80db887e..c9e3c8b5 100644 --- a/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java +++ b/qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java @@ -23,21 +23,26 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException; import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting; +import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction; import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException; import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath; @@ -81,9 +86,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction closer = closable -> + { + if(closable != null) + { + try + { + closable.close(); + } + catch(Exception e) + { + LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName())); + } + } + }; + + closer.accept(sshClient); + closer.accept(clientSession); + closer.accept(sftpClient); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException + { + this.sshClient = SshClient.setUpDefaultClient(); + sshClient.start(); + + this.clientSession = sshClient.connect(username, hostName, port).verify().getSession(); + clientSession.addPasswordIdentity(password); + clientSession.auth().verify(); + + this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession); + return (this.sftpClient); + } + + + /*************************************************************************** ** ***************************************************************************/ @@ -195,8 +239,22 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction rs = new ArrayList<>(); + String fullPath = getFullBasePath(table, backendBase); + + // todo - move somewhere shared + // todo - should all do this? + if(filter != null) + { + for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria())) + { + if(isPathEqualsCriteria(criteria)) + { + fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar); + } + } + } + + List rs = new ArrayList<>(); for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath)) { @@ -211,10 +269,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction patternExpectedCountMap = Map.of( + "%.txt", 3, + "sub%", 2, + "%1%", 1, + "%", 3, + "*", 0 + ); + + for(Map.Entry entry : patternExpectedCountMap.entrySet()) + { + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter() + .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")) + .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey()))); + QueryOutput queryOutput = new QueryAction().execute(queryInput); + Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey()); + } + } + finally + { + rmrfInContainer(subfolderPath); + } + } + + + /******************************************************************************* ** *******************************************************************************/ @@ -71,7 +139,7 @@ class SFTPQueryActionTest extends BaseSFTPTest mkdirInSftpContainerUnderHomeTestuser("empty-folder/files"); - QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); + QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS); assertThatThrownBy(() -> new QueryAction().execute(queryInput)) .hasMessageContaining("Could not find Backend Variant information for Backend"); @@ -91,15 +159,4 @@ class SFTPQueryActionTest extends BaseSFTPTest // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query"); } - - - /******************************************************************************* - ** - *******************************************************************************/ - private QueryInput initQueryRequest() throws QException - { - QueryInput queryInput = new QueryInput(); - return queryInput; - } - } \ No newline at end of file