Move makeConnection to its own method (for use by test process); add postAction to try to close the things; add looking for 'path' criteria and adding it to readDir call

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/AbstractSFTPAction.java

@@ -23,21 +23,26 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
 
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QRuntimeException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantSetting;
+import com.kingsrook.qqq.backend.core.model.metadata.variants.BackendVariantsUtil;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
 import com.kingsrook.qqq.backend.module.filesystem.exceptions.FilesystemException;
 import com.kingsrook.qqq.backend.module.filesystem.sftp.model.SFTPDirEntryWithPath;
@@ -81,9 +86,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
       {
          SFTPBackendMetaData sftpBackendMetaData = getBackendMetaData(SFTPBackendMetaData.class, backendMetaData);
 
-         this.sshClient = SshClient.setUpDefaultClient();
-         sshClient.start();
-
          String  username = sftpBackendMetaData.getUsername();
          String  password = sftpBackendMetaData.getPassword();
          String  hostName = sftpBackendMetaData.getHostName();
@@ -91,7 +93,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
 
          if(backendMetaData.getUsesVariants())
          {
-            QRecord variantRecord = getVariantRecord(backendMetaData);
+            QRecord variantRecord = BackendVariantsUtil.getVariantRecord(backendMetaData);
             LOG.debug("Getting SFTP connection credentials from variant record",
                logPair("tableName", backendMetaData.getBackendVariantsConfig().getOptionsTableName()),
                logPair("id", variantRecord.getValue("id")),
@@ -119,11 +121,7 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
             }
          }
 
-         this.clientSession = sshClient.connect(username, hostName, port).verify().getSession();
+         makeConnection(username, hostName, port, password);
-         clientSession.addPasswordIdentity(password);
-         clientSession.auth().verify();
-
-         this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
       }
       catch(IOException e)
       {
@@ -133,6 +131,52 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
 
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   @Override
+   public void postAction()
+   {
+      Consumer<AutoCloseable> closer = closable ->
+      {
+        if(closable != null)
+        {
+           try
+           {
+              closable.close();
+           }
+           catch(Exception e)
+           {
+              LOG.info("Error closing SFTP resource", e, logPair("type", closable.getClass().getSimpleName()));
+           }
+        }
+      };
+
+      closer.accept(sshClient);
+      closer.accept(clientSession);
+      closer.accept(sftpClient);
+   }
+
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected SftpClient makeConnection(String username, String hostName, Integer port, String password) throws IOException
+   {
+      this.sshClient = SshClient.setUpDefaultClient();
+      sshClient.start();
+
+      this.clientSession = sshClient.connect(username, hostName, port).verify().getSession();
+      clientSession.addPasswordIdentity(password);
+      clientSession.auth().verify();
+
+      this.sftpClient = SftpClientFactory.instance().createSftpClient(clientSession);
+      return (this.sftpClient);
+   }
+
+
+
    /***************************************************************************
     **
     ***************************************************************************/
@@ -195,8 +239,22 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
    {
       try
       {
-         String                     fullPath = getFullBasePath(table, backendBase);
+         String fullPath = getFullBasePath(table, backendBase);
-         List<SFTPDirEntryWithPath> rs       = new ArrayList<>();
+
+         // todo - move somewhere shared
+         // todo - should all do this?
+         if(filter != null)
+         {
+            for(QFilterCriteria criteria : CollectionUtils.nonNullList(filter.getCriteria()))
+            {
+               if(isPathEqualsCriteria(criteria))
+               {
+                  fullPath = stripDuplicatedSlashes(fullPath + File.separatorChar + criteria.getValues().get(0) + File.separatorChar);
+               }
+            }
+         }
+
+         List<SFTPDirEntryWithPath> rs = new ArrayList<>();
 
          for(SftpClient.DirEntry dirEntry : sftpClient.readDir(fullPath))
          {
@@ -211,10 +269,6 @@ public class AbstractSFTPAction extends AbstractBaseFilesystemAction<SFTPDirEntr
                continue;
             }
 
-            // todo filter/glob
-            // todo skip
-            // todo limit
-            // todo order by
             rs.add(new SFTPDirEntryWithPath(fullPath, dirEntry));
          }
 

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/BaseSFTPTest.java

@@ -25,7 +25,6 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp;
 import com.kingsrook.qqq.backend.module.filesystem.BaseTest;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.testcontainers.containers.Container;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.utility.MountableFile;
 
@@ -63,7 +62,7 @@ public class BaseSFTPTest extends BaseTest
 
       for(int i = 0; i < 5; i++)
       {
-         sftpContainer.copyFileToContainer(MountableFile.forClasspathResource("files/testfile.txt"), REMOTE_DIR + "/testfile-" + i + ".txt");
+         copyFileToContainer("files/testfile.txt", REMOTE_DIR + "/testfile-" + i + ".txt");
       }
 
       grantUploadFilesDirWritePermission();
@@ -72,6 +71,23 @@ public class BaseSFTPTest extends BaseTest
    }
 
 
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected static void copyFileToContainer(String sourceFileClasspathResourceName, String fullRemotePath)
+   {
+      sftpContainer.copyFileToContainer(MountableFile.forClasspathResource(sourceFileClasspathResourceName), fullRemotePath);
+   }
+
+
+   /***************************************************************************
+    **
+    ***************************************************************************/
+   protected static void rmrfInContainer(String fullRemotePath) throws Exception
+   {
+      sftpContainer.execInContainer("rm", "-rf", fullRemotePath);
+   }
+
 
    /***************************************************************************
     **
@@ -133,7 +149,6 @@ public class BaseSFTPTest extends BaseTest
     ***************************************************************************/
    protected void mkdirInSftpContainerUnderHomeTestuser(String path) throws Exception
    {
-      Container.ExecResult mkdir = sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
+      sftpContainer.execInContainer("mkdir", "-p", "/home/testuser/" + path);
-      System.out.println(mkdir.getExitCode());
    }
 }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/sftp/actions/SFTPQueryActionTest.java

@@ -23,11 +23,15 @@ package com.kingsrook.qqq.backend.module.filesystem.sftp.actions;
 
 
 import java.util.List;
+import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -49,7 +53,7 @@ class SFTPQueryActionTest extends BaseSFTPTest
     **
     *******************************************************************************/
    @Test
-   public void testQuery1() throws QException
+   public void testSimpleQuery() throws QException
    {
       QueryInput  queryInput  = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE);
       QueryOutput queryOutput = new QueryAction().execute(queryInput);
@@ -57,6 +61,70 @@ class SFTPQueryActionTest extends BaseSFTPTest
    }
 
 
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryWithPath() throws Exception
+   {
+      String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
+      try
+      {
+         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
+         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
+
+         QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE)
+            .withFilter(new QQueryFilter(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder")));
+         QueryOutput queryOutput = new QueryAction().execute(queryInput);
+         Assertions.assertEquals(2, queryOutput.getRecords().size(), "Expected # of rows from subfolder path query");
+      }
+      finally
+      {
+         rmrfInContainer(subfolderPath);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testQueryWithPathAndNameLike() throws Exception
+   {
+      String subfolderPath = "/home/" + USERNAME + "/" + BACKEND_FOLDER + "/" + TABLE_FOLDER + "/subfolder/";
+      try
+      {
+         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub1.txt");
+         copyFileToContainer("files/testfile.txt", subfolderPath + "/sub2.txt");
+         copyFileToContainer("files/testfile.txt", subfolderPath + "/who.txt");
+
+         Map<String, Integer> patternExpectedCountMap = Map.of(
+            "%.txt", 3,
+            "sub%", 2,
+            "%1%", 1,
+            "%", 3,
+            "*", 0
+         );
+
+         for(Map.Entry<String, Integer> entry : patternExpectedCountMap.entrySet())
+         {
+            QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE).withFilter(new QQueryFilter()
+               .withCriteria(new QFilterCriteria("path", QCriteriaOperator.EQUALS, "subfolder"))
+               .withCriteria(new QFilterCriteria("baseName", QCriteriaOperator.LIKE, entry.getKey())));
+            QueryOutput queryOutput = new QueryAction().execute(queryInput);
+            Assertions.assertEquals(entry.getValue(), queryOutput.getRecords().size(), "Expected # of rows from subfolder path, baseName like: " + entry.getKey());
+         }
+      }
+      finally
+      {
+         rmrfInContainer(subfolderPath);
+      }
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
@@ -71,7 +139,7 @@ class SFTPQueryActionTest extends BaseSFTPTest
 
       mkdirInSftpContainerUnderHomeTestuser("empty-folder/files");
 
-      QueryInput  queryInput  = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
+      QueryInput queryInput = new QueryInput(TestUtils.TABLE_NAME_SFTP_FILE_VARIANTS);
       assertThatThrownBy(() -> new QueryAction().execute(queryInput))
          .hasMessageContaining("Could not find Backend Variant information for Backend");
 
@@ -91,15 +159,4 @@ class SFTPQueryActionTest extends BaseSFTPTest
       // Assertions.assertEquals(5, queryOutput.getRecords().size(), "Expected # of rows from unfiltered query");
    }
 
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private QueryInput initQueryRequest() throws QException
-   {
-      QueryInput queryInput = new QueryInput();
-      return queryInput;
-   }
-
 }