Merge pull request #40 from Kingsrook/dev

refreshign CE-609 with dev

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/reporting/ExportAction.java

@@ -197,7 +197,27 @@ public class ExportAction
                String   joinTableName = parts[0];
                if(!addedJoinNames.contains(joinTableName))
                {
-                  queryJoins.add(new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true));
+                  QueryJoin queryJoin = new QueryJoin(joinTableName).withType(QueryJoin.Type.LEFT).withSelect(true);
+                  queryJoins.add(queryJoin);
+
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  // in at least some cases, we need to let the queryJoin know what join-meta-data to use... //
+                  // This code basically mirrors what QFMD is doing right now, so it's better -              //
+                  // but shouldn't all of this just be in JoinsContext?  it does some of this...             //
+                  /////////////////////////////////////////////////////////////////////////////////////////////
+                  QTableMetaData        table               = exportInput.getTable();
+                  Optional<ExposedJoin> exposedJoinOptional = CollectionUtils.nonNullList(table.getExposedJoins()).stream().filter(ej -> ej.getJoinTable().equals(joinTableName)).findFirst();
+                  if(exposedJoinOptional.isEmpty())
+                  {
+                     throw (new QException("Could not find exposed join between base table " + table.getName() + " and requested join table " + joinTableName));
+                  }
+                  ExposedJoin exposedJoin = exposedJoinOptional.get();
+
+                  if(exposedJoin.getJoinPath().size() == 1)
+                  {
+                     queryJoin.setJoinMetaData(QContext.getQInstance().getJoin(exposedJoin.getJoinPath().get(exposedJoin.getJoinPath().size() - 1)));
+                  }
+
                   addedJoinNames.add(joinTableName);
                }
             }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/scripts/ExecuteCodeAction.java

@@ -33,6 +33,7 @@ import com.kingsrook.qqq.backend.core.actions.scripts.logging.QCodeExecutionLogg
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.ScriptExecutionLoggerInterface;
 import com.kingsrook.qqq.backend.core.actions.scripts.logging.StoreScriptLogAndScriptLogLineExecutionLogger;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -50,6 +51,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision;
 import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.ObjectUtils;
 import com.kingsrook.qqq.backend.core.utils.StringUtils;
 
 
@@ -112,6 +114,11 @@ public class ExecuteCodeAction
             context.putAll(input.getInput());
          }
 
+         //////////////////////////////////////////
+         // safely always set the deploymentMode //
+         //////////////////////////////////////////
+         context.put("deploymentMode", ObjectUtils.tryAndRequireNonNullElse(() -> QContext.getQInstance().getDeploymentMode(), null));
+
          /////////////////////////////////////////////////////////////////////////////////
          // set the qCodeExecutor into any context objects which are QCodeExecutorAware //
          /////////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/ReplaceAction.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.actions.AbstractQActionFunction;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.tables.helpers.UniqueKeyHelper;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
@@ -61,6 +62,9 @@ import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
  *******************************************************************************/
 public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, ReplaceOutput>
 {
+   private static final QLogger LOG = QLogger.getLogger(ReplaceAction.class);
+
+
 
    /*******************************************************************************
     **
@@ -159,6 +163,7 @@ public class ReplaceAction extends AbstractQActionFunction<ReplaceInput, Replace
       {
          if(weOwnTheTransaction)
          {
+            LOG.warn("Caught top-level ReplaceAction exception - rolling back exception", e);
             transaction.rollback();
          }
          throw (new QException("Error executing replace action", e));

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/SearchPossibleValueSourceAction.java

@@ -244,8 +244,6 @@ public class SearchPossibleValueSourceAction
          }
       }
 
-      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
-
       // todo - skip & limit as params
       queryFilter.setLimit(250);
 
@@ -257,6 +255,9 @@ public class SearchPossibleValueSourceAction
          input.getDefaultQueryFilter().addSubFilter(queryFilter);
          queryFilter = input.getDefaultQueryFilter();
       }
+
+      queryFilter.setOrderBys(possibleValueSource.getOrderByFields());
+
       queryInput.setFilter(queryFilter);
 
       QueryOutput             queryOutput     = new QueryAction().execute(queryInput);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreter.java

@@ -347,7 +347,7 @@ public class QMetaDataVariableInterpreter
          if(canParseAsInteger(envValue))
          {
             LOG.info("Read env var [" + environmentVariableName + "] as integer " + environmentVariableName);
-            return (Integer.parseInt(propertyValue));
+            return (Integer.parseInt(envValue));
          }
          else
          {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/audits/AuditsMetaDataProvider.java

@@ -116,16 +116,19 @@ public class AuditsMetaDataProvider
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT_TABLE)
          .withTableName(TABLE_NAME_AUDIT_TABLE)
+         .withOrderByField("name")
       );
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT_USER)
          .withTableName(TABLE_NAME_AUDIT_USER)
+         .withOrderByField("name")
       );
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(TABLE_NAME_AUDIT)
          .withTableName(TABLE_NAME_AUDIT)
+         .withOrderByField("id", false)
       );
    }
 

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/QInstance.java

@@ -93,6 +93,7 @@ public class QInstance
 
    private Map<String, QSupplementalInstanceMetaData> supplementalMetaData = new LinkedHashMap<>();
 
+   private String              deploymentMode;
    private Map<String, String> environmentValues = new LinkedHashMap<>();
    private String              defaultTimeZoneId = "UTC";
 
@@ -1165,4 +1166,36 @@ public class QInstance
       }
       this.joinGraph = joinGraph;
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for deploymentMode
+    *******************************************************************************/
+   public String getDeploymentMode()
+   {
+      return (this.deploymentMode);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for deploymentMode
+    *******************************************************************************/
+   public void setDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for deploymentMode
+    *******************************************************************************/
+   public QInstance withDeploymentMode(String deploymentMode)
+   {
+      this.deploymentMode = deploymentMode;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/querystats/QueryStatMetaDataProvider.java

@@ -187,7 +187,8 @@ public class QueryStatMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QueryStat.TABLE_NAME)
-         .withTableName(QueryStat.TABLE_NAME));
+         .withTableName(QueryStat.TABLE_NAME))
+         .withOrderByField("id", false);
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/savedfilters/SavedFiltersMetaDataProvider.java

@@ -88,7 +88,8 @@ public class SavedFiltersMetaDataProvider
          .withName(SavedFilter.TABLE_NAME)
          .withType(QPossibleValueSourceType.TABLE)
          .withTableName(SavedFilter.TABLE_NAME)
-         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY);
+         .withValueFormatAndFields(PVSValueFormatAndFields.LABEL_ONLY)
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/scripts/ScriptsMetaDataProvider.java

@@ -303,19 +303,24 @@ public class ScriptsMetaDataProvider
    {
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(Script.TABLE_NAME)
-         .withTableName(Script.TABLE_NAME));
+         .withTableName(Script.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptRevision.TABLE_NAME)
-         .withTableName(ScriptRevision.TABLE_NAME));
+         .withTableName(ScriptRevision.TABLE_NAME)
+         .withOrderByField("scriptId")
+         .withOrderByField("sequenceNo", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptType.TABLE_NAME)
-         .withTableName(ScriptType.TABLE_NAME));
+         .withTableName(ScriptType.TABLE_NAME)
+         .withOrderByField("name"));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptLog.TABLE_NAME)
-         .withTableName(ScriptLog.TABLE_NAME));
+         .withTableName(ScriptLog.TABLE_NAME)
+         .withOrderByField("id", false));
 
       instance.addPossibleValueSource(new QPossibleValueSource()
          .withName(ScriptTypeFileMode.NAME)

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/tables/QQQTablesMetaDataProvider.java

@@ -127,7 +127,8 @@ public class QQQTablesMetaDataProvider
       return (new QPossibleValueSource()
          .withType(QPossibleValueSourceType.TABLE)
          .withName(QQQTable.TABLE_NAME)
-         .withTableName(QQQTable.TABLE_NAME));
+         .withTableName(QQQTable.TABLE_NAME))
+         .withOrderByField("label");
    }
 
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamed/StreamedETLBackendStep.java

@@ -92,6 +92,7 @@ public class StreamedETLBackendStep implements BackendStep
          ////////////////////////////////////////////////////////////////////////////////
          // rollback the work, then re-throw the error for up-stream to catch & report //
          ////////////////////////////////////////////////////////////////////////////////
+         LOG.warn("Caught top-level process exception - rolling back transaction", e);
          transaction.rollback();
          throw (e);
       }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/etl/streamedwithfrontend/StreamedETLExecuteStep.java

@@ -198,8 +198,13 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
          ////////////////////////////////////////////////////////////////////////////////
          if(transaction.isPresent())
          {
+            LOG.warn("Caught top-level process exception - rolling back transaction", e);
             transaction.get().rollback();
          }
+         else
+         {
+            LOG.warn("Caught top-level process exception - would roll back transaction, but none is present", e);
+         }
          throw (e);
       }
       finally
@@ -302,6 +307,7 @@ public class StreamedETLExecuteStep extends BaseStreamedETLStep implements Backe
       {
          if(doPageLevelTransaction && transaction.isPresent())
          {
+            LOG.warn("Caught page-level process exception - rolling back transaction", e);
             transaction.get().rollback();
          }
          throw (e);

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/instances/QMetaDataVariableInterpreterTest.java

@@ -273,6 +273,52 @@ class QMetaDataVariableInterpreterTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testGetIntegerFromPropertyOrEnvironment()
+   {
+      QMetaDataVariableInterpreter interpreter = new QMetaDataVariableInterpreter();
+
+      //////////////////////////////////////////////////////////
+      // if neither prop nor env is set, get back the default //
+      //////////////////////////////////////////////////////////
+      assertEquals(1, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 1));
+      assertEquals(2, interpreter.getIntegerFromPropertyOrEnvironment("notSet", "NOT_SET", 2));
+
+      /////////////////////////////////////////////
+      // unrecognized values are same as not set //
+      /////////////////////////////////////////////
+      System.setProperty("unrecognized", "asdf");
+      interpreter.setEnvironmentOverrides(Map.of("UNRECOGNIZED", "qwerty"));
+      assertEquals(3, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 3));
+      assertEquals(4, interpreter.getIntegerFromPropertyOrEnvironment("unrecognized", "UNRECOGNIZED", 4));
+
+      /////////////////////////////////
+      // if only prop is set, get it //
+      /////////////////////////////////
+      assertEquals(5, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 5));
+      System.setProperty("foo.size", "6");
+      assertEquals(6, interpreter.getIntegerFromPropertyOrEnvironment("foo.size", "FOO_SIZE", 7));
+
+      ////////////////////////////////
+      // if only env is set, get it //
+      ////////////////////////////////
+      assertEquals(8, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 8));
+      interpreter.setEnvironmentOverrides(Map.of("BAR_SIZE", "9"));
+      assertEquals(9, interpreter.getIntegerFromPropertyOrEnvironment("bar.size", "BAR_SIZE", 10));
+
+      ///////////////////////////////////
+      // if both are set, get the prop //
+      ///////////////////////////////////
+      System.setProperty("baz.size", "11");
+      interpreter.setEnvironmentOverrides(Map.of("BAZ_SIZE", "12"));
+      assertEquals(11, interpreter.getIntegerFromPropertyOrEnvironment("baz.size", "BAZ_SIZE", 13));
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-dev-tools/bin/xbar-circleci-latest.sh

@@ -47,10 +47,10 @@ checkBuild()
    shortRepo="$repo"
    case $repo in
     qqq)                             shortRepo="qqq";;
-    qqq-frontend-core)               shortRepo="f'core";;
+    qqq-frontend-core)               shortRepo="fc";;
-    qqq-frontend-material-dashboard) shortRepo="m-db";;
+    qqq-frontend-material-dashboard) shortRepo="qfmd";;
     ColdTrack-Live)                  shortRepo="ctl";;
-    ColdTrack-Live-Scripts)          shortRepo="ct1-scr";;
+    ColdTrack-Live-Scripts)          shortRepo="cls";;
    esac
 
    timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S%z" $(echo "$startDate" | sed 's/\....Z/+0000/') +%s)

qqq-language-support-javascript/src/test/java/com/kingsrook/qqq/languages/javascript/ExecuteCodeActionTest.java

@@ -30,6 +30,7 @@ import java.util.Map;
 import com.kingsrook.qqq.backend.core.actions.scripts.ExecuteCodeAction;
 import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutor;
 import com.kingsrook.qqq.backend.core.actions.scripts.QCodeExecutorAware;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QCodeException;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
@@ -307,6 +308,32 @@ class ExecuteCodeActionTest extends BaseTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testDeploymentModeIsInContext() throws QException
+   {
+      String scriptSource = """
+         return (deploymentMode);
+         """;
+
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      // first, with no deployment mode in the qInstance, make sure we can run, but get a null output //
+      //////////////////////////////////////////////////////////////////////////////////////////////////
+      OneTestOutput oneTestOutput = testOne(null, scriptSource, new HashMap<>());
+      assertNull(oneTestOutput.executeCodeOutput.getOutput());
+
+      /////////////////////////////////////////////////////////////////////
+      // next, set a deploymentMode, and assert that we get it back out. //
+      /////////////////////////////////////////////////////////////////////
+      QContext.getQInstance().setDeploymentMode("unit-test");
+      oneTestOutput = testOne(null, scriptSource, new HashMap<>());
+      assertEquals("unit-test", oneTestOutput.executeCodeOutput.getOutput());
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/