Merge branch 'dev' into feature/CTLE-434-oms-update-business-logic

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -129,29 +129,14 @@ public class GetAction
             ///////////////////////////////////////////////////////////////////////
             // if the record wasn't found, see if we should look in cache-source //
             ///////////////////////////////////////////////////////////////////////
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput);
             if(recordFromSource != null)
             {
+               /////////////////////////////////////////////////////////////////////////////////////////////////
+               // good, we found a record from the source, make sure we should cache it, and if so, do it now //
+               /////////////////////////////////////////////////////////////////////////////////////////////////
                QRecord recordToCache     = mapSourceRecordToCacheRecord(table, recordFromSource);
-               boolean shouldCacheRecord = true;
+               boolean shouldCacheRecord = shouldCacheRecord(table, recordToCache);
-
-               ////////////////////////////////////////////////////////////////////////////////
-               // see if there are any exclustions that need to be considered for this table //
-               ////////////////////////////////////////////////////////////////////////////////
-               recordMatchExclusionLoop:
-               for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
-               {
-                  for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
-                  {
-                     if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
-                     {
-                        LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
-                        shouldCacheRecord = false;
-                        break recordMatchExclusionLoop;
-                     }
-                  }
-               }
-
                if(shouldCacheRecord)
                {
                   InsertInput insertInput = new InsertInput();
@@ -184,12 +169,50 @@ public class GetAction
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private boolean shouldCacheRecord(QTableMetaData table, QRecord recordToCache)
+   {
+      boolean shouldCacheRecord = true;
+      recordMatchExclusionLoop:
+      for(CacheUseCase useCase : CollectionUtils.nonNullList(table.getCacheOf().getUseCases()))
+      {
+         for(QQueryFilter filter : CollectionUtils.nonNullList(useCase.getExcludeRecordsMatching()))
+         {
+            if(BackendQueryFilterUtils.doesRecordMatch(filter, recordToCache))
+            {
+               LOG.info("Not caching record because it matches a use case's filter exclusion", new LogPair("record", recordToCache), new LogPair("filter", filter));
+               shouldCacheRecord = false;
+               break recordMatchExclusionLoop;
+            }
+         }
+      }
+
+      return (shouldCacheRecord);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/
    private static QRecord mapSourceRecordToCacheRecord(QTableMetaData table, QRecord recordFromSource)
    {
       QRecord cacheRecord = new QRecord(recordFromSource);
+
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      // make sure every value in the qRecord is set, because we will possibly be doing an update //
+      // on this record and want to null out any fields not set, not leave them populated         //
+      //////////////////////////////////////////////////////////////////////////////////////////////
+      for(String fieldName : table.getFields().keySet())
+      {
+         if(!cacheRecord.getValues().containsKey(fieldName))
+         {
+            cacheRecord.setValue(fieldName, null);
+         }
+      }
+
       if(StringUtils.hasContent(table.getCacheOf().getCachedDateFieldName()))
       {
          cacheRecord.setValue(table.getCacheOf().getCachedDateFieldName(), Instant.now());
@@ -212,33 +235,54 @@ public class GetAction
          Instant cachedDate   = cachedRecord.getValueInstant(table.getCacheOf().getCachedDateFieldName());
          if(cachedDate == null || cachedDate.isBefore(Instant.now().minus(expirationSeconds, ChronoUnit.SECONDS)))
          {
-            QRecord recordFromSource = tryToGetFromCacheSource(getInput, getOutput);
+            //////////////////////////////////////////////////////////////////////////
+            // keep the serial key from the old record in case we need to delete it //
+            //////////////////////////////////////////////////////////////////////////
+            Serializable oldRecordPrimaryKey      = getOutput.getRecord().getValue(table.getPrimaryKeyField());
+            boolean      shouldDeleteCachedRecord = true;
+
+            ///////////////////////////////////////////
+            // fetch record from original source now //
+            ///////////////////////////////////////////
+            QRecord recordFromSource = tryToGetFromCacheSource(getInput);
             if(recordFromSource != null)
             {
-               ///////////////////////////////////////////////////////////////////
+               //////////////////////////////////////////////////////////////////////
-               // if the record was found in the source, update it in the cache //
+               // if the record was found in the source, put it into the output    //
-               ///////////////////////////////////////////////////////////////////
+               // object so returned back to caller, check that it should actually //
+               // be cached before doing so                                        //
+               //////////////////////////////////////////////////////////////////////
                QRecord recordToCache = mapSourceRecordToCacheRecord(table, recordFromSource);
                recordToCache.setValue(table.getPrimaryKeyField(), cachedRecord.getValue(table.getPrimaryKeyField()));
+               getOutput.setRecord(recordToCache);
 
+               if(shouldCacheRecord(table, recordToCache))
+               {
                   UpdateInput updateInput = new UpdateInput();
                   updateInput.setTableName(getInput.getTableName());
                   updateInput.setRecords(List.of(recordToCache));
                   UpdateOutput updateOutput = new UpdateAction().execute(updateInput);
-
                   getOutput.setRecord(updateOutput.getRecords().get(0));
+                  shouldDeleteCachedRecord = false;
+               }
             }
             else
+            {
+               ///////////////////////////////////////////////////////////////////////////////////////
+               // if we did not get a record back from the source, empty out the getOutput's record //
+               ///////////////////////////////////////////////////////////////////////////////////////
+               getOutput.setRecord(null);
+            }
+
+            if(shouldDeleteCachedRecord)
             {
                /////////////////////////////////////////////////////////////////////////////
                // if the record is no longer in the source, then remove it from the cache //
                /////////////////////////////////////////////////////////////////////////////
                DeleteInput deleteInput = new DeleteInput();
                deleteInput.setTableName(getInput.getTableName());
-               deleteInput.setPrimaryKeys(List.of(getOutput.getRecord().getValue(table.getPrimaryKeyField())));
+               deleteInput.setPrimaryKeys(List.of(oldRecordPrimaryKey));
                new DeleteAction().execute(deleteInput);
-
-               getOutput.setRecord(null);
             }
          }
       }
@@ -249,7 +293,7 @@ public class GetAction
    /*******************************************************************************
     **
     *******************************************************************************/
-   private QRecord tryToGetFromCacheSource(GetInput getInput, GetOutput getOutput) throws QException
+   private QRecord tryToGetFromCacheSource(GetInput getInput) throws QException
    {
       QRecord        recordFromSource = null;
       QTableMetaData table            = getInput.getTable();

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/JoinsContext.java

@@ -111,13 +111,13 @@ public class JoinsContext
             if(join.getLeftTable().equals(tmpTable.getName()))
             {
                QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join).withType(QueryJoin.Type.INNER);
-               this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+               this.addQueryJoin(queryJoin);
                tmpTable = instance.getTable(join.getRightTable());
             }
             else if(join.getRightTable().equals(tmpTable.getName()))
             {
                QueryJoin queryJoin = new ImplicitQueryJoinForSecurityLock().withJoinMetaData(join.flip()).withType(QueryJoin.Type.INNER);
-               this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+               this.addQueryJoin(queryJoin); //
                tmpTable = instance.getTable(join.getLeftTable());
             }
             else
@@ -145,6 +145,20 @@ public class JoinsContext
 
 
 
+   /*******************************************************************************
+    ** Add a query join to the list of query joins, and "process it"
+    **
+    ** use this method to add to the list, instead of ever adding directly, as it's
+    ** important do to that process step (and we've had bugs when it wasn't done).
+    *******************************************************************************/
+   private void addQueryJoin(QueryJoin queryJoin) throws QException
+   {
+      this.queryJoins.add(queryJoin);
+      processQueryJoin(queryJoin);
+   }
+
+
+
    /*******************************************************************************
     ** If there are any joins in the context that don't have a join meta data, see
     ** if we can find the JoinMetaData to use for them by looking at the main table's
@@ -236,8 +250,7 @@ public class JoinsContext
                                  QueryJoin queryJoinToAdd = makeQueryJoinFromJoinAndTableNames(nextTable, tmpTable, joinToAdd);
                                  queryJoinToAdd.setType(queryJoin.getType());
                                  addedAnyQueryJoins = true;
-                                 this.queryJoins.add(queryJoinToAdd); // todo something else with aliases?  probably.
+                                 this.addQueryJoin(queryJoinToAdd);
-                                 processQueryJoin(queryJoinToAdd);
                               }
                            }
 
@@ -410,8 +423,7 @@ public class JoinsContext
                QueryJoin queryJoin = makeQueryJoinFromJoinAndTableNames(mainTableName, filterTable, join);
                if(queryJoin != null)
                {
-                  this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+                  this.addQueryJoin(queryJoin);
-                  processQueryJoin(queryJoin);
                   found = true;
                   break;
                }
@@ -420,8 +432,7 @@ public class JoinsContext
             if(!found)
             {
                QueryJoin queryJoin = new QueryJoin().withJoinTable(filterTable).withType(QueryJoin.Type.INNER);
-               this.queryJoins.add(queryJoin); // todo something else with aliases?  probably.
+               this.addQueryJoin(queryJoin);
-               processQueryJoin(queryJoin);
             }
          }
       }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -131,10 +131,10 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
             }
          }
 
-         try
-         {
          Long mark = System.currentTimeMillis();
 
+         try
+         {
             //////////////////////////////////////////////
             // execute the query - iterate over results //
             //////////////////////////////////////////////
@@ -173,6 +173,11 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
             return queryOutput;
          }
+         catch(Exception e)
+         {
+            logSQL(sql, params, mark);
+            throw (e);
+         }
          finally
          {
             if(needToCloseConnection)

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -1416,6 +1416,39 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityFromJoinTableAlsoImplicitlyInQuery() throws QException
+   {
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_ORDER_LINE);
+
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      // orders 1, 2, and 3 are from store 1, so their lines (5 in total) should be found.     //
+      // note, order 2 has the line with mis-matched store id - but, that shouldn't apply here //
+      ///////////////////////////////////////////////////////////////////////////////////////////
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4))));
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 1));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(5);
+
+      ///////////////////////////////////////////////////////////////////
+      // order 4 should be the only one found this time (with 2 lines) //
+      ///////////////////////////////////////////////////////////////////
+      queryInput.setFilter(new QQueryFilter(new QFilterCriteria("order.id", QCriteriaOperator.IN, List.of(1, 2, 3, 4))));
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.TABLE_NAME_STORE, 2));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2);
+
+      ////////////////////////////////////////////////////////////////
+      // make sure we're also good if we explicitly join this table //
+      ////////////////////////////////////////////////////////////////
+      queryInput.withQueryJoin(new QueryJoin().withJoinTable(TestUtils.TABLE_NAME_ORDER).withSelect(true));
+      assertThat(new QueryAction().execute(queryInput).getRecords()).hasSize(2);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-dev-tools/bin/createTableToRecordEntity.groovy

@@ -178,10 +178,12 @@ if(writeTableMetaData)
          .withRecordLabelFields("TODO")
          .withBackendName(TODO)
          .withPrimaryKeyField("id")
+         .withUniqueKey(TODO)
+         .withRecordSecurityLock(TODO)
+         .withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.TODO))
          .withFieldsFromEntity({className}.class)
          .withBackendDetails(new RDBMSTableBackendDetails()
-            .withTableName("{tableName}")
+            .withTableName("{tableName}"))
-         )
          .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id")))
          .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of({dataFieldNames})))
          .withSection(new QFieldSection("dates", new QIcon().withName("calendar_month"), Tier.T3, List.of("createDate", "modifyDate")));