Merge pull request #12 from Kingsrook/CTLE-346-add-client-warehouse-int

added ability to log sql to system out, added handling for when joins…

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java

@@ -37,9 +37,10 @@ import java.util.stream.Collectors;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
 import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
 import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface;
+import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.exceptions.QValueException;
-import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
@@ -54,9 +55,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
 import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@@ -953,17 +956,33 @@ public abstract class AbstractRDBMSAction implements QActionInterface
    /*******************************************************************************
     **
     *******************************************************************************/
-   protected void logSQL(CharSequence sql, List<?> params)
+   protected void logSQL(CharSequence sql, List<?> params, Long mark)
    {
       if(System.getProperty("qqq.rdbms.logSQL", "false").equals("true"))
       {
          try
          {
-            LogPair paramsLogPair = params == null ? null :
+            params = params.size() <= 100 ? params : params.subList(0, 99);
-               params.size() <= 100 ? logPair("params", params) :
-                  logPair("first100Params", params.subList(0, 99));
 
-            LOG.debug("Running SQL", logPair("sql", sql), paramsLogPair);
+            if(System.getProperty("qqq.rdbms.logSQL.output", "logger").equalsIgnoreCase("system.out"))
+            {
+               System.out.println("SQL: " + sql);
+               System.out.println("PARAMS: " + params);
+
+               if(mark != null)
+               {
+                  System.out.println("SQL Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - mark)) + "] ms");
+               }
+            }
+            else
+            {
+               LOG.debug("Running SQL", logPair("sql", sql), logPair("params", params));
+
+               if(mark != null)
+               {
+                  LOG.debug("SQL Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - mark)) + "] ms");
+               }
+            }
          }
          catch(Exception e)
          {
@@ -972,4 +991,35 @@ public abstract class AbstractRDBMSAction implements QActionInterface
       }
    }
 
+
+
+   /*******************************************************************************
+    ** method that looks at security lock joins, and if a one-to-many is found where
+    ** the specified field name is on the 'right side' of the join, then a distinct
+    ** needs added to select clause
+    *******************************************************************************/
+   protected boolean doesSelectClauseRequireDistinct(QTableMetaData table)
+   {
+      if(table != null)
+      {
+         for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
+         {
+            for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
+            {
+               QJoinMetaData joinMetaData = QContext.getQInstance().getJoin(joinName);
+               if(JoinType.ONE_TO_MANY.equals(joinMetaData.getType()) && !joinMetaData.getRightTable().equals(table.getName()))
+               {
+                  return (true);
+               }
+               else if(JoinType.MANY_TO_ONE.equals(joinMetaData.getType()) && !joinMetaData.getLeftTable().equals(table.getName()))
+               {
+                  return (true);
+               }
+            }
+         }
+      }
+
+      return (false);
+   }
+
 }

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java

@@ -87,12 +87,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
 
          // todo sql customization - can edit sql and/or param list
 
-         logSQL(sql, params);
-
          AggregateOutput       rs      = new AggregateOutput();
          List<AggregateResult> results = new ArrayList<>();
          rs.setResults(results);
 
+         Long mark = System.currentTimeMillis();
+
          try(Connection connection = getConnection(aggregateInput))
          {
             QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
@@ -127,6 +127,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
             }), params);
          }
 
+         logSQL(sql, params, mark);
+
          return rs;
       }
       catch(Exception e)

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java

@@ -56,9 +56,14 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
       {
          QTableMetaData table = countInput.getTable();
 
-         JoinsContext joinsContext = new JoinsContext(countInput.getInstance(), countInput.getTableName(), countInput.getQueryJoins(), countInput.getFilter());
+         JoinsContext                          joinsContext             = new JoinsContext(countInput.getInstance(), countInput.getTableName(), countInput.getQueryJoins(), countInput.getFilter());
+         JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(table.getPrimaryKeyField());
 
-         String sql = "SELECT count(*) as record_count FROM "
+         boolean requiresDistinct = doesSelectClauseRequireDistinct(table);
+         String  primaryKeyColumn = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(fieldAndTableNameOrAlias.field().getName());
+         String  clausePrefix     = (requiresDistinct) ? "SELECT COUNT (DISTINCT " + primaryKeyColumn + ")" : "SELECT COUNT(*)";
+
+         String sql = clausePrefix + " AS record_count FROM "
             + makeFromClause(countInput.getInstance(), table.getName(), joinsContext);
 
          QQueryFilter       filter = countInput.getFilter();
@@ -66,12 +71,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
          sql += " WHERE " + makeWhereClause(countInput.getInstance(), countInput.getSession(), table, joinsContext, filter, params);
          // todo sql customization - can edit sql and/or param list
 
-         logSQL(sql, params);
-
          CountOutput rs = new CountOutput();
-
          try(Connection connection = getConnection(countInput))
          {
+            long mark = System.currentTimeMillis();
+
             QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
             {
                if(resultSet.next())
@@ -80,6 +84,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
                }
 
             }), params);
+
+            logSQL(sql, params, mark);
          }
 
          return rs;

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java

@@ -117,6 +117,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
             // have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys.        //
             ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             deleteList(connection, deleteInput, deleteOutput);
+
             return (deleteOutput);
          }
          finally
@@ -190,7 +191,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
          + " WHERE "
          + escapeIdentifier(primaryKeyName) + " = ?";
 
-      logSQL(sql, List.of(primaryKey));
+      Long mark = System.currentTimeMillis();
 
       try
       {
@@ -211,6 +212,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
          //    LOG.debug("rowCount 0 trying to delete [" + tableName + "][" + primaryKey + "]");
          //    deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError("Record was not deleted (but no error was given from the database)"));
          // }
+         logSQL(sql, List.of(primaryKey), mark);
       }
       catch(Exception e)
       {
@@ -228,6 +230,8 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
    {
       try
       {
+         long mark = System.currentTimeMillis();
+
          String tableName      = getTableName(table);
          String primaryKeyName = getColumnName(table.getField(table.getPrimaryKeyField()));
          String sql = "DELETE FROM "
@@ -239,10 +243,11 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
             + ")";
 
          // todo sql customization - can edit sql and/or param list
-         logSQL(sql, primaryKeys);
 
          Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys);
          deleteOutput.addToDeletedRecordCount(rowCount);
+
+         logSQL(sql, primaryKeys, mark);
       }
       catch(Exception e)
       {
@@ -270,13 +275,15 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
          + escapeIdentifier(tableName) + " AS " + escapeIdentifier(table.getName())
          + " WHERE "
          + whereClause;
-      logSQL(sql, params);
+
+      Long mark = System.currentTimeMillis();
 
       try
       {
          int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params);
-
          deleteOutput.setDeletedRecordCount(rowCount);
+
+         logSQL(sql, params, mark);
       }
       catch(Exception e)
       {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java

@@ -153,7 +153,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
                   continue;
                }
 
-               logSQL(sql, params);
+               Long mark = System.currentTimeMillis();
 
                ///////////////////////////////////////////////////////////
                // execute the insert, then foreach record in the input, //
@@ -175,6 +175,8 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
                      outputRecord.setValue(table.getPrimaryKeyField(), id);
                   }
                }
+
+               logSQL(sql, params, mark);
             }
          }
          finally

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java

@@ -68,7 +68,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
          QTableMetaData table     = queryInput.getTable();
          String         tableName = queryInput.getTableName();
 
-         StringBuilder sql = new StringBuilder("SELECT ").append(makeSelectClause(queryInput));
+         StringBuilder sql = new StringBuilder(makeSelectClause(queryInput));
 
          JoinsContext joinsContext = new JoinsContext(queryInput.getInstance(), tableName, queryInput.getQueryJoins(), queryInput.getFilter());
          sql.append(" FROM ").append(makeFromClause(queryInput.getInstance(), tableName, joinsContext));
@@ -133,11 +133,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
          try
          {
+            Long mark = System.currentTimeMillis();
+
             //////////////////////////////////////////////
             // execute the query - iterate over results //
             //////////////////////////////////////////////
             QueryOutput queryOutput = new QueryOutput(queryInput);
-            logSQL(sql, params);
 
             PreparedStatement statement = createStatement(connection, sql.toString(), queryInput);
             QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
@@ -168,6 +169,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
 
             }), params);
 
+            logSQL(sql, params, mark);
+
             return queryOutput;
          }
          finally
@@ -195,14 +198,17 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
       QInstance       instance   = queryInput.getInstance();
       String          tableName  = queryInput.getTableName();
       List<QueryJoin> queryJoins = queryInput.getQueryJoins();
+      QTableMetaData  table      = instance.getTable(tableName);
+
+      boolean requiresDistinct = doesSelectClauseRequireDistinct(table);
+      String  clausePrefix     = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT ";
 
-      QTableMetaData       table     = instance.getTable(tableName);
       List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
       String columns = fieldList.stream()
          .filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
          .map(field -> escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field)))
          .collect(Collectors.joining(", "));
-      StringBuilder rs = new StringBuilder(columns);
+      StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
 
       for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins))
       {

qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java

@@ -204,13 +204,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
          rowValues.add(record.getValue(table.getPrimaryKeyField()));
       }
 
-      logSQL(sql, values);
+      Long mark = System.currentTimeMillis();
 
       ////////////////////////////////////////////////////////////////////////////////
       // let query manager do the batch updates - note that it will internally page //
       ////////////////////////////////////////////////////////////////////////////////
       QueryManager.executeBatchUpdate(connection, sql, values);
       incrementStatus(updateInput, recordList.size());
+
+      logSQL(sql, values, mark);
    }
 
 
@@ -263,13 +265,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
             params.add(record.getValue(table.getPrimaryKeyField()));
          }
 
-         logSQL(sql, params);
+         Long mark = System.currentTimeMillis();
 
          /////////////////////////////////////
          // let query manager do the update //
          /////////////////////////////////////
          QueryManager.executeUpdate(connection, sql, params);
          incrementStatus(updateInput, page.size());
+
+         logSQL(sql, params, mark);
       }
    }
 

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java

@@ -55,12 +55,14 @@ public class TestUtils
 {
    public static final String DEFAULT_BACKEND_NAME = "default";
 
-   public static final String TABLE_NAME_PERSON           = "personTable";
+   public static final String TABLE_NAME_PERSON              = "personTable";
-   public static final String TABLE_NAME_PERSONAL_ID_CARD = "personalIdCard";
+   public static final String TABLE_NAME_PERSONAL_ID_CARD    = "personalIdCard";
-   public static final String TABLE_NAME_STORE            = "store";
+   public static final String TABLE_NAME_STORE               = "store";
-   public static final String TABLE_NAME_ORDER            = "order";
+   public static final String TABLE_NAME_ORDER               = "order";
-   public static final String TABLE_NAME_ITEM             = "item";
+   public static final String TABLE_NAME_ITEM                = "item";
-   public static final String TABLE_NAME_ORDER_LINE       = "orderLine";
+   public static final String TABLE_NAME_ORDER_LINE          = "orderLine";
+   public static final String TABLE_NAME_WAREHOUSE           = "warehouse";
+   public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt";
 
    public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess";
 
@@ -248,6 +250,28 @@ public class TestUtils
          .withField(new QFieldMetaData("quantity", QFieldType.INTEGER))
       );
 
+      qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int")
+         .withField(new QFieldMetaData("warehouseId", QFieldType.INTEGER).withBackendName("warehouse_id"))
+         .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id"))
+      );
+
+      qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse")
+         .withRecordSecurityLock(new RecordSecurityLock()
+            .withSecurityKeyType(TABLE_NAME_STORE)
+            .withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeId")
+            .withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)))
+         )
+         .withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name"))
+      );
+
+      qInstance.addJoin(new QJoinMetaData()
+         .withType(JoinType.ONE_TO_MANY)
+         .withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE)
+         .withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)
+         .withInferredName()
+         .withJoinOn(new JoinOn("id", "warehouseId"))
+      );
+
       qInstance.addJoin(new QJoinMetaData()
          .withName("orderJoinStore")
          .withLeftTable(TABLE_NAME_ORDER)

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java

@@ -200,4 +200,19 @@ public class RDBMSCountActionTest extends RDBMSActionTest
       assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(5);
    }
 
-}
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException
+   {
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      CountInput countInput = new CountInput();
+      countInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
+
+      assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(1);
+   }
+
+}

qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java

@@ -1383,4 +1383,20 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
          .allMatch(r -> r.getValueInteger("storeId").equals(1));
    }
 
-}
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Test
+   void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException
+   {
+      QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
+      QueryInput queryInput = new QueryInput();
+      queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
+
+      assertThat(new QueryAction().execute(queryInput).getRecords())
+         .hasSize(1);
+   }
+
+}

qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql

@@ -82,7 +82,9 @@ INSERT INTO carrier (id, name, company_code, service_level) VALUES (11, 'GSO', '
 DROP TABLE IF EXISTS order_line;
 DROP TABLE IF EXISTS item;
 DROP TABLE IF EXISTS `order`;
+DROP TABLE IF EXISTS warehouse_store_int;
 DROP TABLE IF EXISTS store;
+DROP TABLE IF EXISTS warehouse;
 
 CREATE TABLE store
 (
@@ -152,3 +154,26 @@ INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (5, 'QRU-1', 2
 INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (6, 'QD-1', 3, 1);
 INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (7, 'QD-1', 3, 2);
 INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (8, 'QD-1', 3, 3);
+
+
+CREATE TABLE warehouse
+(
+   id INT AUTO_INCREMENT PRIMARY KEY,
+   name VARCHAR(80)
+);
+
+INSERT INTO warehouse (name) VALUES ('Patterson');
+INSERT INTO warehouse (name) VALUES ('Edison');
+INSERT INTO warehouse (name) VALUES ('Stockton');
+INSERT INTO warehouse (name) VALUES ('Somewhere in Texas');
+
+CREATE TABLE warehouse_store_int
+(
+   id INT AUTO_INCREMENT PRIMARY KEY,
+   warehouse_id INT REFERENCES `warehouse`,
+   store_id INT REFERENCES `store`
+);
+
+INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 1);
+INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 2);
+INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 3);