QRunIO/qqq
2
0
Fork 0
mirror of https://github.com/Kingsrook/qqq.git synced 2025-07-18 13:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

added ability to log sql to system out, added handling for when joins happen and the key field is on the many side

Browse Source
This commit is contained in:
Tim Chamberlain
2023-03-08 12:31:36 -06:00
parent c091440848
commit d7abab2fd1
11 changed files with 190 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/AbstractRDBMSAction.java
View File

@ -37,9 +37,10 @@ import java.util.stream.Collectors;
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.interfaces.QActionInterface;
import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QValueException;
import com.kingsrook.qqq.backend.core.logging.LogPair;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
@ -54,9 +55,11 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
@ -946,17 +949,34 @@ public abstract class AbstractRDBMSAction implements QActionInterface
/*******************************************************************************
**
*******************************************************************************/
protected void logSQL(CharSequence sql, List<?> params)
protected void logSQL(CharSequence sql, List<?> params, Long mark)
{
if(System.getProperty("qqq.rdbms.logSQL", "false").equals("true"))
{
try
{
LogPair paramsLogPair = params == null ? null :
params.size() <= 100 ? logPair("params", params) :
logPair("first100Params", params.subList(0, 99));
params = params.size() <= 100 ? params : params.subList(0, 99);
LOG.debug("Running SQL", logPair("sql", sql), paramsLogPair);
LOG.debug("PARAMS: " + params);
if(System.getProperty("qqq.rdbms.logSQL.output", "logger").equals("system.out"))
{
System.out.println("SQL: " + sql);
System.out.println("PARAMS: " + params);
if(mark != null)
{
System.out.println("SQL Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - mark)) + "] ms");
}
}
else
{
LOG.debug("Running SQL", logPair("sql", sql), logPair("params", params));
if(mark != null)
{
LOG.debug("SQL Took [" + QValueFormatter.formatValue(DisplayFormat.COMMAS, (System.currentTimeMillis() - mark)) + "] ms");
}
}
}
catch(Exception e)
{
@ -965,4 +985,35 @@ public abstract class AbstractRDBMSAction implements QActionInterface
}
}
/*******************************************************************************
** method that looks at security lock joins, and if a one-to-many is found where
** the specified field name is on the 'right side' of the join, then a distinct
** needs added to select clause
*******************************************************************************/
protected boolean doesSelectClauseRequireDistinct(QTableMetaData table)
{
if(table != null)
{
for(RecordSecurityLock recordSecurityLock : CollectionUtils.nonNullList(table.getRecordSecurityLocks()))
{
for(String joinName : CollectionUtils.nonNullList(recordSecurityLock.getJoinNameChain()))
{
QJoinMetaData joinMetaData = QContext.getQInstance().getJoin(joinName);
if(JoinType.ONE_TO_MANY.equals(joinMetaData.getType()) && !joinMetaData.getRightTable().equals(table.getName()))
{
return (true);
}
else if(JoinType.MANY_TO_ONE.equals(joinMetaData.getType()) && !joinMetaData.getLeftTable().equals(table.getName()))
{
return (true);
}
}
}
}
return (false);
}
}

6
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSAggregateAction.java
View File

@ -87,12 +87,12 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
// todo sql customization - can edit sql and/or param list
logSQL(sql, params);
AggregateOutput rs = new AggregateOutput();
List<AggregateResult> results = new ArrayList<>();
rs.setResults(results);
Long mark = System.currentTimeMillis();
try(Connection connection = getConnection(aggregateInput))
{
QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
@ -127,6 +127,8 @@ public class RDBMSAggregateAction extends AbstractRDBMSAction implements Aggrega
}), params);
}
logSQL(sql, params, mark);
return rs;
}
catch(Exception e)

14
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountAction.java
View File

@ -57,8 +57,13 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
QTableMetaData table = countInput.getTable();
JoinsContext joinsContext = new JoinsContext(countInput.getInstance(), countInput.getTableName(), countInput.getQueryJoins(), countInput.getFilter());
JoinsContext.FieldAndTableNameOrAlias fieldAndTableNameOrAlias = joinsContext.getFieldAndTableNameOrAlias(table.getPrimaryKeyField());
String sql = "SELECT count(*) as record_count FROM "
boolean requiresDistinct = doesSelectClauseRequireDistinct(table);
String primaryKeyColumn = escapeIdentifier(fieldAndTableNameOrAlias.tableNameOrAlias()) + "." + escapeIdentifier(fieldAndTableNameOrAlias.field().getName());
String clausePrefix = (requiresDistinct) ? "SELECT COUNT (DISTINCT " + primaryKeyColumn + ")" : "SELECT COUNT(*)";
String sql = clausePrefix + " AS record_count FROM "
+ makeFromClause(countInput.getInstance(), table.getName(), joinsContext);
QQueryFilter filter = countInput.getFilter();
@ -66,12 +71,11 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
sql += " WHERE " + makeWhereClause(countInput.getInstance(), countInput.getSession(), table, joinsContext, filter, params);
// todo sql customization - can edit sql and/or param list
logSQL(sql, params);
CountOutput rs = new CountOutput();
try(Connection connection = getConnection(countInput))
{
long mark = System.currentTimeMillis();
QueryManager.executeStatement(connection, sql, ((ResultSet resultSet) ->
{
if(resultSet.next())
@ -80,6 +84,8 @@ public class RDBMSCountAction extends AbstractRDBMSAction implements CountInterf
}
}), params);
logSQL(sql, params, mark);
}
return rs;

15
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSDeleteAction.java
View File

@ -117,6 +117,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
// have been converted to a list of primary keys in the deleteInput). so, proceed now by deleting a list of pkeys. //
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
deleteList(connection, deleteInput, deleteOutput);
return (deleteOutput);
}
finally
@ -190,7 +191,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
+ " WHERE "
+ escapeIdentifier(primaryKeyName) + " = ?";
logSQL(sql, List.of(primaryKey));
Long mark = System.currentTimeMillis();
try
{
@ -211,6 +212,7 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
// LOG.debug("rowCount 0 trying to delete [" + tableName + "][" + primaryKey + "]");
// deleteOutput.addRecordWithError(new QRecord(table, primaryKey).withError("Record was not deleted (but no error was given from the database)"));
// }
logSQL(sql, List.of(primaryKey), mark);
}
catch(Exception e)
{
@ -228,6 +230,8 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
{
try
{
long mark = System.currentTimeMillis();
String tableName = getTableName(table);
String primaryKeyName = getColumnName(table.getField(table.getPrimaryKeyField()));
String sql = "DELETE FROM "
@ -239,10 +243,11 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
+ ")";
// todo sql customization - can edit sql and/or param list
logSQL(sql, primaryKeys);
Integer rowCount = QueryManager.executeUpdateForRowCount(connection, sql, primaryKeys);
deleteOutput.addToDeletedRecordCount(rowCount);
logSQL(sql, primaryKeys, mark);
}
catch(Exception e)
{
@ -270,13 +275,15 @@ public class RDBMSDeleteAction extends AbstractRDBMSAction implements DeleteInte
+ escapeIdentifier(tableName) + " AS " + escapeIdentifier(table.getName())
+ " WHERE "
+ whereClause;
logSQL(sql, params);
Long mark = System.currentTimeMillis();
try
{
int rowCount = QueryManager.executeUpdateForRowCount(connection, sql, params);
deleteOutput.setDeletedRecordCount(rowCount);
logSQL(sql, params, mark);
}
catch(Exception e)
{

4
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSInsertAction.java
View File

@ -153,7 +153,7 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
continue;
}
logSQL(sql, params);
Long mark = System.currentTimeMillis();
///////////////////////////////////////////////////////////
// execute the insert, then foreach record in the input, //
@ -175,6 +175,8 @@ public class RDBMSInsertAction extends AbstractRDBMSAction implements InsertInte
outputRecord.setValue(table.getPrimaryKeyField(), id);
}
}
logSQL(sql, params, mark);
}
}
finally

14
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryAction.java
View File

@ -68,7 +68,7 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
QTableMetaData table = queryInput.getTable();
String tableName = queryInput.getTableName();
StringBuilder sql = new StringBuilder("SELECT ").append(makeSelectClause(queryInput));
StringBuilder sql = new StringBuilder(makeSelectClause(queryInput));
JoinsContext joinsContext = new JoinsContext(queryInput.getInstance(), tableName, queryInput.getQueryJoins(), queryInput.getFilter());
sql.append(" FROM ").append(makeFromClause(queryInput.getInstance(), tableName, joinsContext));
@ -133,11 +133,12 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
try
{
Long mark = System.currentTimeMillis();
//////////////////////////////////////////////
// execute the query - iterate over results //
//////////////////////////////////////////////
QueryOutput queryOutput = new QueryOutput(queryInput);
logSQL(sql, params);
PreparedStatement statement = createStatement(connection, sql.toString(), queryInput);
QueryManager.executeStatement(statement, ((ResultSet resultSet) ->
@ -168,6 +169,8 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
}), params);
logSQL(sql, params, mark);
return queryOutput;
}
finally
@ -195,14 +198,17 @@ public class RDBMSQueryAction extends AbstractRDBMSAction implements QueryInterf
QInstance instance = queryInput.getInstance();
String tableName = queryInput.getTableName();
List<QueryJoin> queryJoins = queryInput.getQueryJoins();
QTableMetaData table = instance.getTable(tableName);
boolean requiresDistinct = doesSelectClauseRequireDistinct(table);
String clausePrefix = (requiresDistinct) ? "SELECT DISTINCT " : "SELECT ";
List<QFieldMetaData> fieldList = new ArrayList<>(table.getFields().values());
String columns = fieldList.stream()
.filter(field -> filterOutHeavyFieldsIfNeeded(field, queryInput.getShouldFetchHeavyFields()))
.map(field -> escapeIdentifier(tableName) + "." + escapeIdentifier(getColumnName(field)))
.collect(Collectors.joining(", "));
StringBuilder rs = new StringBuilder(columns);
StringBuilder rs = new StringBuilder(clausePrefix).append(columns);
for(QueryJoin queryJoin : CollectionUtils.nonNullList(queryJoins))
{

8
qqq-backend-module-rdbms/src/main/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSUpdateAction.java
View File

@ -204,13 +204,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
rowValues.add(record.getValue(table.getPrimaryKeyField()));
}
logSQL(sql, values);
Long mark = System.currentTimeMillis();
////////////////////////////////////////////////////////////////////////////////
// let query manager do the batch updates - note that it will internally page //
////////////////////////////////////////////////////////////////////////////////
QueryManager.executeBatchUpdate(connection, sql, values);
incrementStatus(updateInput, recordList.size());
logSQL(sql, values, mark);
}
@ -263,13 +265,15 @@ public class RDBMSUpdateAction extends AbstractRDBMSAction implements UpdateInte
params.add(record.getValue(table.getPrimaryKeyField()));
}
logSQL(sql, params);
Long mark = System.currentTimeMillis();
/////////////////////////////////////
// let query manager do the update //
/////////////////////////////////////
QueryManager.executeUpdate(connection, sql, params);
incrementStatus(updateInput, page.size());
logSQL(sql, params, mark);
}
}

24
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/TestUtils.java
View File

@ -61,6 +61,8 @@ public class TestUtils
public static final String TABLE_NAME_ORDER = "order";
public static final String TABLE_NAME_ITEM = "item";
public static final String TABLE_NAME_ORDER_LINE = "orderLine";
public static final String TABLE_NAME_WAREHOUSE = "warehouse";
public static final String TABLE_NAME_WAREHOUSE_STORE_INT = "warehouseStoreInt";
public static final String SECURITY_KEY_STORE_ALL_ACCESS = "storeAllAccess";
@ -248,6 +250,28 @@ public class TestUtils
.withField(new QFieldMetaData("quantity", QFieldType.INTEGER))
);
qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE_STORE_INT, "warehouse_store_int")
.withField(new QFieldMetaData("warehouseId", QFieldType.INTEGER).withBackendName("warehouse_id"))
.withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withBackendName("store_id"))
);
qInstance.addTable(defineBaseTable(TABLE_NAME_WAREHOUSE, "warehouse")
.withRecordSecurityLock(new RecordSecurityLock()
.withSecurityKeyType(TABLE_NAME_STORE)
.withFieldName(TABLE_NAME_WAREHOUSE_STORE_INT + ".storeId")
.withJoinNameChain(List.of(QJoinMetaData.makeInferredJoinName(TestUtils.TABLE_NAME_WAREHOUSE, TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)))
)
.withField(new QFieldMetaData("name", QFieldType.STRING).withBackendName("name"))
);
qInstance.addJoin(new QJoinMetaData()
.withType(JoinType.ONE_TO_MANY)
.withLeftTable(TestUtils.TABLE_NAME_WAREHOUSE)
.withRightTable(TestUtils.TABLE_NAME_WAREHOUSE_STORE_INT)
.withInferredName()
.withJoinOn(new JoinOn("id", "warehouseId"))
);
qInstance.addJoin(new QJoinMetaData()
.withName("orderJoinStore")
.withLeftTable(TABLE_NAME_ORDER)

15
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSCountActionTest.java
View File

@ -200,4 +200,19 @@ public class RDBMSCountActionTest extends RDBMSActionTest
assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(5);
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
CountInput countInput = new CountInput();
countInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
assertThat(new CountAction().execute(countInput).getCount()).isEqualTo(1);
}
}

16
qqq-backend-module-rdbms/src/test/java/com/kingsrook/qqq/backend/module/rdbms/actions/RDBMSQueryActionTest.java
View File

@ -1381,4 +1381,20 @@ public class RDBMSQueryActionTest extends RDBMSActionTest
.allMatch(r -> r.getValueInteger("storeId").equals(1));
}
/*******************************************************************************
**
*******************************************************************************/
@Test
void testRecordSecurityWithLockFromJoinTableWhereTheKeyIsOnTheManySide() throws QException
{
QContext.setQSession(new QSession().withSecurityKeyValue(TestUtils.SECURITY_KEY_STORE_ALL_ACCESS, true));
QueryInput queryInput = new QueryInput();
queryInput.setTableName(TestUtils.TABLE_NAME_WAREHOUSE);
assertThat(new QueryAction().execute(queryInput).getRecords())
.hasSize(1);
}
}

25
qqq-backend-module-rdbms/src/test/resources/prime-test-database.sql
View File

@ -82,7 +82,9 @@ INSERT INTO carrier (id, name, company_code, service_level) VALUES (11, 'GSO', '
DROP TABLE IF EXISTS order_line;
DROP TABLE IF EXISTS item;
DROP TABLE IF EXISTS `order`;
DROP TABLE IF EXISTS warehouse_store_int;
DROP TABLE IF EXISTS store;
DROP TABLE IF EXISTS warehouse;
CREATE TABLE store
(
@ -152,3 +154,26 @@ INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (5, 'QRU-1', 2
INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (6, 'QD-1', 3, 1);
INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (7, 'QD-1', 3, 2);
INSERT INTO order_line (order_id, sku, store_id, quantity) VALUES (8, 'QD-1', 3, 3);
CREATE TABLE warehouse
(
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(80)
);
INSERT INTO warehouse (name) VALUES ('Patterson');
INSERT INTO warehouse (name) VALUES ('Edison');
INSERT INTO warehouse (name) VALUES ('Stockton');
INSERT INTO warehouse (name) VALUES ('Somewhere in Texas');
CREATE TABLE warehouse_store_int
(
id INT AUTO_INCREMENT PRIMARY KEY,
warehouse_id INT REFERENCES `warehouse`,
store_id INT REFERENCES `store`
);
INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 1);
INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 2);
INSERT INTO warehouse_store_int (warehouse_id, store_id) VALUES (1, 3);