CTLE-421: implemented fieldLevel is hidden, updated to mask password fields

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/GetAction.java

@@ -35,6 +35,7 @@ import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
 import com.kingsrook.qqq.backend.core.actions.interfaces.GetInterface;
 import com.kingsrook.qqq.backend.core.actions.values.QPossibleValueTranslator;
 import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter;
+import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.logging.LogPair;
 import com.kingsrook.qqq.backend.core.logging.QLogger;
@@ -51,6 +52,8 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.cache.CacheUseCase;
 import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
@@ -372,6 +375,53 @@ public class GetAction
          QValueFormatter.setDisplayValuesInRecords(getInput.getTable(), List.of(returnRecord));
       }
 
+      if(getInput.getShouldOmitHiddenFields() || getInput.getShouldMaskPasswords())
+      {
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(record.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldType fieldType = fields.get(fieldName).getType();
+            if(fieldType != null && fieldType.needsMasked())
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               returnRecord.removeValue(fieldName);
+               returnRecord.setValue(fieldName, "************");
+               if(getInput.getShouldGenerateDisplayValues())
+               {
+                  returnRecord.setDisplayValue(fieldName, record.getValueString(fieldName));
+               }
+            }
+         }
+         QValueFormatter.setDisplayValuesInRecords(getInput.getTable(), List.of(returnRecord));
+      }
+
+      //////////////////////////////
+      // mask any password fields //
+      //////////////////////////////
+      Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(record.getTableName()).getFields();
+      for(String fieldName : fields.keySet())
+      {
+         QFieldMetaData field = fields.get(fieldName);
+         if(getInput.getShouldOmitHiddenFields())
+         {
+            if(field.getIsHidden())
+            {
+               returnRecord.removeValue(fieldName);
+            }
+         }
+         else if(getInput.getShouldMaskPasswords())
+         {
+            if(field.getType() != null && field.getType().needsMasked())
+            {
+               returnRecord.setValue(fieldName, "************");
+               returnRecord.setDisplayValue(fieldName, "************");
+            }
+         }
+      }
+
       //////////////////////////////////////////////////////////////////////////////
       // note - shouldFetchHeavyFields should be handled by the underlying action //
       //////////////////////////////////////////////////////////////////////////////

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/QueryAction.java

@@ -27,6 +27,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import com.kingsrook.qqq.backend.core.actions.ActionHelper;
@@ -45,6 +46,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
 import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.Association;
@@ -247,5 +249,59 @@ public class QueryAction
       {
          QValueFormatter.setDisplayValuesInRecords(queryInput.getTable(), records);
       }
+
+      //////////////////////////////
+      // mask any password fields //
+      //////////////////////////////
+      if(queryInput.getShouldOmitHiddenFields() || queryInput.getShouldMaskPasswords())
+      {
+         Set<String> maskedFields = new HashSet<>();
+         Set<String> hiddenFields = new HashSet<>();
+
+         //////////////////////////////////////////////////
+         // build up sets of passwords and hidden fields //
+         //////////////////////////////////////////////////
+         Map<String, QFieldMetaData> fields = QContext.getQInstance().getTable(queryInput.getTableName()).getFields();
+         for(String fieldName : fields.keySet())
+         {
+            QFieldMetaData field = fields.get(fieldName);
+            if(queryInput.getShouldOmitHiddenFields() && field.getIsHidden())
+            {
+               hiddenFields.add(fieldName);
+            }
+            else if(field.getType() != null && field.getType().needsMasked())
+            {
+               maskedFields.add(fieldName);
+            }
+         }
+
+         /////////////////////////////////////////////////////
+         // iterate over records replacing values with mask //
+         /////////////////////////////////////////////////////
+         for(QRecord record : records)
+         {
+            /////////////////////////
+            // clear hidden fields //
+            /////////////////////////
+            for(String hiddenFieldName : hiddenFields)
+            {
+               record.removeValue(hiddenFieldName);
+            }
+
+            for(String maskedFieldName : maskedFields)
+            {
+               //////////////////////////////////////////////////////////////////////
+               // empty out the value completely first (which will remove from     //
+               // display fields as well) then update display value if flag is set //
+               //////////////////////////////////////////////////////////////////////
+               record.removeValue(maskedFieldName);
+               record.setValue(maskedFieldName, "************");
+               if(queryInput.getShouldGenerateDisplayValues())
+               {
+                  record.setDisplayValue(maskedFieldName, record.getValueString(maskedFieldName));
+               }
+            }
+         }
+      }
    }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceEnricher.java

@@ -955,7 +955,8 @@ public class QInstanceEnricher
       {
          for(String fieldName : table.getFields().keySet())
          {
-            if(!usedFieldNames.contains(fieldName))
+            QFieldMetaData field = table.getField(fieldName);
+            if(!field.getIsHidden() && !usedFieldNames.contains(fieldName))
             {
                otherSection.getFieldNames().add(fieldName);
                usedFieldNames.add(fieldName);

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/instances/QInstanceValidator.java

@@ -436,9 +436,16 @@ public class QInstanceValidator
             }
 
             for(String fieldName : CollectionUtils.nonNullMap(table.getFields()).keySet())
+            {
+               if(table.getField(fieldName).getIsHidden())
+               {
+                  assertCondition(!fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is listed in a field section, but it is marked as hidden.");
+               }
+               else
                {
                   assertCondition(fieldNamesInSections.contains(fieldName), "Table " + tableName + " field " + fieldName + " is not listed in any field sections.");
                }
+            }
 
             if(table.getRecordLabelFields() != null && table.getFields() != null)
             {

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/get/GetInput.java

@@ -43,6 +43,8 @@ public class GetInput extends AbstractTableActionInput
    private boolean shouldTranslatePossibleValues = false;
    private boolean shouldGenerateDisplayValues   = false;
    private boolean shouldFetchHeavyFields        = true;
+   private boolean shouldOmitHiddenFields        = true;
+   private boolean shouldMaskPasswords           = true;
 
 
    ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -323,4 +325,66 @@ public class GetInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shouldMaskPasswords
+    *******************************************************************************/
+   public boolean getShouldMaskPasswords()
+   {
+      return (this.shouldMaskPasswords);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldMaskPasswords
+    *******************************************************************************/
+   public void setShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldMaskPasswords
+    *******************************************************************************/
+   public GetInput withShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public boolean getShouldOmitHiddenFields()
+   {
+      return (this.shouldOmitHiddenFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public void setShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public GetInput withShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/query/QueryInput.java

@@ -47,6 +47,8 @@ public class QueryInput extends AbstractTableActionInput
    private boolean shouldTranslatePossibleValues = false;
    private boolean shouldGenerateDisplayValues   = false;
    private boolean shouldFetchHeavyFields        = false;
+   private boolean shouldOmitHiddenFields        = true;
+   private boolean shouldMaskPasswords           = true;
 
    /////////////////////////////////////////////////////////////////////////////////////////
    // this field - only applies if shouldTranslatePossibleValues is true.                 //
@@ -497,4 +499,66 @@ public class QueryInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for shouldMaskPasswords
+    *******************************************************************************/
+   public boolean getShouldMaskPasswords()
+   {
+      return (this.shouldMaskPasswords);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldMaskPasswords
+    *******************************************************************************/
+   public void setShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldMaskPasswords
+    *******************************************************************************/
+   public QueryInput withShouldMaskPasswords(boolean shouldMaskPasswords)
+   {
+      this.shouldMaskPasswords = shouldMaskPasswords;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public boolean getShouldOmitHiddenFields()
+   {
+      return (this.shouldOmitHiddenFields);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public void setShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for shouldOmitHiddenFields
+    *******************************************************************************/
+   public QueryInput withShouldOmitHiddenFields(boolean shouldOmitHiddenFields)
+   {
+      this.shouldOmitHiddenFields = shouldOmitHiddenFields;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QField.java

@@ -58,6 +58,11 @@ public @interface QField
     *******************************************************************************/
    boolean isEditable() default true;
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   boolean isHidden() default false;
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/data/QRecord.java

@@ -196,6 +196,17 @@ public class QRecord implements Serializable
 
 
 
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public void removeValue(String fieldName)
+   {
+      values.remove(fieldName);
+      displayValues.remove(fieldName);
+   }
+
+
+
    /*******************************************************************************
     **
     *******************************************************************************/

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldMetaData.java

@@ -54,6 +54,7 @@ public class QFieldMetaData implements Cloneable
    private QFieldType type;
    private boolean    isRequired = false;
    private boolean    isEditable = true;
+   private boolean    isHidden   = false;
    private boolean    isHeavy    = false;
 
    private FieldSecurityLock fieldSecurityLock;
@@ -183,6 +184,7 @@ public class QFieldMetaData implements Cloneable
             QField fieldAnnotation = optionalFieldAnnotation.get();
             setIsRequired(fieldAnnotation.isRequired());
             setIsEditable(fieldAnnotation.isEditable());
+            setIsHidden(fieldAnnotation.isHidden());
 
             if(StringUtils.hasContent(fieldAnnotation.label()))
             {
@@ -851,4 +853,36 @@ public class QFieldMetaData implements Cloneable
       this.middlewareMetaData.put(middlewareMetaData.getType(), middlewareMetaData);
       return (this);
    }
+
+
+
+   /*******************************************************************************
+    ** Getter for isHidden
+    *******************************************************************************/
+   public boolean getIsHidden()
+   {
+      return (this.isHidden);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for isHidden
+    *******************************************************************************/
+   public void setIsHidden(boolean isHidden)
+   {
+      this.isHidden = isHidden;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for isHidden
+    *******************************************************************************/
+   public QFieldMetaData withIsHidden(boolean isHidden)
+   {
+      this.isHidden = isHidden;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/metadata/fields/QFieldType.java

@@ -108,4 +108,14 @@ public enum QFieldType
    {
       return this == QFieldType.INTEGER || this == QFieldType.DECIMAL;
    }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public boolean needsMasked()
+   {
+      return this == QFieldType.PASSWORD;
+   }
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/modules/backend/implementations/mock/MockQueryAction.java

@@ -67,7 +67,7 @@ public class MockQueryAction implements QueryInterface
 
             for(String field : table.getFields().keySet())
             {
-               Serializable value = field.equals("id") ? (i + 1) : getValue(table, field);
+               Serializable value = field.equals("id") ? (i + 1) : getMockValue(table, field);
                record.setValue(field, value);
             }
 
@@ -95,7 +95,7 @@ public class MockQueryAction implements QueryInterface
     **
     *******************************************************************************/
    @SuppressWarnings("checkstyle:MagicNumber")
-   private Serializable getValue(QTableMetaData table, String field)
+   public static Serializable getMockValue(QTableMetaData table, String field)
    {
       // @formatter:off // IJ can't do new-style switch correctly yet...
       return switch(table.getField(field).getType())

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/actions/tables/QueryActionTest.java

@@ -22,6 +22,7 @@
 package com.kingsrook.qqq.backend.core.actions.tables;
 
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import com.kingsrook.qqq.backend.core.BaseTest;
@@ -33,6 +34,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.mock.MockQueryAction;
 import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
 import com.kingsrook.qqq.backend.core.utils.TestUtils;
 import org.junit.jupiter.api.Test;
@@ -73,17 +75,40 @@ class QueryActionTest extends BaseTest
          // this SHOULD be empty, based on the default for the should //
          ///////////////////////////////////////////////////////////////
          assertThat(record.getDisplayValues()).isEmpty();
+
+         //////////////////////////////////////////
+         // hidden field should not be in record //
+         //////////////////////////////////////////
+         assertThat(record.getValue("superSecret")).isNull();
+
+         /////////////////////////////////////
+         // password field should be masked //
+         /////////////////////////////////////
+         assertThat(record.getValueString("ssn")).contains("****");
       }
 
-      ////////////////////////////////////
+      ////////////////////////////////////////////////////////
-      // now flip that field and re-run //
+      // now flip some fields, re-run, and validate results //
-      ////////////////////////////////////
+      ////////////////////////////////////////////////////////
       queryInput.setShouldGenerateDisplayValues(true);
+      queryInput.setShouldMaskPasswords(false);
+      queryInput.setShouldOmitHiddenFields(false);
       assertThat(queryOutput.getRecords()).isNotEmpty();
       queryOutput = new QueryAction().execute(queryInput);
       for(QRecord record : queryOutput.getRecords())
       {
          assertThat(record.getDisplayValues()).isNotEmpty();
+
+         //////////////////////////////////////////
+         // hidden field should now be in record //
+         //////////////////////////////////////////
+         assertThat(record.getValue("superSecret")).isNotNull();
+
+         /////////////////////////////////////
+         // password field should be masked //
+         /////////////////////////////////////
+         Serializable mockValue = MockQueryAction.getMockValue(QContext.getQInstance().getTable("person"), "ssn");
+         assertThat(record.getValueString("ssn")).isEqualTo(mockValue);
       }
    }
 

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/utils/TestUtils.java

@@ -528,7 +528,8 @@ public class TestUtils
          .withField(new QFieldMetaData("noOfShoes", QFieldType.INTEGER).withDisplayFormat(DisplayFormat.COMMAS))
          .withField(new QFieldMetaData("cost", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
          .withField(new QFieldMetaData("price", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY))
-         ;
+         .withField(new QFieldMetaData("ssn", QFieldType.STRING).withType(QFieldType.PASSWORD))
+         .withField(new QFieldMetaData("superSecret", QFieldType.STRING).withType(QFieldType.PASSWORD).withIsHidden(true));
    }