CE-1772: fix aws expecting content type if object metadata is given

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/values/QValueFormatter.java

@@ -28,7 +28,6 @@ import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -468,7 +467,8 @@ public class QValueFormatter
    {
       for(QFieldMetaData field : table.getFields().values())
       {
-         if(field.getType().equals(QFieldType.BLOB))
+         Optional<FieldAdornment> fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
+         if(fileDownloadAdornment.isPresent())
          {
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
             // file name comes from:                                                                                            //
@@ -478,20 +478,7 @@ public class QValueFormatter
             // - tableLabel primaryKey fieldLabel                                                                               //
             // - and - if the FILE_DOWNLOAD adornment had a DEFAULT_EXTENSION, then it gets added (preceded by a dot)           //
             //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-            Optional<FieldAdornment>  fileDownloadAdornment = field.getAdornment(AdornmentType.FILE_DOWNLOAD);
-            Map<String, Serializable> adornmentValues       = Collections.emptyMap();
-
-            if(fileDownloadAdornment.isPresent())
-            {
-               adornmentValues = fileDownloadAdornment.get().getValues();
-            }
-            else
-            {
-               ///////////////////////////////////////////////////////
-               // don't change blobs unless they are file-downloads //
-               ///////////////////////////////////////////////////////
-               continue;
-            }
+            Map<String, Serializable> adornmentValues = fileDownloadAdornment.get().getValues();
 
             String fileNameField    = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FIELD));
             String fileNameFormat   = ValueUtils.getValueAsString(adornmentValues.get(AdornmentType.FileDownloadValues.FILE_NAME_FORMAT));
@@ -542,7 +529,13 @@ public class QValueFormatter
                   }
                }
 
-               record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               /////////////////////////////////////////////
+               // if field type is blob, update its value //
+               /////////////////////////////////////////////
+               if(QFieldType.BLOB.equals(field.getType()))
+               {
+                  record.setValue(field.getName(), "/data/" + table.getName() + "/" + primaryKey + "/" + field.getName() + "/" + fileName);
+               }
                record.setDisplayValue(field.getName(), fileName);
             }
          }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/model/actions/tables/storage/StorageInput.java

@@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput;
 public class StorageInput extends AbstractTableActionInput
 {
    private String reference;
+   private String contentType;
 
 
 
@@ -74,4 +75,35 @@ public class StorageInput extends AbstractTableActionInput
       return (this);
    }
 
+
+
+   /*******************************************************************************
+    ** Getter for contentType
+    *******************************************************************************/
+   public String getContentType()
+   {
+      return (this.contentType);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for contentType
+    *******************************************************************************/
+   public void setContentType(String contentType)
+   {
+      this.contentType = contentType;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for contentType
+    *******************************************************************************/
+   public StorageInput withContentType(String contentType)
+   {
+      this.contentType = contentType;
+      return (this);
+   }
+
 }

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStep.java

@@ -1,287 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
-
-
-import java.time.Instant;
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.actions.async.AsyncJobCallback;
-import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
-import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
-import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateResult;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
-import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
-import com.kingsrook.qqq.backend.core.utils.ValueUtils;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-/*******************************************************************************
- **
- *******************************************************************************/
-public class GenericGarbageCollectorExecuteStep implements BackendStep
-{
-   private static final QLogger LOG = QLogger.getLogger(GenericGarbageCollectorExecuteStep.class);
-
-   private ProcessSummaryLine partitionsLine = new ProcessSummaryLine(Status.INFO)
-      .withSingularPastMessage("partition was processed")
-      .withPluralPastMessage("partitions were processed");
-
-   private ProcessSummaryLine deletedLine = new ProcessSummaryLine(Status.OK)
-      .withSingularPastMessage("record was deleted")
-      .withPluralPastMessage("records were deleted");
-
-   private ProcessSummaryLine warningLine = new ProcessSummaryLine(Status.WARNING, "had an warning");
-   private ProcessSummaryLine errorLine   = new ProcessSummaryLine(Status.ERROR, "had an error");
-
-   private AsyncJobCallback asyncJobCallback;
-
-   private Integer total        = null;
-   private Integer deletedSoFar = 0;
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   @Override
-   public void run(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException
-   {
-      String         tableName = runBackendStepInput.getValueString("table");
-      QTableMetaData table     = QContext.getQInstance().getTable(tableName);
-      if(table == null)
-      {
-         throw new QUserFacingException("Unrecognized table: " + tableName);
-      }
-
-      String         fieldName = runBackendStepInput.getValueString("field");
-      QFieldMetaData field     = table.getFields().get(fieldName);
-      if(field == null)
-      {
-         throw new QUserFacingException("Unrecognized field: " + fieldName);
-      }
-
-      if(!QFieldType.DATE_TIME.equals(field.getType()) && !QFieldType.DATE.equals(field.getType()))
-      {
-         throw new QUserFacingException("Field " + field + " is not a date-time or date type field.");
-      }
-
-      Integer daysBack = runBackendStepInput.getValueInteger("daysBack");
-      if(daysBack == null || daysBack < 0)
-      {
-         throw new QUserFacingException("Illegal value for daysBack: " + daysBack + "; Must be positive.");
-      }
-
-      Integer maxPageSize = runBackendStepInput.getValueInteger("maxPageSize");
-      if(maxPageSize == null || maxPageSize < 0)
-      {
-         throw new QUserFacingException("Illegal value for maxPageSize: " + maxPageSize + "; Must be positive.");
-      }
-
-      asyncJobCallback = runBackendStepInput.getAsyncJobCallback();
-      long start = System.currentTimeMillis();
-      execute(table, field, daysBack, maxPageSize);
-      long end = System.currentTimeMillis();
-
-      deletedLine.prepareForFrontend(true);
-      partitionsLine.prepareForFrontend(true);
-
-      ArrayList<ProcessSummaryLineInterface> processSummary = new ArrayList<>();
-      processSummary.add(partitionsLine);
-      processSummary.add(deletedLine);
-      warningLine.addSelfToListIfAnyCount(processSummary);
-      errorLine.addSelfToListIfAnyCount(processSummary);
-      processSummary.add(new ProcessSummaryLine(Status.INFO, "Total time:  " + String.format("%,d", ((end - start) / 1000)) + " seconds"));
-      runBackendStepOutput.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, processSummary);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void execute(QTableMetaData table, QFieldMetaData field, Integer daysBack, Integer maxPageSize) throws QException
-   {
-      asyncJobCallback.updateStatus("Counting records");
-      Instant maxDate = Instant.now().minusSeconds(daysBack * 60 * 60 * 24);
-      Instant minDate = findMinDateInTable(table, field);
-
-      processDateRange(table, field, maxPageSize, minDate, maxDate);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void processDateRange(QTableMetaData table, QFieldMetaData field, Integer maxPageSize, Instant minDate, Instant maxDate) throws QException
-   {
-      partitionsLine.incrementCount();
-
-      LOG.info("Counting", logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-      Integer count = count(table, field, minDate, maxDate);
-      LOG.info("Count", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-
-      if(this.total == null)
-      {
-         this.total = count;
-      }
-
-      if(count == 0)
-      {
-         LOG.info("0 rows in this partition - nothing to delete", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-      }
-      else if(count <= maxPageSize)
-      {
-         asyncJobCallback.updateStatus("Deleting records", deletedSoFar, total);
-         LOG.info("Deleting", logPair("count", count), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-         delete(table, field, minDate, maxDate);
-         this.deletedSoFar += count;
-      }
-      else
-      {
-         if(deletedSoFar == 0)
-         {
-            asyncJobCallback.updateStatus("Partitioning table", deletedSoFar, total);
-         }
-
-         LOG.info("Too many rows", logPair("count", count), logPair("maxPageSize", maxPageSize), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-         partition(table, field, minDate, maxDate, count, maxPageSize);
-      }
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void partition(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate, Integer count, Integer maxPageSize) throws QException
-   {
-      int  noOfPartitions    = (int) Math.ceil((float) count / (float) maxPageSize);
-      long milliDiff         = maxDate.toEpochMilli() - minDate.toEpochMilli();
-      long milliPerPartition = milliDiff / noOfPartitions;
-
-      if(milliPerPartition < 1000)
-      {
-         throw (new QUserFacingException("To find a maxPageSize under " + String.format("%,d", maxPageSize) + ", the partition size would become smaller than 1 second (between " + minDate + " and " + maxDate + " there are " + String.format("%,d", count) + " rows) - you must use a larger maxPageSize to continue."));
-      }
-
-      LOG.info("Partitioning", logPair("count", count), logPair("noOfPartitions", noOfPartitions), logPair("milliDiff", milliDiff), logPair("milliPerPartition", milliPerPartition), logPair("table", table.getName()), logPair("field", field.getName()), logPair("minDate", minDate), logPair("maxDate", maxDate));
-      for(int i = 0; i < noOfPartitions; i++)
-      {
-         maxDate = minDate.plusMillis(milliPerPartition);
-         processDateRange(table, field, maxPageSize, minDate, maxDate);
-         minDate = maxDate;
-      }
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void delete(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
-   {
-      DeleteOutput deleteOutput = new DeleteAction().execute(new DeleteInput(table.getName())
-         .withQueryFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate))));
-
-      deletedLine.incrementCount(deleteOutput.getDeletedRecordCount());
-
-      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
-      {
-         warningLine.incrementCount(deleteOutput.getRecordsWithWarnings().size());
-      }
-
-      if(CollectionUtils.nullSafeHasContents(deleteOutput.getRecordsWithErrors()))
-      {
-         errorLine.incrementCount(deleteOutput.getRecordsWithErrors().size());
-      }
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private Integer count(QTableMetaData table, QFieldMetaData field, Instant minDate, Instant maxDate) throws QException
-   {
-      Aggregate      countDateField = new Aggregate(field.getName(), AggregateOperator.COUNT).withFieldType(QFieldType.INTEGER);
-      AggregateInput aggregateInput = new AggregateInput();
-      aggregateInput.setTableName(table.getName());
-      aggregateInput.withFilter(new QQueryFilter(new QFilterCriteria(field.getName(), QCriteriaOperator.BETWEEN, minDate, maxDate)));
-      aggregateInput.withAggregate(countDateField);
-
-      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
-      List<AggregateResult> results         = aggregateOutput.getResults();
-      if(CollectionUtils.nullSafeIsEmpty(results))
-      {
-         throw (new QUserFacingException("Could not count rows table (null or empty aggregate result)."));
-      }
-
-      return (ValueUtils.getValueAsInteger(results.get(0).getAggregateValue(countDateField)));
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private Instant findMinDateInTable(QTableMetaData table, QFieldMetaData field) throws QException
-   {
-      Aggregate      minDate        = new Aggregate(field.getName(), AggregateOperator.MIN);
-      AggregateInput aggregateInput = new AggregateInput();
-      aggregateInput.setTableName(table.getName());
-      aggregateInput.withAggregate(minDate);
-
-      AggregateOutput       aggregateOutput = new AggregateAction().execute(aggregateInput);
-      List<AggregateResult> results         = aggregateOutput.getResults();
-      if(CollectionUtils.nullSafeIsEmpty(results))
-      {
-         throw (new QUserFacingException("Could not find min date value in table (null or empty aggregate result)."));
-      }
-
-      return (ValueUtils.getValueAsInstant(results.get(0).getAggregateValue(minDate)));
-   }
-
-}

qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorProcessMetaDataProducer.java

@@ -1,88 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2023.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-
-
-/*******************************************************************************
- ** Generic process that can perform garbage collection on any table (at least,
- ** any table with a date or date-time field).
- **
- ** When running, this process prompts for:
- ** - table name
- ** - field name (e.g., the date/date-time field on that table)
- ** - daysBack - any records older than that many days ago will be deleted.
- ** - maxPageSize - to avoid running "1 huge query", if there are more than
- ** this number of records between the min-date in the table and the max-date
- ** (based on daysBack), then the time range is partitioned recursively until
- ** pages smaller than this parameter are found.  The partitioning attempts to
- ** be smart (e.g., not just ÷ 2), by doing count / maxPageSize.
- *******************************************************************************/
-public class GenericGarbageCollectorProcessMetaDataProducer extends MetaDataProducer<QProcessMetaData>
-{
-   public static final String NAME = "GenericGarbageCollector";
-
-
-
-   /*******************************************************************************
-    ** See class header for param descriptions.
-    *******************************************************************************/
-   @Override
-   public QProcessMetaData produce(QInstance qInstance) throws QException
-   {
-      QProcessMetaData processMetaData = new QProcessMetaData()
-         .withName(NAME)
-         .withIcon(new QIcon().withName("auto_delete"))
-         .withStepList(List.of(
-            new QFrontendStepMetaData()
-               .withName("input")
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
-               .withFormField(new QFieldMetaData("table", QFieldType.STRING))
-               .withFormField(new QFieldMetaData("field", QFieldType.STRING))
-               .withFormField(new QFieldMetaData("daysBack", QFieldType.INTEGER).withDefaultValue(90))
-               .withFormField(new QFieldMetaData("maxPageSize", QFieldType.INTEGER).withDefaultValue(100000)),
-            new QBackendStepMetaData()
-               .withName("execute")
-               .withCode(new QCodeReference(GenericGarbageCollectorExecuteStep.class)),
-            new QFrontendStepMetaData()
-               .withName("result")
-               .withComponent(new QFrontendComponentMetaData().withType(QComponentType.PROCESS_SUMMARY_RESULTS))
-         ));
-
-      return (processMetaData);
-   }
-
-}

qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/garbagecollector/GenericGarbageCollectorExecuteStepTest.java

@@ -1,243 +0,0 @@
-/*
- * QQQ - Low-code Application Framework for Engineers.
- * Copyright (C) 2021-2024.  Kingsrook, LLC
- * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
- * contact@kingsrook.com
- * https://github.com/Kingsrook/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package com.kingsrook.qqq.backend.core.processes.implementations.garbagecollector;
-
-
-import java.time.Instant;
-import java.time.temporal.ChronoUnit;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-import com.kingsrook.qqq.backend.core.BaseTest;
-import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
-import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
-import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
-import com.kingsrook.qqq.backend.core.context.QContext;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.exceptions.QUserFacingException;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
-import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
-import com.kingsrook.qqq.backend.core.utils.TestUtils;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-
-
-/*******************************************************************************
- ** Unit test for GenericGarbageCollectorExecuteStep 
- *******************************************************************************/
-class GenericGarbageCollectorExecuteStepTest extends BaseTest
-{
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @BeforeEach
-   @AfterEach
-   void beforeAndAfterEach()
-   {
-      MemoryRecordStore.getInstance().reset();
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void testErrors() throws Exception
-   {
-      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
-
-      RunProcessInput input = new RunProcessInput();
-      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
-      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
-
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: null");
-
-      input.addValue("table", "notATable");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized table: notATable");
-
-      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: null");
-
-      input.addValue("field", "notAField");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Unrecognized field: notAField");
-
-      input.addValue("field", "firstName");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("not a date");
-
-      input.addValue("field", "timestamp");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: null");
-
-      input.addValue("daysBack", "-1");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("daysBack: -1");
-
-      input.addValue("daysBack", "1");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: null");
-
-      input.addValue("maxPageSize", "-1");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("maxPageSize: -1");
-
-      input.addValue("maxPageSize", "1");
-      assertThatThrownBy(() -> runAndThrow(input)).isInstanceOf(QUserFacingException.class).hasMessageContaining("Could not find min date value in table");
-   }
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void runAndThrow(RunProcessInput input) throws Exception
-   {
-      input.setStartAfterStep(null);
-      input.setProcessUUID(null);
-      RunProcessOutput runProcessOutput = new RunProcessAction().execute(input);
-      if(runProcessOutput.getException().isPresent())
-      {
-         throw (runProcessOutput.getException().get());
-      }
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test30days() throws QException
-   {
-      insertAndRunGC(30);
-      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
-      assertEquals(2, queryOutput.getRecords().size());
-      assertEquals(Set.of(4, 5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   @Disabled("memory aggregator is failing to return an aggregate when no rows found, which is throwing an error...")
-   void test100days() throws QException
-   {
-      insertAndRunGC(100);
-      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
-      assertEquals(5, queryOutput.getRecords().size());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test10days() throws QException
-   {
-      insertAndRunGC(10);
-      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
-      assertEquals(1, queryOutput.getRecords().size());
-      assertEquals(Set.of(5), queryOutput.getRecords().stream().map(r -> r.getValueInteger("id")).collect(Collectors.toSet()));
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test1day() throws QException
-   {
-      insertAndRunGC(1);
-      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
-      assertEquals(0, queryOutput.getRecords().size());
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   @Test
-   void test1dayPartitioned() throws QException
-   {
-      insertAndRunGC(1, 2);
-      QueryOutput queryOutput = new QueryAction().execute(new QueryInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withFilter(new QQueryFilter()));
-      assertEquals(0, queryOutput.getRecords().size());
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void insertAndRunGC(Integer daysBack) throws QException
-   {
-      insertAndRunGC(daysBack, 1000);
-   }
-
-
-
-   /***************************************************************************
-    **
-    ***************************************************************************/
-   private void insertAndRunGC(Integer daysBack, Integer maxPageSize) throws QException
-   {
-      new InsertAction().execute(new InsertInput(TestUtils.TABLE_NAME_PERSON_MEMORY).withRecords(getPersonRecords()));
-      QContext.getQInstance().addProcess(new GenericGarbageCollectorProcessMetaDataProducer().produce(QContext.getQInstance()));
-
-      RunProcessInput input = new RunProcessInput();
-      input.setProcessName(GenericGarbageCollectorProcessMetaDataProducer.NAME);
-      input.addValue("table", TestUtils.TABLE_NAME_PERSON_MEMORY);
-      input.addValue("field", "timestamp");
-      input.addValue("daysBack", daysBack);
-      input.addValue("maxPageSize", maxPageSize);
-      input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
-      new RunProcessAction().execute(input);
-   }
-
-
-
-   /*******************************************************************************
-    **
-    *******************************************************************************/
-   private static List<QRecord> getPersonRecords()
-   {
-      List<QRecord> records = List.of(
-         new QRecord().withValue("id", 1).withValue("timestamp", Instant.now().minus(90, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 2).withValue("timestamp", Instant.now().minus(31, ChronoUnit.DAYS)),
-         new QRecord().withValue("id", 3).withValue("timestamp", Instant.now().minus(30, ChronoUnit.DAYS).minus(5, ChronoUnit.MINUTES)),
-         new QRecord().withValue("id", 4).withValue("timestamp", Instant.now().minus(29, ChronoUnit.DAYS).minus(23, ChronoUnit.HOURS)),
-         new QRecord().withValue("id", 5).withValue("timestamp", Instant.now().minus(5, ChronoUnit.DAYS)));
-      return records;
-   }
-
-}

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/actions/S3StorageAction.java

@@ -58,7 +58,7 @@ public class S3StorageAction extends AbstractS3Action implements QStorageInterfa
 
          AmazonS3             amazonS3             = getS3Utils().getAmazonS3();
          String               fullPath             = getFullPath(storageInput);
-         S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, backend.getBucketName(), fullPath);
+         S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(amazonS3, backend.getBucketName(), fullPath, storageInput.getContentType());
          return (s3UploadOutputStream);
       }
       catch(Exception e)

qqq-backend-module-filesystem/src/main/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStream.java

@@ -53,6 +53,7 @@ public class S3UploadOutputStream extends OutputStream
    private final AmazonS3 amazonS3;
    private final String   bucketName;
    private final String   key;
+   private final String   contentType;
 
    private byte[] buffer = new byte[5 * 1024 * 1024];
    private int    offset = 0;
@@ -68,11 +69,12 @@ public class S3UploadOutputStream extends OutputStream
     ** Constructor
     **
     *******************************************************************************/
-   public S3UploadOutputStream(AmazonS3 amazonS3, String bucketName, String key)
+   public S3UploadOutputStream(AmazonS3 amazonS3, String bucketName, String key, String contentType)
    {
       this.amazonS3 = amazonS3;
       this.bucketName = bucketName;
       this.key = key;
+      this.contentType = contentType;
    }
 
 
@@ -96,6 +98,13 @@ public class S3UploadOutputStream extends OutputStream
     *******************************************************************************/
    private void uploadIfNeeded()
    {
+      ObjectMetadata objectMetadata = null;
+      if(this.contentType != null)
+      {
+         objectMetadata = new ObjectMetadata();
+         objectMetadata.setContentType(this.contentType);
+      }
+
       if(offset == buffer.length)
       {
          //////////////////////////////////////////
@@ -104,7 +113,8 @@ public class S3UploadOutputStream extends OutputStream
          if(initiateMultipartUploadResult == null)
          {
             LOG.info("Initiating a multipart upload", logPair("key", key));
-            initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key));
+            initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key, objectMetadata));
+
             uploadPartResultList = new ArrayList<>();
          }
 
@@ -115,7 +125,8 @@ public class S3UploadOutputStream extends OutputStream
             .withInputStream(new ByteArrayInputStream(buffer))
             .withBucketName(bucketName)
             .withKey(key)
-            .withPartSize(buffer.length);
+            .withPartSize(buffer.length)
+            .withObjectMetadata(objectMetadata);
 
          uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest));
 
@@ -166,6 +177,13 @@ public class S3UploadOutputStream extends OutputStream
          return;
       }
 
+      ObjectMetadata objectMetadata = null;
+      if(this.contentType != null)
+      {
+         objectMetadata = new ObjectMetadata();
+         objectMetadata.setContentType(this.contentType);
+      }
+
       if(initiateMultipartUploadResult != null)
       {
          if(offset > 0)
@@ -180,7 +198,8 @@ public class S3UploadOutputStream extends OutputStream
                .withInputStream(new ByteArrayInputStream(buffer, 0, offset))
                .withBucketName(bucketName)
                .withKey(key)
-               .withPartSize(offset);
+               .withPartSize(offset)
+               .withObjectMetadata(objectMetadata);
             uploadPartResultList.add(amazonS3.uploadPart(uploadPartRequest));
          }
 
@@ -193,8 +212,12 @@ public class S3UploadOutputStream extends OutputStream
       }
       else
       {
+         if(objectMetadata == null)
+         {
+            objectMetadata = new ObjectMetadata();
+         }
+
          LOG.info("Putting object (non-multipart)", logPair("key", key), logPair("length", offset));
-         ObjectMetadata objectMetadata = new ObjectMetadata();
          objectMetadata.setContentLength(offset);
          PutObjectResult putObjectResult = amazonS3.putObject(bucketName, key, new ByteArrayInputStream(buffer, 0, offset), objectMetadata);
       }

qqq-backend-module-filesystem/src/test/java/com/kingsrook/qqq/backend/module/filesystem/s3/utils/S3UploadOutputStreamTest.java

@@ -31,7 +31,7 @@ import org.junit.jupiter.api.Test;
 
 
 /*******************************************************************************
- ** Unit test for S3UploadOutputStream 
+ ** Unit test for S3UploadOutputStream
  *******************************************************************************/
 class S3UploadOutputStreamTest extends BaseS3Test
 {
@@ -57,11 +57,11 @@ class S3UploadOutputStreamTest extends BaseS3Test
       outputStream.write("\n]\n".getBytes(StandardCharsets.UTF_8));
       outputStream.close();
 
-      S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(getS3Utils().getAmazonS3(), bucketName, key);
+      S3UploadOutputStream s3UploadOutputStream = new S3UploadOutputStream(getS3Utils().getAmazonS3(), bucketName, key, null);
       s3UploadOutputStream.write(outputStream.toByteArray(), 0, 5 * 1024 * 1024);
       s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024);
       s3UploadOutputStream.write(outputStream.toByteArray(), 0, 3 * 1024 * 1024);
       s3UploadOutputStream.close();
    }
 
-}
+}