diff --git a/qqq-backend-core/pom.xml b/qqq-backend-core/pom.xml
index 0944e1e9..21723b83 100644
--- a/qqq-backend-core/pom.xml
+++ b/qqq-backend-core/pom.xml
@@ -100,7 +100,7 @@
org.dhatim
fastexcel
- 0.12.15
+ 0.18.4
org.dhatim
diff --git a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
index 206ca722..de0b1ccb 100644
--- a/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
+++ b/qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRows.java
@@ -26,14 +26,16 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Serializable;
import java.math.BigDecimal;
-import java.math.MathContext;
-import java.time.LocalDateTime;
import java.util.Optional;
+import java.util.regex.Pattern;
import java.util.stream.Stream;
import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.processes.implementations.bulk.insert.model.BulkLoadFileRow;
import org.dhatim.fastexcel.reader.Cell;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
+import org.dhatim.fastexcel.reader.ReadingOptions;
+import org.dhatim.fastexcel.reader.Row;
import org.dhatim.fastexcel.reader.Sheet;
@@ -42,6 +44,10 @@ import org.dhatim.fastexcel.reader.Sheet;
*******************************************************************************/
public class XlsxFileToRows extends AbstractIteratorBasedFileToRows implements FileToRowsInterface
{
+ private static final QLogger LOG = QLogger.getLogger(XlsxFileToRows.class);
+
+ private static final Pattern DAY_PATTERN = Pattern.compile(".*\\b(d|dd)\\b.*");
+
private ReadableWorkbook workbook;
private Stream rows;
@@ -55,7 +61,7 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows
- {
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- // ... with fastexcel reader, we don't get styles... so, we just know type = number, for dates and ints & decimals... //
- ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
- Optional dateTime = readerRow.getCellAsDate(i);
- if(dateTime.isPresent() && dateTime.get().getYear() > 1915 && dateTime.get().getYear() < 2100)
- {
- yield dateTime.get();
- }
-
- Optional optionalBigDecimal = readerRow.getCellAsNumber(i);
- if(optionalBigDecimal.isPresent())
- {
- BigDecimal bigDecimal = optionalBigDecimal.get();
- if(bigDecimal.subtract(bigDecimal.round(new MathContext(0))).compareTo(BigDecimal.ZERO) == 0)
- {
- yield bigDecimal.intValue();
- }
-
- yield bigDecimal;
- }
-
- yield (null);
- }
- case BOOLEAN -> readerRow.getCellAsBoolean(i).orElse(null);
- case STRING, FORMULA -> cell.getText();
- case EMPTY, ERROR -> null;
- };
- }
+ values[i] = processCell(readerRow, i);
}
return new BulkLoadFileRow(values, getRowNo());
@@ -121,6 +93,150 @@ public class XlsxFileToRows extends AbstractIteratorBasedFileToRows
+ {
+ /////////////////////////////////////////////////////////////////////////////////////
+ // dates, date-times, integers, and decimals are all identified as type = "number" //
+ // so go through this process to try to identify what user means it as //
+ /////////////////////////////////////////////////////////////////////////////////////
+ if(isDateTimeFormat(dataFormatString))
+ {
+ ////////////////////////////////////////////////////////////////////////////////////////
+ // first - if it has a date-time looking format string, then treat it as a date-time. //
+ ////////////////////////////////////////////////////////////////////////////////////////
+ return (cell.asDate());
+ }
+ else if(isDateFormat(dataFormatString))
+ {
+ ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+ // second, if it has a date looking format string (which is a sub-set of date-time), then treat as date. //
+ ///////////////////////////////////////////////////////////////////////////////////////////////////////////
+ return (cell.asDate().toLocalDate());
+ }
+ else
+ {
+ ////////////////////////////////////////////////////////////////////////////////////////
+ // now assume it's a number - but in case this optional is empty (why?) return a null //
+ ////////////////////////////////////////////////////////////////////////////////////////
+ Optional bigDecimal = readerRow.getCellAsNumber(columnIndex);
+ if(bigDecimal.isEmpty())
+ {
+ return (null);
+ }
+
+ try
+ {
+ ////////////////////////////////////////////////////////////
+ // now if the bigDecimal is an exact integer, return that //
+ ////////////////////////////////////////////////////////////
+ Integer i = bigDecimal.get().intValueExact();
+ return (i);
+ }
+ catch(ArithmeticException e)
+ {
+ /////////////////////////////////
+ // else, end up with a decimal //
+ /////////////////////////////////
+ return (bigDecimal.get());
+ }
+ }
+ }
+ case STRING ->
+ {
+ return cell.asString();
+ }
+ case BOOLEAN ->
+ {
+ return cell.asBoolean();
+ }
+ case EMPTY, ERROR, FORMULA ->
+ {
+ LOG.debug("cell type: " + cell.getType() + " had value string: " + cell.asString());
+ return (null);
+ }
+ default ->
+ {
+ return (null);
+ }
+ }
+ }
+
+
+
+ /***************************************************************************
+ **
+ ***************************************************************************/
+ static boolean isDateTimeFormat(String dataFormatString)
+ {
+ if(dataFormatString == null)
+ {
+ return (false);
+ }
+
+ if(hasDay(dataFormatString) && hasHour(dataFormatString))
+ {
+ return (true);
+ }
+
+ return false;
+ }
+
+
+
+ /***************************************************************************
+ **
+ ***************************************************************************/
+ static boolean hasHour(String dataFormatString)
+ {
+ return dataFormatString.contains("h");
+ }
+
+
+
+ /***************************************************************************
+ **
+ ***************************************************************************/
+ static boolean hasDay(String dataFormatString)
+ {
+ return DAY_PATTERN.matcher(dataFormatString).matches();
+ }
+
+
+
+ /***************************************************************************
+ **
+ ***************************************************************************/
+ static boolean isDateFormat(String dataFormatString)
+ {
+ if(dataFormatString == null)
+ {
+ return (false);
+ }
+
+ if(hasDay(dataFormatString))
+ {
+ return (true);
+ }
+
+ return false;
+ }
+
+
+
/***************************************************************************
**
***************************************************************************/
diff --git a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java
index 95052d56..8d144ad6 100644
--- a/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java
+++ b/qqq-backend-core/src/test/java/com/kingsrook/qqq/backend/core/processes/implementations/bulk/insert/filehandling/XlsxFileToRowsTest.java
@@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.Serializable;
import java.time.LocalDate;
-import java.time.LocalDateTime;
import java.time.Month;
import java.util.Map;
import com.kingsrook.qqq.backend.core.BaseTest;
@@ -45,6 +44,7 @@ import org.junit.jupiter.api.Test;
import static com.kingsrook.qqq.backend.core.actions.reporting.GenerateReportActionTest.REPORT_NAME;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/*******************************************************************************
@@ -61,13 +61,14 @@ class XlsxFileToRowsTest extends BaseTest
{
byte[] byteArray = writeExcelBytes();
- FileToRowsInterface fileToRowsInterface = FileToRowsInterface.forFile("someFile.xlsx", new ByteArrayInputStream(byteArray));
+ FileToRowsInterface fileToRowsInterface = new XlsxFileToRows();
+ fileToRowsInterface.init(new ByteArrayInputStream(byteArray));
BulkLoadFileRow headerRow = fileToRowsInterface.next();
BulkLoadFileRow bodyRow = fileToRowsInterface.next();
- assertEquals(new BulkLoadFileRow(new String[] {"Id", "First Name", "Last Name", "Birth Date"}, 1), headerRow);
- assertEquals(new BulkLoadFileRow(new Serializable[] {1, "Darin", "Jonson", LocalDateTime.of(1980, Month.JANUARY, 31, 0, 0)}, 2), bodyRow);
+ assertEquals(new BulkLoadFileRow(new String[] { "Id", "First Name", "Last Name", "Birth Date" }, 1), headerRow);
+ assertEquals(new BulkLoadFileRow(new Serializable[] { 1, "Darin", "Jonson", LocalDate.of(1980, Month.JANUARY, 31) }, 2), bodyRow);
///////////////////////////////////////////////////////////////////////////////////////
// make sure there's at least a limit (less than 20) to how many more rows there are //
@@ -107,4 +108,54 @@ class XlsxFileToRowsTest extends BaseTest
return byteArray;
}
+
+
+ /*******************************************************************************
+ **
+ *******************************************************************************/
+ @Test
+ void testDateTimeFormats()
+ {
+ assertFormatDateAndOrDateTime(true, false, "dddd, m/d/yy at h:mm");
+ assertFormatDateAndOrDateTime(true, false, "h PM, ddd mmm dd");
+ assertFormatDateAndOrDateTime(true, false, "dd/mm/yyyy hh:mm");
+ assertFormatDateAndOrDateTime(true, false, "yyyy-mm-dd hh:mm:ss.000");
+ assertFormatDateAndOrDateTime(true, false, "hh:mm dd/mm/yyyy");
+
+ assertFormatDateAndOrDateTime(false, true, "yyyy-mm-dd");
+ assertFormatDateAndOrDateTime(false, true, "mmmm d \\[dddd\\]");
+ assertFormatDateAndOrDateTime(false, true, "mmm dd, yyyy");
+ assertFormatDateAndOrDateTime(false, true, "d-mmm");
+ assertFormatDateAndOrDateTime(false, true, "dd.mm.yyyy");
+
+ assertFormatDateAndOrDateTime(false, false, "yyyy");
+ assertFormatDateAndOrDateTime(false, false, "mmm-yyyy");
+ assertFormatDateAndOrDateTime(false, false, "hh");
+ assertFormatDateAndOrDateTime(false, false, "hh:mm");
+ }
+
+
+
+ /***************************************************************************
+ *
+ ***************************************************************************/
+ private void assertFormatDateAndOrDateTime(boolean expectDateTime, boolean expectDate, String format)
+ {
+ if(XlsxFileToRows.isDateTimeFormat(format))
+ {
+ assertTrue(expectDateTime, format + " was considered a dateTime, but wasn't expected to.");
+ assertFalse(expectDate, format + " was considered a dateTime, but was expected to be a date.");
+ }
+ else if(XlsxFileToRows.isDateFormat(format))
+ {
+ assertFalse(expectDateTime, format + " was considered a date, but was expected to be a dateTime.");
+ assertTrue(expectDate, format + " was considered a date, but was expected to.");
+ }
+ else
+ {
+ assertFalse(expectDateTime, format + " was not considered a dateTime, but was expected to.");
+ assertFalse(expectDate, format + " was considered a date, but was expected to.");
+ }
+ }
+
}
\ No newline at end of file