dev/add datatypes to excel columns

excel/pom.xml

@@ -40,7 +40,7 @@ under the License.
 		<dependency>
 			<groupId>org.apache.poi</groupId>
 			<artifactId>poi-ooxml</artifactId>
-			<version>4.0.1</version>
+			<version>4.1.0</version>
 			<exclusions>
 				<exclusion>
 					<groupId>commons-logging</groupId>

excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java

@@ -41,6 +41,7 @@
 import org.apache.metamodel.util.FileHelper;
 import org.apache.metamodel.util.Resource;
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellType;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
@@ -103,9 +104,8 @@ public void notifyTablesModified() {
         // do nothing
     }
 
-    private MutableTable createTable(final Workbook wb, final Sheet sheet) {
+    private MutableTable createTable(Workbook wb, Sheet sheet) {
         final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE);
-
         if (sheet.getPhysicalNumberOfRows() <= 0) {
             // no physical rows in sheet
             return table;
@@ -117,9 +117,7 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
             // no physical rows in sheet
             return table;
         }
-
         Row row = null;
-
         if (_configuration.isSkipEmptyLines()) {
             while (row == null && rowIterator.hasNext()) {
                 row = rowIterator.next();
@@ -128,6 +126,12 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
             row = rowIterator.next();
         }
 
+        // Get first 1000 rows for the eager-read
+        final Iterator<Row> data = ExcelUtils.getRowIterator(sheet, _configuration, false);
+        int rowLength = row.getLastCellNum();
+        ColumnType[] columnTypes = new ColumnType[rowLength];
+
+        setColumnType(data, rowLength, columnTypes);
         final int columnNameLineNumber = _configuration.getColumnNameLineNumber();
         if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) {
 
@@ -137,27 +141,22 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
             while (row == null && rowIterator.hasNext()) {
                 row = rowIterator.next();
             }
-
             // build columns without any intrinsic column names
             final ColumnNamingStrategy columnNamingStrategy = _configuration.getColumnNamingStrategy();
             try (final ColumnNamingSession columnNamingSession = columnNamingStrategy.startColumnNamingSession()) {
                 final int offset = getColumnOffset(row);
                 for (int i = 0; i < offset; i++) {
                     columnNamingSession.getNextColumnName(new ColumnNamingContextImpl(i));
                 }
-
                 for (int j = offset; j < row.getLastCellNum(); j++) {
                     final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j);
                     final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext),
-                            ColumnType.STRING, table, j, true);
+                            columnTypes[j], table, j, true);
                     table.addColumn(column);
                 }
             }
-
         } else {
-
             boolean hasColumns = true;
-
             // iterate to the column name line number (if above 1)
             for (int j = 1; j < columnNameLineNumber; j++) {
                 if (rowIterator.hasNext()) {
@@ -167,50 +166,114 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
                     break;
                 }
             }
-
             if (hasColumns) {
-                createColumns(table, wb, row);
+                createColumns(table, wb, row, columnTypes);
             }
         }
-
         return table;
     }
 
+    private void setColumnType(Iterator<Row> data, int rowLength, ColumnType[] columnTypes) {
+        while (data.hasNext()) {
+            Row row = data.next();
+            for (int index = 0; index < rowLength; index++) {
+                if (row.getLastCellNum() == 0) {
+                    continue;
+                }
+                if (row.getCell(index) == null) {
+                    columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index);
+                } else {
+                    CellType cellType = row.getCell(index).getCellType();
+                    if (cellType.getCode() != 0 && cellType.getCode() <= 2) {
+                        columnTypes = checkColumnType(ColumnType.STRING, columnTypes, index);
+                    } else if (cellType.getCode() == 0) {
+                        columnTypes = checkColumnType((row.getCell(index).getNumericCellValue() % 1 == 0)
+                                ? ColumnType.INTEGER : ColumnType.DOUBLE, columnTypes, index);
+                    } else if (cellType.getCode() == 4) {
+                        columnTypes = checkColumnType(ColumnType.BOOLEAN, columnTypes, index);
+                    }
+                }
+            }
+        }
+    }
+
+    private ColumnType[] checkColumnType(ColumnType columnType, ColumnType[] columnTypes, int index) {
+        if (columnTypes[index] != null) {
+            if (!columnTypes[index].equals(ColumnType.STRING) && !columnTypes[index].equals(columnType)) {
+                columnTypes[index] = ColumnType.STRING;
+            }
+        } else {
+            columnTypes[index] = columnType;
+        }
+        return columnTypes;
+    }
+
+    private void determineColumnDatatype(Object[] datatypes, Row row) {
+        for (int index = 0; index < row.getLastCellNum(); index++) {
+            CellType type = ((Cell) row.getCell(index)).getCellType();
+
+            if (datatypes[index] instanceof Object) {
+                datatypes[index] = type;
+            } else if (datatypes[index] instanceof CellType) {
+                if (datatypes[index].equals(type)) {
+                    continue;
+                } else {
+                    datatypes[index] = CellType.STRING;
+                }
+            }
+        }
+    }
+
     /**
      * Builds columns based on row/cell values.
      * 
      * @param table
      * @param wb
      * @param row
      */
-    private void createColumns(MutableTable table, Workbook wb, Row row) {
+    private void createColumns(MutableTable table, Workbook wb, Row row, ColumnType[] columTypes) {
         if (row == null) {
             logger.warn("Cannot create columns based on null row!");
             return;
         }
         final short rowLength = row.getLastCellNum();
-
         final int offset = getColumnOffset(row);
-
         // build columns based on cell values.
-        try (final ColumnNamingSession columnNamingSession = _configuration.getColumnNamingStrategy()
+        try (final ColumnNamingSession columnNamingSession = _configuration
+                .getColumnNamingStrategy()
                 .startColumnNamingSession()) {
             for (int j = offset; j < rowLength; j++) {
                 final Cell cell = row.getCell(j);
                 final String intrinsicColumnName = ExcelUtils.getCellValue(wb, cell);
                 final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName,
                         j);
                 final String columnName = columnNamingSession.getNextColumnName(columnNamingContext);
-                final Column column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true);
+                Column column = null;
+                if (columTypes == null) {
+                    column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true);
+                } else {
+                    column = new MutableColumn(columnName, columTypes[j], table, j, true);
+                }
                 table.addColumn(column);
             }
         }
     }
 
     /**
-     * Gets the column offset (first column to include). This is dependent on
-     * the row used for column processing and whether the skip empty columns
-     * property is set.
+     * Builds columns based on row/cell values.
+     * 
+     * @param table
+     * @param wb
+     * @param row
+     */
+    private void createColumns(MutableTable table, Workbook wb, Row row) {
+        createColumns(table, wb, row, null);
+    }
+
+    /**
+     * Gets the column offset (first column to include). This is dependent on the
+     * row used for column processing and whether the skip empty columns property is
+     * set.
      * 
      * @param row
      * @return

excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java

@@ -402,10 +402,10 @@ public void testMissingValues() throws Exception {
         assertEquals(2, schema.getTableCount());
 
         Table table = schema.getTables().get(0);
-        assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=c,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+        assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=c,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
 
         Query q = new Query().select(table.getColumns()).from(table);
@@ -426,10 +426,10 @@ public void testMissingColumnHeader() throws Exception {
         assertEquals(2, schema.getTableCount());
 
         Table table = schema.getTables().get(0);
-        assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=A,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+        assertEquals("[Column[name=a,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=b,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=A,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
 
         Query q = new Query().select(table.getColumns()).from(table);
@@ -532,11 +532,10 @@ public void testTicket99defect() throws Exception {
 
         Table table = schema.getTableByName("Sheet1");
         assertEquals(
-
-                "[Column[name=Pkg No.,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Description,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Room,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Level,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+                "[Column[name=Pkg No.,columnNumber=0,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Description,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Room,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Level,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
     }