dev/add datatypes to excel columns

excel/pom.xml

@@ -40,7 +40,7 @@ under the License.
 		<dependency>
 			<groupId>org.apache.poi</groupId>
 			<artifactId>poi-ooxml</artifactId>
-			<version>4.0.1</version>
+			<version>4.1.0</version>
 			<exclusions>
 				<exclusion>
 					<groupId>commons-logging</groupId>

excel/src/main/java/org/apache/metamodel/excel/DefaultSpreadsheetReaderDelegate.java

@@ -41,6 +41,8 @@
 import org.apache.metamodel.util.FileHelper;
 import org.apache.metamodel.util.Resource;
 import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.CellType;
+import org.apache.poi.ss.usermodel.DateUtil;
 import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.ss.usermodel.Workbook;
@@ -63,14 +65,13 @@ public DefaultSpreadsheetReaderDelegate(Resource resource, ExcelConfiguration co
         _configuration = configuration;
     }
 
-    @Override
     public Schema createSchema(String schemaName) {
         final MutableSchema schema = new MutableSchema(schemaName);
         final Workbook wb = ExcelUtils.readWorkbook(_resource, true);
         try {
             for (int i = 0; i < wb.getNumberOfSheets(); i++) {
                 final Sheet currentSheet = wb.getSheetAt(i);
-                final MutableTable table = createTable(wb, currentSheet);
+                final MutableTable table = createTable(wb, currentSheet, _configuration.isValidateColumnTypes());
                 table.setSchema(schema);
                 schema.addTable(table);
             }
@@ -103,7 +104,7 @@ public void notifyTablesModified() {
         // do nothing
     }
 
-    private MutableTable createTable(final Workbook wb, final Sheet sheet) {
+    private MutableTable createTable(final Workbook wb, final Sheet sheet, boolean validateColumnTypes) {
         final MutableTable table = new MutableTable(sheet.getSheetName(), TableType.TABLE);
 
         if (sheet.getPhysicalNumberOfRows() <= 0) {
@@ -124,11 +125,14 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
             while (row == null && rowIterator.hasNext()) {
                 row = rowIterator.next();
             }
+
         } else {
             row = rowIterator.next();
         }
 
         final int columnNameLineNumber = _configuration.getColumnNameLineNumber();
+        final ColumnType[] columnTypes = getColumnTypes(sheet, row);
+
         if (columnNameLineNumber == ExcelConfiguration.NO_COLUMN_NAME_LINE) {
 
             // get to the first non-empty line (no matter if lines are skipped
@@ -148,8 +152,17 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
 
                 for (int j = offset; j < row.getLastCellNum(); j++) {
                     final ColumnNamingContext namingContext = new ColumnNamingContextImpl(table, null, j);
-                    final Column column = new MutableColumn(columnNamingSession.getNextColumnName(namingContext),
-                            ColumnType.STRING, table, j, true);
+                    final Column column;
+                    if (validateColumnTypes) {
+
+                        column =
+                                new MutableColumn(columnNamingSession.getNextColumnName(namingContext), columnTypes[j],
+                                        table, j, true);
+                    } else {
+                        column =
+                                new MutableColumn(columnNamingSession.getNextColumnName(namingContext),
+                                        ColumnType.STRING, table, j, true);
+                    }
                     table.addColumn(column);
                 }
             }
@@ -169,21 +182,84 @@ private MutableTable createTable(final Workbook wb, final Sheet sheet) {
             }
 
             if (hasColumns) {
-                createColumns(table, wb, row);
+                createColumns(table, wb, row, columnTypes);
             }
         }
 
         return table;
     }
 
+    private ColumnType[] getColumnTypes(final Sheet sheet, final Row row) {
+        final Iterator<Row> data = ExcelUtils.getRowIterator(sheet, _configuration, false);
+        final int rowLength = row.getLastCellNum();
+        int eagerness = ExcelConfiguration.EAGER_READ;
+        final ColumnType[] columnTypes = new ColumnType[rowLength];
+
+        while (data.hasNext() && eagerness-- > 0) {
+            final Row currentRow = data.next();
+            if (currentRow.getRowNum() < _configuration.getColumnNameLineNumber()) {
+                continue;
+            }
+            for (int index = 0; index < rowLength; index++) {
+                if (currentRow.getLastCellNum() == 0) {
+                    continue;
+                }
+                columnTypes[index] = getColumnTypeFromRow(columnTypes[index], currentRow, index);
+            }
+        }
+        return columnTypes;
+    }
+
+	private ColumnType getColumnTypeFromRow(final ColumnType columnType, final Row currentRow, int index) {
+		if (currentRow.getCell(index) == null) {
+		    return checkColumnType(ColumnType.STRING, columnType);
+		} else {
+		    CellType cellType = currentRow.getCell(index).getCellType();
+		    switch (cellType) {
+		    case NUMERIC:
+		        if (DateUtil.isCellDateFormatted(currentRow.getCell(index))) {
+		            return checkColumnType(ColumnType.DATE, columnType);
+		        } else {
+		            return checkColumnType((currentRow.getCell(index).getNumericCellValue() % 1 == 0)
+		                    ? ColumnType.INTEGER : ColumnType.DOUBLE, columnType);
+		        }
+		    case BOOLEAN:
+		        return checkColumnType(ColumnType.BOOLEAN, columnType);
+		    case ERROR:
+		        // fall through
+		    case _NONE:
+		        // fall through
+		    case STRING:
+		        // fall through
+		    case FORMULA:
+		        // fall through
+		    case BLANK:
+		    	// fall through
+		    default :
+		        return checkColumnType(ColumnType.STRING, columnType);
+		    }
+		}
+	}
+
+    private ColumnType checkColumnType(final ColumnType expecetedColumnType, ColumnType columnType) {
+        if (columnType != null) {
+            if (!columnType.equals(ColumnType.STRING) && !columnType.equals(expecetedColumnType)) {
+                return ColumnType.VARCHAR;
+            }
+        } else {
+            return expecetedColumnType;
+        }
+        return columnType;
+    }
+
     /**
      * Builds columns based on row/cell values.
      * 
      * @param table
      * @param wb
      * @param row
      */
-    private void createColumns(MutableTable table, Workbook wb, Row row) {
+    private void createColumns(final MutableTable table, final Workbook wb, final Row row, final ColumnType[] columTypes) {
         if (row == null) {
             logger.warn("Cannot create columns based on null row!");
             return;
@@ -201,7 +277,12 @@ private void createColumns(MutableTable table, Workbook wb, Row row) {
                 final ColumnNamingContext columnNamingContext = new ColumnNamingContextImpl(table, intrinsicColumnName,
                         j);
                 final String columnName = columnNamingSession.getNextColumnName(columnNamingContext);
-                final Column column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true);
+                final Column column;
+                if (columTypes == null) {
+                    column = new MutableColumn(columnName, ColumnType.VARCHAR, table, j, true);
+                } else {
+                    column = new MutableColumn(columnName, columTypes[j], table, j, true);
+                }
                 table.addColumn(column);
             }
         }

excel/src/main/java/org/apache/metamodel/excel/ExcelConfiguration.java

@@ -37,26 +37,38 @@ public final class ExcelConfiguration extends BaseObject implements
 
 	public static final int NO_COLUMN_NAME_LINE = 0;
 	public static final int DEFAULT_COLUMN_NAME_LINE = 1;
+	public static final int EAGER_READ = 1000;
 
 	private final int columnNameLineNumber;
 	private final ColumnNamingStrategy columnNamingStrategy;
 	private final boolean skipEmptyLines;
 	private final boolean skipEmptyColumns;
+    private final boolean validateColumnTypes;
 
 	public ExcelConfiguration() {
 		this(DEFAULT_COLUMN_NAME_LINE, true, false);
 	}
 
     public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns) {
-        this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns);
+        this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, false);
+    }
+
+    public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy,
+            Boolean skipEmptyLines, Boolean skipEmptyColumns) {
+        this(columnNameLineNumber, columnNamingStrategy, skipEmptyLines, skipEmptyColumns, false);
+    }
+
+    public ExcelConfiguration(int columnNameLineNumber, boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) {
+        this(columnNameLineNumber, null, skipEmptyLines, skipEmptyColumns, validateColumnTypes);
     }
 
     public ExcelConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy,
-            boolean skipEmptyLines, boolean skipEmptyColumns) {
+            boolean skipEmptyLines, boolean skipEmptyColumns, boolean validateColumnTypes) {
         this.columnNameLineNumber = columnNameLineNumber;
         this.skipEmptyLines = skipEmptyLines;
         this.skipEmptyColumns = skipEmptyColumns;
         this.columnNamingStrategy = columnNamingStrategy;
+        this.validateColumnTypes = validateColumnTypes;
     }
 
     /**
@@ -102,17 +114,29 @@ public boolean isSkipEmptyColumns() {
 		return skipEmptyColumns;
 	}
 
+    /**
+     * Defines if columns in the excel spreadsheet should be validated on datatypes while
+     * reading the spreadsheet.
+     * 
+     * @return a boolean indicating whether or not to validate column types.
+     */
+    public boolean isValidateColumnTypes() {
+        return validateColumnTypes;
+    }
+
 	@Override
 	protected void decorateIdentity(List<Object> identifiers) {
 		identifiers.add(columnNameLineNumber);
 		identifiers.add(skipEmptyLines);
 		identifiers.add(skipEmptyColumns);
+        identifiers.add(validateColumnTypes);
 	}
 
 	@Override
 	public String toString() {
 		return "ExcelConfiguration[columnNameLineNumber="
 				+ columnNameLineNumber + ", skipEmptyLines=" + skipEmptyLines
-				+ ", skipEmptyColumns=" + skipEmptyColumns + "]";
+				+ ", skipEmptyColumns=" + skipEmptyColumns +", validateColumnTypes="
+				+ validateColumnTypes + "]";
 	}
 }

excel/src/test/java/org/apache/metamodel/excel/ExcelConfigurationTest.java

@@ -27,7 +27,7 @@ public class ExcelConfigurationTest extends TestCase {
 	public void testToString() throws Exception {
 		ExcelConfiguration conf = new ExcelConfiguration(1, true, false);
 		assertEquals(
-				"ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false]",
+				"ExcelConfiguration[columnNameLineNumber=1, skipEmptyLines=true, skipEmptyColumns=false, validateColumnTypes=false]",
 				conf.toString());
 	}
 

excel/src/test/java/org/apache/metamodel/excel/ExcelDataContextTest.java

@@ -402,10 +402,10 @@ public void testMissingValues() throws Exception {
         assertEquals(2, schema.getTableCount());
 
         Table table = schema.getTables().get(0);
-        assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
+        assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
                 + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=c,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+                + "Column[name=c,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=d,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
 
         Query q = new Query().select(table.getColumns()).from(table);
@@ -419,17 +419,31 @@ public void testMissingValues() throws Exception {
         assertFalse(ds.next());
     }
 
+    public void testDifferentDataTypes() throws Exception {
+        DataContext dc = new ExcelDataContext(copyOf("src/test/resources/different_datatypes.xls"), 
+                new ExcelConfiguration(ExcelConfiguration.DEFAULT_COLUMN_NAME_LINE, true, false, true));
+
+        Schema schema = dc.getDefaultSchema();
+        assertEquals(2, schema.getTableCount());
+
+        Table table = schema.getTables().get(0);
+        assertEquals("[Column[name=INTEGER,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=TEXT,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=FORMULA,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
+                Arrays.toString(table.getColumns().toArray()));
+    }
+
     public void testMissingColumnHeader() throws Exception {
         File file = copyOf("src/test/resources/xls_missing_column_header.xls");
         DataContext dc = new ExcelDataContext(file);
         Schema schema = dc.getDefaultSchema();
         assertEquals(2, schema.getTableCount());
 
         Table table = schema.getTables().get(0);
-        assertEquals("[Column[name=a,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=b,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=A,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                + "Column[name=d,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+        assertEquals("[Column[name=a,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=b,columnNumber=1,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=A,columnNumber=2,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                + "Column[name=d,columnNumber=3,type=INTEGER,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
 
         Query q = new Query().select(table.getColumns()).from(table);
@@ -532,11 +546,10 @@ public void testTicket99defect() throws Exception {
 
         Table table = schema.getTableByName("Sheet1");
         assertEquals(
-
-                "[Column[name=Pkg No.,columnNumber=0,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Description,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Room,columnNumber=2,type=VARCHAR,nullable=true,nativeType=null,columnSize=null], "
-                        + "Column[name=Level,columnNumber=3,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]]",
+                "[Column[name=Pkg No.,columnNumber=0,type=INTEGER,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Description,columnNumber=1,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Room,columnNumber=2,type=STRING,nullable=true,nativeType=null,columnSize=null], "
+                        + "Column[name=Level,columnNumber=3,type=STRING,nullable=true,nativeType=null,columnSize=null]]",
                 Arrays.toString(table.getColumns().toArray()));
     }