diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingContext.java b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingContext.java new file mode 100644 index 000000000..05b523990 --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingContext.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + +import org.apache.metamodel.schema.Table; + + +/** + * Defines the context for configuring the type for a single column in a + * {@link ColumnTypingStrategy} session. + */ +public class ColumnTypingContext { + + private final int columnIndex; + + private final Table table; + + + /** + * Creates a context to conifgure a column for a specific table. + * + * @param table The table that contains the column + * @param columnIndex the index in the table of the column being configured. + */ + public ColumnTypingContext(final Table table, final int columnIndex) { + this.table = table; + this.columnIndex = columnIndex; + } + + + /** + * Creates a context a column to be configured. + * + * @param columnIndex the index in the table of the column being configured. + */ + public ColumnTypingContext(final int columnIndex) { + this(null, columnIndex); + } + + + /** + * Gets the index of the column being configured. + * + * @return the column index + */ + public int getColumnIndex() { + return columnIndex; + } + + + /** + * Gets the {@link Table} that the column is to pertain to. If the table is + * not yet available then this may return null. + * + * @return the associated table + */ + public Table getTable() { + return table; + } + +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingSession.java b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingSession.java new file mode 100644 index 000000000..2610ad516 --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingSession.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + + +import org.apache.metamodel.schema.Column; +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.schema.Table; + +import java.io.Closeable; + +/** + * Represents a 'session' in which a single {@link Table}'s {@link Column}s are + * assigned {@link ColumnType}s. + */ +public interface ColumnTypingSession extends Closeable { + + /** + * Provides the type to apply for a given column. + * + * @param ctx the context of the column naming taking place. This contains + * column index, intrinsic type etc. if available. + * @return the type to provide to the column. + */ + ColumnType getNextColumnType(ColumnTypingContext ctx); + + /** + * Ends the column typing session. + */ + @Override + default void close() { + } +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategies.java b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategies.java new file mode 100644 index 000000000..489ca7aba --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategies.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + +import org.apache.metamodel.schema.ColumnType; + +import java.util.List; + +/** + * Constructors and common utilities for {@link ColumnTypingStrategy} objects. + */ +public class ColumnTypingStrategies { + + private static final DefaultColumnTypingStrategy DEFAULT_STRATEGY = new DefaultColumnTypingStrategy(); + + + private ColumnTypingStrategies() { + } + + + /** + * The default strategy assumes all column types are text. + * + * @return a strategy that defaults to text + */ + public static ColumnTypingStrategy getDefaultStrategy() { + return DEFAULT_STRATEGY; + } + + + /** + * Utility to create a {@link CustomColumnTypingStrategy}. + * + * @param columnTypes the types of each column + * @return a strategy for custom type configuration + */ + public static ColumnTypingStrategy getCustomStrategy(final List columnTypes) { + return new CustomColumnTypingStrategy(columnTypes); + } + + + /** + * Utility to create a {@link CustomColumnTypingStrategy}. + * + * @param columnTypes the types of each column + * @return a strategy for custom type configuration + */ + public static ColumnTypingStrategy getCustomStrategy(final ColumnType... columnTypes) { + return new CustomColumnTypingStrategy(columnTypes); + } +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategy.java b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategy.java new file mode 100644 index 000000000..c062f6e2f --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/ColumnTypingStrategy.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + + +import java.io.Serializable; + +/** + * A strategy that defines the data types of columns. Such strategies are + * mostly used when a particular datastore is not itself intrinsically + * specifying the column type. + */ +public interface ColumnTypingStrategy extends Serializable { + + ColumnTypingSession startColumnTypingSession(); + +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategy.java b/core/src/main/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategy.java new file mode 100644 index 000000000..093ac9e79 --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategy.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + +import org.apache.metamodel.schema.ColumnType; +import org.apache.metamodel.util.SimpleTableDef; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +/** + * A {@link ColumnTypingStrategy} that allows the user to supply his own list of column types + */ +public class CustomColumnTypingStrategy implements ColumnTypingStrategy { + + private static final long serialVersionUID = 1L; + + private final List columnTypes; + + + /** + * Creates the strategy based on the provided types. + * + * @param columnTypes a list of column types to be applied to a table. + */ + public CustomColumnTypingStrategy(final List columnTypes) { + this.columnTypes = columnTypes; + } + + + /** + * Creates the strategy based on the provided types. + * + * @param columnTypes a list of column types to be applied to a table. + */ + public CustomColumnTypingStrategy(final ColumnType... columnTypes) { + this(Arrays.asList(columnTypes)); + } + + + public CustomColumnTypingStrategy(final SimpleTableDef tableDef) { + this(Arrays.asList(tableDef.getColumnTypes())); + } + + + @Override + public ColumnTypingSession startColumnTypingSession() { + final Iterator iterator = columnTypes.iterator(); + return ctx -> { + if (iterator.hasNext()) { + return iterator.next(); + } + return null; + }; + } +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/DefaultColumnTypingStrategy.java b/core/src/main/java/org/apache/metamodel/schema/typing/DefaultColumnTypingStrategy.java new file mode 100644 index 000000000..ff5ed0dbb --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/DefaultColumnTypingStrategy.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + +import org.apache.metamodel.schema.ColumnType; + +public class DefaultColumnTypingStrategy implements ColumnTypingStrategy { + + private static final long serialVersionUID = 1L; + + + public DefaultColumnTypingStrategy() { + } + + + @Override + public ColumnTypingSession startColumnTypingSession() { + return ctx -> ColumnType.STRING; + } +} diff --git a/core/src/main/java/org/apache/metamodel/schema/typing/package-info.java b/core/src/main/java/org/apache/metamodel/schema/typing/package-info.java new file mode 100644 index 000000000..098b708bf --- /dev/null +++ b/core/src/main/java/org/apache/metamodel/schema/typing/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * API for column data type configuration + */ +package org.apache.metamodel.schema.typing; + diff --git a/core/src/test/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategyTest.java b/core/src/test/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategyTest.java new file mode 100644 index 000000000..2bed81b66 --- /dev/null +++ b/core/src/test/java/org/apache/metamodel/schema/typing/CustomColumnTypingStrategyTest.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.metamodel.schema.typing; + +import org.apache.metamodel.schema.ColumnType; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class CustomColumnTypingStrategyTest { + + @Test + public void testTypeConfiguration() throws Exception { + ColumnTypingStrategy strategy = ColumnTypingStrategies.getCustomStrategy(ColumnType.STRING, ColumnType.NUMBER); + try (ColumnTypingSession session = strategy.startColumnTypingSession()) { + assertEquals(ColumnType.STRING, session.getNextColumnType(new ColumnTypingContext(0))); + assertEquals(ColumnType.NUMBER, session.getNextColumnType(new ColumnTypingContext(1))); + } + } +} diff --git a/csv/src/main/java/org/apache/metamodel/csv/CsvConfiguration.java b/csv/src/main/java/org/apache/metamodel/csv/CsvConfiguration.java index 332ef4b66..82230d4de 100644 --- a/csv/src/main/java/org/apache/metamodel/csv/CsvConfiguration.java +++ b/csv/src/main/java/org/apache/metamodel/csv/CsvConfiguration.java @@ -23,6 +23,8 @@ import org.apache.metamodel.schema.naming.ColumnNamingStrategies; import org.apache.metamodel.schema.naming.ColumnNamingStrategy; +import org.apache.metamodel.schema.typing.ColumnTypingStrategies; +import org.apache.metamodel.schema.typing.ColumnTypingStrategy; import org.apache.metamodel.util.BaseObject; import org.apache.metamodel.util.FileHelper; @@ -53,6 +55,7 @@ public final class CsvConfiguration extends BaseObject implements Serializable { private final boolean failOnInconsistentRowLength; private final boolean multilineValues; private final ColumnNamingStrategy columnNamingStrategy; + private final ColumnTypingStrategy columnTypingStrategy; public CsvConfiguration() { this(DEFAULT_COLUMN_NAME_LINE); @@ -80,13 +83,15 @@ public CsvConfiguration(int columnNameLineNumber, String encoding, char separato public CsvConfiguration(int columnNameLineNumber, String encoding, char separatorChar, char quoteChar, char escapeChar, boolean failOnInconsistentRowLength, boolean multilineValues) { - this(columnNameLineNumber, null, encoding, separatorChar, quoteChar, escapeChar, failOnInconsistentRowLength, - multilineValues); + this(columnNameLineNumber, null, null, encoding, separatorChar, quoteChar, escapeChar, + failOnInconsistentRowLength, multilineValues); } - public CsvConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, String encoding, - char separatorChar, char quoteChar, char escapeChar, boolean failOnInconsistentRowLength, - boolean multilineValues) { + + public CsvConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNamingStrategy, + ColumnTypingStrategy columnTypingStrategy, String encoding, char separatorChar, + char quoteChar, char escapeChar, boolean failOnInconsistentRowLength, + boolean multilineValues) { this.columnNameLineNumber = columnNameLineNumber; this.encoding = encoding; this.separatorChar = separatorChar; @@ -95,6 +100,7 @@ public CsvConfiguration(int columnNameLineNumber, ColumnNamingStrategy columnNam this.failOnInconsistentRowLength = failOnInconsistentRowLength; this.multilineValues = multilineValues; this.columnNamingStrategy = columnNamingStrategy; + this.columnTypingStrategy = columnTypingStrategy; } /** @@ -108,6 +114,17 @@ public ColumnNamingStrategy getColumnNamingStrategy() { return columnNamingStrategy; } + /** + * Gets a {@link ColumnTypingStrategy} to use if needed. + * @return The configured strategy, or {@link ColumnTypingStrategies#getDefaultStrategy}. + */ + public ColumnTypingStrategy getColumnTypingStrategy() { + if (columnTypingStrategy == null) { + return ColumnTypingStrategies.getDefaultStrategy(); + } + return columnTypingStrategy; + } + /** * Determines whether to fail (by throwing an * {@link InconsistentRowLengthException}) if a line in the CSV file has diff --git a/csv/src/main/java/org/apache/metamodel/csv/CsvDataContextFactory.java b/csv/src/main/java/org/apache/metamodel/csv/CsvDataContextFactory.java index 8f1afb09d..412de688a 100644 --- a/csv/src/main/java/org/apache/metamodel/csv/CsvDataContextFactory.java +++ b/csv/src/main/java/org/apache/metamodel/csv/CsvDataContextFactory.java @@ -24,6 +24,8 @@ import org.apache.metamodel.factory.ResourceFactoryRegistry; import org.apache.metamodel.schema.naming.ColumnNamingStrategy; import org.apache.metamodel.schema.naming.CustomColumnNamingStrategy; +import org.apache.metamodel.schema.typing.ColumnTypingStrategy; +import org.apache.metamodel.schema.typing.CustomColumnTypingStrategy; import org.apache.metamodel.util.FileHelper; import org.apache.metamodel.util.Resource; import org.apache.metamodel.util.SimpleTableDef; @@ -59,7 +61,15 @@ public DataContext create(DataContextProperties properties, ResourceFactoryRegis columnNamingStrategy = new CustomColumnNamingStrategy(columnNames); } - final CsvConfiguration configuration = new CsvConfiguration(columnNameLineNumber, columnNamingStrategy, + final ColumnTypingStrategy columnTypingStrategy; + if (properties.getTableDefs() == null) { + columnTypingStrategy = null; + } else { + final SimpleTableDef firstTable = properties.getTableDefs()[0]; + columnTypingStrategy = new CustomColumnTypingStrategy(firstTable); + } + + final CsvConfiguration configuration = new CsvConfiguration(columnNameLineNumber, columnNamingStrategy, columnTypingStrategy, encoding, separatorChar, quoteChar, escapeChar, failOnInconsistentRowLength, multilineValuesEnabled); return new CsvDataContext(resource, configuration); } diff --git a/csv/src/main/java/org/apache/metamodel/csv/CsvDataSet.java b/csv/src/main/java/org/apache/metamodel/csv/CsvDataSet.java index 779da7f1f..39c8755de 100644 --- a/csv/src/main/java/org/apache/metamodel/csv/CsvDataSet.java +++ b/csv/src/main/java/org/apache/metamodel/csv/CsvDataSet.java @@ -23,11 +23,15 @@ import java.util.stream.Collectors; import org.apache.metamodel.MetaModelException; +import org.apache.metamodel.convert.StringToBooleanConverter; +import org.apache.metamodel.convert.StringToDateConverter; +import org.apache.metamodel.convert.StringToDoubleConverter; import org.apache.metamodel.data.AbstractDataSet; import org.apache.metamodel.data.DefaultRow; import org.apache.metamodel.data.Row; import org.apache.metamodel.query.SelectItem; import org.apache.metamodel.schema.Column; +import org.apache.metamodel.schema.SuperColumnType; import org.apache.metamodel.util.FileHelper; import com.opencsv.CSVReader; @@ -110,7 +114,7 @@ private boolean nextInternal() { Column column = getHeader().getSelectItem(i).getColumn(); int columnNumber = column.getColumnNumber(); if (columnNumber < csvValues.length) { - rowValues[i] = csvValues[columnNumber]; + rowValues[i] = getTypedValue(csvValues[columnNumber], column.getType().getSuperType()); } else { // Ticket #125: Missing values should be enterpreted as // null. @@ -128,4 +132,17 @@ private boolean nextInternal() { return true; } + + private Object getTypedValue(String value, SuperColumnType columnType) { + switch (columnType) { + case BOOLEAN_TYPE: + return new StringToBooleanConverter().toVirtualValue(value); + case NUMBER_TYPE: + return new StringToDoubleConverter().toVirtualValue(value); + case TIME_TYPE: + return new StringToDateConverter().toVirtualValue(value); + default: + return value; + } + } } \ No newline at end of file diff --git a/csv/src/main/java/org/apache/metamodel/csv/CsvTable.java b/csv/src/main/java/org/apache/metamodel/csv/CsvTable.java index 5e0081c99..91e4232db 100644 --- a/csv/src/main/java/org/apache/metamodel/csv/CsvTable.java +++ b/csv/src/main/java/org/apache/metamodel/csv/CsvTable.java @@ -38,6 +38,9 @@ import org.apache.metamodel.schema.naming.ColumnNamingContextImpl; import org.apache.metamodel.schema.naming.ColumnNamingSession; import org.apache.metamodel.schema.naming.ColumnNamingStrategy; +import org.apache.metamodel.schema.typing.ColumnTypingContext; +import org.apache.metamodel.schema.typing.ColumnTypingSession; +import org.apache.metamodel.schema.typing.ColumnTypingStrategy; import org.apache.metamodel.util.FileHelper; import org.apache.metamodel.util.LegacyDeserializationObjectInputStream; @@ -125,17 +128,22 @@ private List buildColumns(final List columnNames) { final int columnNameLineNumber = configuration.getColumnNameLineNumber(); final boolean nullable = !configuration.isFailOnInconsistentRowLength(); final ColumnNamingStrategy columnNamingStrategy = configuration.getColumnNamingStrategy(); + final ColumnTypingStrategy columnTypingStrategy = configuration.getColumnTypingStrategy(); List columns = new ArrayList<>(); - try (final ColumnNamingSession namingSession = columnNamingStrategy.startColumnNamingSession()) { + try (final ColumnNamingSession namingSession = columnNamingStrategy.startColumnNamingSession(); + final ColumnTypingSession typingSession = columnTypingStrategy.startColumnTypingSession()) { for (int i = 0; i < columnNames.size(); i++) { final String intrinsicColumnName = columnNameLineNumber == CsvConfiguration.NO_COLUMN_NAME_LINE ? null : columnNames.get(i); final String columnName = namingSession.getNextColumnName(new ColumnNamingContextImpl(this, intrinsicColumnName, i)); - final Column column = new MutableColumn(columnName, ColumnType.STRING, this, i, null, null, nullable, - null, false, null); + + final ColumnType columnType = typingSession.getNextColumnType(new ColumnTypingContext(i)); + + final Column column = new MutableColumn( columnName, columnType, this, i, null, null, nullable, null, + false, null ); columns.add(column); } } diff --git a/csv/src/test/java/org/apache/metamodel/csv/CsvDataContextTest.java b/csv/src/test/java/org/apache/metamodel/csv/CsvDataContextTest.java index 23a140f47..910860b25 100644 --- a/csv/src/test/java/org/apache/metamodel/csv/CsvDataContextTest.java +++ b/csv/src/test/java/org/apache/metamodel/csv/CsvDataContextTest.java @@ -42,11 +42,9 @@ import org.apache.metamodel.query.OperatorType; import org.apache.metamodel.query.Query; import org.apache.metamodel.query.SelectItem; -import org.apache.metamodel.schema.Column; -import org.apache.metamodel.schema.MutableColumn; -import org.apache.metamodel.schema.Schema; -import org.apache.metamodel.schema.Table; +import org.apache.metamodel.schema.*; import org.apache.metamodel.schema.naming.CustomColumnNamingStrategy; +import org.apache.metamodel.schema.typing.CustomColumnTypingStrategy; import org.apache.metamodel.util.FileHelper; import org.apache.metamodel.util.MutableRef; @@ -492,7 +490,7 @@ public void testMaterializeTable() throws Exception { public void testAlternativeDelimitors() throws Exception { File file = new File("src/test/resources/csv_semicolon_singlequote.csv"); CsvDataContext dc = new CsvDataContext(file, semicolonConfiguration); - Table table = dc.getSchemas().get(0).getTables().get(0); + Table table = dc.getDefaultSchema().getTables().get(0); DataSet dataSet = dc.materializeMainSchemaTable(table, table.getColumns(), -1); assertTrue(dataSet.next()); assertEquals("Row[values=[1, mike, male, 18]]", dataSet.getRow().toString()); @@ -835,6 +833,7 @@ public void testCustomColumnNames() throws Exception { final CsvConfiguration configuration = new CsvConfiguration(CsvConfiguration.DEFAULT_COLUMN_NAME_LINE, new CustomColumnNamingStrategy(firstColumnName, secondColumnName, thirdColumnName, fourthColumnName), + new CustomColumnTypingStrategy(ColumnType.STRING, ColumnType.STRING, ColumnType.STRING, ColumnType.NUMBER), FileHelper.DEFAULT_ENCODING, CsvConfiguration.DEFAULT_SEPARATOR_CHAR, CsvConfiguration.DEFAULT_QUOTE_CHAR, CsvConfiguration.DEFAULT_ESCAPE_CHAR, false, true); diff --git a/csv/src/test/java/org/apache/metamodel/csv/SingleLineCsvDataSetTest.java b/csv/src/test/java/org/apache/metamodel/csv/SingleLineCsvDataSetTest.java index 2de30d47e..34ab8c20d 100644 --- a/csv/src/test/java/org/apache/metamodel/csv/SingleLineCsvDataSetTest.java +++ b/csv/src/test/java/org/apache/metamodel/csv/SingleLineCsvDataSetTest.java @@ -19,12 +19,18 @@ package org.apache.metamodel.csv; import java.io.File; +import java.text.DateFormat; import java.util.Arrays; +import org.apache.metamodel.DataContext; import org.apache.metamodel.data.DataSet; +import org.apache.metamodel.schema.ColumnType; import org.apache.metamodel.schema.Table; import junit.framework.TestCase; +import org.apache.metamodel.schema.typing.CustomColumnTypingStrategy; +import org.apache.metamodel.util.DateUtils; +import org.apache.metamodel.util.FileHelper; public class SingleLineCsvDataSetTest extends TestCase { @@ -64,4 +70,74 @@ public void testMalformedLineParsing() throws Exception { assertFalse(ds.next()); ds.close(); } -} \ No newline at end of file + + public void testCustomTyping() throws Exception { + + final CsvConfiguration configuration = new CsvConfiguration( CsvConfiguration.DEFAULT_COLUMN_NAME_LINE, null, + new CustomColumnTypingStrategy(ColumnType.DATE, ColumnType.TIME, ColumnType.STRING, ColumnType.NUMBER, + ColumnType.BOOLEAN), FileHelper.DEFAULT_ENCODING, CsvConfiguration.DEFAULT_SEPARATOR_CHAR, + CsvConfiguration.DEFAULT_QUOTE_CHAR, CsvConfiguration.DEFAULT_ESCAPE_CHAR, false, true ); + + final DataContext dc = new CsvDataContext( new File( "src/test/resources/csv_various_types.csv" ), configuration ); + final Table table = dc.getDefaultSchema().getTable(0); + + DateFormat dateFormat = DateUtils.createDateFormat( "yyyy-MM-dd" ); + DateFormat timeFormat = DateUtils.createDateFormat( "HH:mm" ); + + DataSet dataSet = dc.query().from(table).selectAll().execute(); + + assertTrue(dataSet.next()); + assertEquals(dateFormat.parse("2008-11-04"), dataSet.getRow().getValues()[0]); + assertEquals(timeFormat.parse("12:00"), dataSet.getRow().getValues()[1]); + assertEquals("election day", dataSet.getRow().getValues()[2]); + assertEquals(8.8, (Double) dataSet.getRow().getValues()[3], Math.ulp( 8.8 )); + assertFalse((Boolean) dataSet.getRow().getValues()[4]); + + assertTrue(dataSet.next()); + assertEquals(dateFormat.parse("2008-12-24"), dataSet.getRow().getValues()[0]); + assertEquals(timeFormat.parse("00:00"), dataSet.getRow().getValues()[1]); + assertEquals("christmas day", dataSet.getRow().getValues()[2]); + assertEquals(9.0, (Double) dataSet.getRow().getValues()[3], Math.ulp( 9.0 )); + assertTrue((Boolean) dataSet.getRow().getValues()[4]); + + assertTrue(dataSet.next()); + assertEquals(dateFormat.parse( "2007-12-31"), dataSet.getRow().getValues()[0]); + assertEquals(timeFormat.parse( "23:59"), dataSet.getRow().getValues()[1]); + assertEquals("new years eve", dataSet.getRow().getValues()[2]); + assertEquals(6.4, (Double) dataSet.getRow().getValues()[3], Math.ulp( 6.4 )); + assertNull(dataSet.getRow().getValues()[4]); + } + + + public void testTypeBasedFiltering() throws Exception { + File dataFile = new File("src/test/resources/csv_various_types.csv"); + + final CsvConfiguration typedConfiguration = new CsvConfiguration(CsvConfiguration.DEFAULT_COLUMN_NAME_LINE, + null, new CustomColumnTypingStrategy(ColumnType.DATE, ColumnType.TIME, ColumnType.STRING, + ColumnType.NUMBER, ColumnType.BOOLEAN), FileHelper.DEFAULT_ENCODING, + CsvConfiguration.DEFAULT_SEPARATOR_CHAR, CsvConfiguration.DEFAULT_QUOTE_CHAR, + CsvConfiguration.DEFAULT_ESCAPE_CHAR, false, true); + final DataContext typedDc = new CsvDataContext(dataFile, + typedConfiguration); + final Table typedTable = typedDc.getDefaultSchema().getTable(0); + + DataSet typedDataSet = + typedDc.query().from(typedTable).selectCount().where("rating").greaterThan(10.0).execute(); + assertTrue(typedDataSet.next()); + assertEquals(0L, typedDataSet.getRow().getValue(0)); + + // Same query and data set as above, but force toString based comparisons of values for a different result. + final CsvConfiguration untypedConfiguration = new CsvConfiguration(CsvConfiguration.DEFAULT_COLUMN_NAME_LINE, + null, null, FileHelper.DEFAULT_ENCODING, CsvConfiguration.DEFAULT_SEPARATOR_CHAR, + CsvConfiguration.DEFAULT_QUOTE_CHAR, CsvConfiguration.DEFAULT_ESCAPE_CHAR, false, true); + + final DataContext untypedDataContext = + new CsvDataContext(dataFile, untypedConfiguration); + final Table untypedTable = untypedDataContext.getDefaultSchema().getTable(0); + + DataSet untypedDataSet = + untypedDataContext.query().from(untypedTable).selectCount().where("rating").greaterThan(10.0).execute(); + assertTrue(untypedDataSet.next()); + assertEquals(3L, untypedDataSet.getRow().getValue(0)); + } +}