From e2aa86e50f5211ef126ea2e11d7c7031969cacaa Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Tue, 24 Feb 2026 17:46:44 +0800 Subject: [PATCH 1/4] [feature](iceberg) support fs.s3a aliases for S3/MinIO storage props --- .../property/storage/MinioProperties.java | 28 ++++++++---- .../property/storage/S3Properties.java | 28 ++++++------ .../property/storage/MinioPropertiesTest.java | 43 ++++++++++++++++++- .../property/storage/S3PropertiesTest.java | 28 ++++++++++++ 4 files changed, 103 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java index 73969b99c6426d..4bacbd429322dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -31,16 +31,20 @@ public class MinioProperties extends AbstractS3CompatibleProperties { @Setter @Getter - @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"}, + @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", + "fs.s3a.endpoint"}, required = false, description = "The endpoint of Minio.") protected String endpoint = ""; @Getter @Setter + @ConnectorProperty(names = {"minio.region", "s3.region", "AWS_REGION", "region", "REGION", + "fs.s3a.endpoint.region"}, + required = false, description = "The region of Minio.") protected String region = "us-east-1"; @Getter @ConnectorProperty(names = {"minio.access_key", "s3.access-key-id", "AWS_ACCESS_KEY", "ACCESS_KEY", - "access_key", "s3.access_key"}, + "access_key", "s3.access_key", "fs.s3a.access.key"}, required = false, sensitive = true, description = "The access key of Minio.") @@ -48,14 +52,15 @@ public class MinioProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"minio.secret_key", "s3.secret-access-key", "s3.secret_key", "AWS_SECRET_KEY", - "secret_key", "SECRET_KEY"}, + "secret_key", "SECRET_KEY", "fs.s3a.secret.key"}, required = false, sensitive = true, description = "The secret key of Minio.") protected String secretKey = ""; @Getter - @ConnectorProperty(names = {"minio.session_token", "s3.session-token", "s3.session_token", "session_token"}, + @ConnectorProperty(names = {"minio.session_token", "s3.session-token", "s3.session_token", "session_token", + "fs.s3a.session.token"}, required = false, sensitive = true, description = "The session token of Minio.") @@ -66,7 +71,8 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.maximum", "s3.connection.maximum"}, required = false, + @ConnectorProperty(names = {"minio.connection.maximum", "s3.connection.maximum", "fs.s3a.connection.maximum"}, + required = false, description = "Maximum number of connections.") protected String maxConnections = "100"; @@ -75,7 +81,8 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.request.timeout", "s3.connection.request.timeout"}, required = false, + @ConnectorProperty(names = {"minio.connection.request.timeout", "s3.connection.request.timeout", + "fs.s3a.connection.request.timeout"}, required = false, description = "Request timeout in seconds.") protected String requestTimeoutS = "10000"; @@ -84,7 +91,8 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.timeout", "s3.connection.timeout"}, required = false, + @ConnectorProperty(names = {"minio.connection.timeout", "s3.connection.timeout", "fs.s3a.connection.timeout"}, + required = false, description = "Connection timeout in seconds.") protected String connectionTimeoutS = "10000"; @@ -94,7 +102,8 @@ public class MinioProperties extends AbstractS3CompatibleProperties { */ @Setter @Getter - @ConnectorProperty(names = {"minio.use_path_style", "use_path_style", "s3.path-style-access"}, required = false, + @ConnectorProperty(names = {"minio.use_path_style", "use_path_style", "s3.path-style-access", + "fs.s3a.path.style.access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "false"; @@ -106,7 +115,8 @@ public class MinioProperties extends AbstractS3CompatibleProperties { protected String forceParsingByStandardUrl = "false"; private static final Set IDENTIFIERS = ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", - "access_key", "s3.access_key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"); + "access_key", "s3.access_key", "fs.s3a.access.key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", + "endpoint", "ENDPOINT", "fs.s3a.endpoint", "fs.s3a.secret.key"); /** * Constructor to initialize the object storage properties with the provided type and original properties map. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index df09803be61766..e0699f1799860e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -61,17 +61,18 @@ public class S3Properties extends AbstractS3CompatibleProperties { private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", - "aws.glue.endpoint" + "aws.glue.endpoint", "fs.s3a.endpoint" }; private static final String[] REGION_NAMES_FOR_GUESSING = { - "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region", "client.region" + "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region", "client.region", + "fs.s3a.endpoint.region" }; @Setter @Getter @ConnectorProperty(names = {"s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", - "aws.glue.endpoint"}, + "aws.glue.endpoint", "fs.s3a.endpoint"}, required = false, description = "The endpoint of S3.") protected String endpoint = ""; @@ -79,7 +80,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Setter @Getter @ConnectorProperty(names = {"s3.region", "AWS_REGION", "region", "REGION", "aws.region", "glue.region", - "aws.glue.region", "iceberg.rest.signing-region", "client.region"}, + "aws.glue.region", "iceberg.rest.signing-region", "client.region", "fs.s3a.endpoint.region"}, required = false, description = "The region of S3.") protected String region = ""; @@ -87,7 +88,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY", "glue.access_key", "aws.glue.access-key", "client.credentials-provider.glue.access_key", "iceberg.rest.access-key-id", - "s3.access-key-id"}, + "s3.access-key-id", "fs.s3a.access.key"}, required = false, sensitive = true, description = "The access key of S3. Optional for anonymous access to public datasets.") @@ -96,14 +97,14 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", "glue.secret_key", "aws.glue.secret-key", "client.credentials-provider.glue.secret_key", "iceberg.rest.secret-access-key", - "s3.secret-access-key"}, + "s3.secret-access-key", "fs.s3a.secret.key"}, required = false, sensitive = true, description = "The secret key of S3. Optional for anonymous access to public datasets.") protected String secretKey = ""; @Getter - @ConnectorProperty(names = {"s3.session_token", "session_token", "s3.session-token"}, + @ConnectorProperty(names = {"s3.session_token", "session_token", "s3.session-token", "fs.s3a.session.token"}, required = false, description = "The session token of S3.") protected String sessionToken = ""; @@ -115,29 +116,28 @@ public class S3Properties extends AbstractS3CompatibleProperties { protected String sessionTokenExpiresAtMs = ""; @Getter - @ConnectorProperty(names = {"s3.connection.maximum", - "AWS_MAX_CONNECTIONS"}, + @ConnectorProperty(names = {"s3.connection.maximum", "AWS_MAX_CONNECTIONS", "fs.s3a.connection.maximum"}, required = false, description = "The maximum number of connections to S3.") protected String maxConnections = "50"; @Getter - @ConnectorProperty(names = {"s3.connection.request.timeout", - "AWS_REQUEST_TIMEOUT_MS"}, + @ConnectorProperty(names = {"s3.connection.request.timeout", "AWS_REQUEST_TIMEOUT_MS", + "fs.s3a.connection.request.timeout"}, required = false, description = "The request timeout of S3 in milliseconds,") protected String requestTimeoutS = "3000"; @Getter - @ConnectorProperty(names = {"s3.connection.timeout", - "AWS_CONNECTION_TIMEOUT_MS"}, + @ConnectorProperty(names = {"s3.connection.timeout", "AWS_CONNECTION_TIMEOUT_MS", + "fs.s3a.connection.timeout"}, required = false, description = "The connection timeout of S3 in milliseconds,") protected String connectionTimeoutS = "1000"; @Setter @Getter - @ConnectorProperty(names = {USE_PATH_STYLE, "s3.path-style-access"}, required = false, + @ConnectorProperty(names = {USE_PATH_STYLE, "s3.path-style-access", "fs.s3a.path.style.access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "false"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java index bd6fbf2239a17a..542e1f8d483383 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import java.util.HashMap; +import java.util.List; import java.util.Map; public class MinioPropertiesTest { @@ -108,5 +109,45 @@ public void testBackendConfigProperties() { Assertions.assertEquals("minioSecretKey", backendProps.get("AWS_SECRET_KEY")); Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); } -} + @Test + public void testFsS3aAliasPropertiesForMinio() { + origProps.put("fs.s3a.endpoint", "http://localhost:9878"); + origProps.put("fs.s3a.access.key", "hadoop"); + origProps.put("fs.s3a.secret.key", "hadoop"); + origProps.put("fs.s3a.endpoint.region", "us-east-1"); + origProps.put("fs.s3a.path.style.access", "true"); + + MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps); + Map backendProps = minioProperties.getBackendConfigProperties(); + + Assertions.assertEquals("http://localhost:9878", minioProperties.getEndpoint()); + Assertions.assertEquals("hadoop", minioProperties.getAccessKey()); + Assertions.assertEquals("hadoop", minioProperties.getSecretKey()); + Assertions.assertEquals("us-east-1", minioProperties.getRegion()); + Assertions.assertEquals("http://localhost:9878", backendProps.get("AWS_ENDPOINT")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_SECRET_KEY")); + Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); + Assertions.assertEquals("true", backendProps.get("use_path_style")); + } + + @Test + public void testFsS3aAliasPropertiesWithDefaultFs() throws UserException { + origProps.put("fs.defaultFS", "s3a://dn-data/"); + origProps.put("fs.s3a.endpoint", "http://localhost:9878"); + origProps.put("fs.s3a.access.key", "hadoop"); + origProps.put("fs.s3a.secret.key", "hadoop"); + + List allProperties = StorageProperties.createAll(origProps); + MinioProperties minioProperties = allProperties.stream() + .filter(MinioProperties.class::isInstance) + .map(MinioProperties.class::cast) + .findFirst() + .orElse(null); + + Assertions.assertNotNull(minioProperties); + Assertions.assertEquals("http://localhost:9878", + minioProperties.getBackendConfigProperties().get("AWS_ENDPOINT")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index aa1c95b433d142..d09f93abfea65a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -495,4 +495,32 @@ public void testS3DisableHadoopCache() throws UserException { Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3.impl.disable.cache")); Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3n.impl.disable.cache")); } + + @Test + public void testFsS3aAliasPropertiesForS3() throws UserException { + Map props = Maps.newHashMap(); + props.put("fs.s3a.endpoint", "s3.us-west-2.amazonaws.com"); + props.put("fs.s3a.endpoint.region", "us-west-2"); + props.put("fs.s3a.access.key", "myS3AccessKey"); + props.put("fs.s3a.secret.key", "myS3SecretKey"); + props.put("fs.s3a.session.token", "myS3SessionToken"); + props.put("fs.s3a.path.style.access", "true"); + props.put("fs.s3a.connection.maximum", "120"); + + S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(props); + Map backendProps = s3Properties.getBackendConfigProperties(); + + Assertions.assertEquals("s3.us-west-2.amazonaws.com", s3Properties.getEndpoint()); + Assertions.assertEquals("us-west-2", s3Properties.getRegion()); + Assertions.assertEquals("myS3AccessKey", s3Properties.getAccessKey()); + Assertions.assertEquals("myS3SecretKey", s3Properties.getSecretKey()); + Assertions.assertEquals("myS3SessionToken", s3Properties.getSessionToken()); + Assertions.assertEquals("s3.us-west-2.amazonaws.com", backendProps.get("AWS_ENDPOINT")); + Assertions.assertEquals("us-west-2", backendProps.get("AWS_REGION")); + Assertions.assertEquals("myS3AccessKey", backendProps.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("myS3SecretKey", backendProps.get("AWS_SECRET_KEY")); + Assertions.assertEquals("myS3SessionToken", backendProps.get("AWS_TOKEN")); + Assertions.assertEquals("true", backendProps.get("use_path_style")); + Assertions.assertEquals("120", backendProps.get("AWS_MAX_CONNECTIONS")); + } } From 8f0fd5b42cbcc36462cd906685dc5275cd497d5c Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Wed, 25 Feb 2026 19:20:27 +0800 Subject: [PATCH 2/4] feat(storage): add dedicated ozone properties for iceberg s3a catalogs --- .../doris/common/util/LocationPath.java | 4 + .../property/storage/MinioProperties.java | 28 ++-- .../property/storage/OzoneProperties.java | 139 +++++++++++++++++ .../property/storage/S3Properties.java | 28 ++-- .../property/storage/StorageProperties.java | 4 + .../property/storage/MinioPropertiesTest.java | 43 +----- .../property/storage/OzonePropertiesTest.java | 143 ++++++++++++++++++ .../property/storage/S3PropertiesTest.java | 28 ---- 8 files changed, 314 insertions(+), 103 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java index 9bb64d09737c3e..1ce6576635de05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java @@ -313,6 +313,10 @@ private static StorageProperties findStorageProperties(StorageProperties.Type ty && storagePropertiesMap.containsKey(StorageProperties.Type.MINIO)) { return storagePropertiesMap.get(StorageProperties.Type.MINIO); } + if (type == StorageProperties.Type.S3 + && storagePropertiesMap.containsKey(StorageProperties.Type.OZONE)) { + return storagePropertiesMap.get(StorageProperties.Type.OZONE); + } // Step 3: Compatibility fallback based on schema // In previous configurations, the schema name may not strictly match the actual storage type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java index 4bacbd429322dd..73969b99c6426d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -31,20 +31,16 @@ public class MinioProperties extends AbstractS3CompatibleProperties { @Setter @Getter - @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", - "fs.s3a.endpoint"}, + @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"}, required = false, description = "The endpoint of Minio.") protected String endpoint = ""; @Getter @Setter - @ConnectorProperty(names = {"minio.region", "s3.region", "AWS_REGION", "region", "REGION", - "fs.s3a.endpoint.region"}, - required = false, description = "The region of Minio.") protected String region = "us-east-1"; @Getter @ConnectorProperty(names = {"minio.access_key", "s3.access-key-id", "AWS_ACCESS_KEY", "ACCESS_KEY", - "access_key", "s3.access_key", "fs.s3a.access.key"}, + "access_key", "s3.access_key"}, required = false, sensitive = true, description = "The access key of Minio.") @@ -52,15 +48,14 @@ public class MinioProperties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"minio.secret_key", "s3.secret-access-key", "s3.secret_key", "AWS_SECRET_KEY", - "secret_key", "SECRET_KEY", "fs.s3a.secret.key"}, + "secret_key", "SECRET_KEY"}, required = false, sensitive = true, description = "The secret key of Minio.") protected String secretKey = ""; @Getter - @ConnectorProperty(names = {"minio.session_token", "s3.session-token", "s3.session_token", "session_token", - "fs.s3a.session.token"}, + @ConnectorProperty(names = {"minio.session_token", "s3.session-token", "s3.session_token", "session_token"}, required = false, sensitive = true, description = "The session token of Minio.") @@ -71,8 +66,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.maximum", "s3.connection.maximum", "fs.s3a.connection.maximum"}, - required = false, + @ConnectorProperty(names = {"minio.connection.maximum", "s3.connection.maximum"}, required = false, description = "Maximum number of connections.") protected String maxConnections = "100"; @@ -81,8 +75,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.request.timeout", "s3.connection.request.timeout", - "fs.s3a.connection.request.timeout"}, required = false, + @ConnectorProperty(names = {"minio.connection.request.timeout", "s3.connection.request.timeout"}, required = false, description = "Request timeout in seconds.") protected String requestTimeoutS = "10000"; @@ -91,8 +84,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { * This value is optional and can be configured by the user. */ @Getter - @ConnectorProperty(names = {"minio.connection.timeout", "s3.connection.timeout", "fs.s3a.connection.timeout"}, - required = false, + @ConnectorProperty(names = {"minio.connection.timeout", "s3.connection.timeout"}, required = false, description = "Connection timeout in seconds.") protected String connectionTimeoutS = "10000"; @@ -102,8 +94,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { */ @Setter @Getter - @ConnectorProperty(names = {"minio.use_path_style", "use_path_style", "s3.path-style-access", - "fs.s3a.path.style.access"}, required = false, + @ConnectorProperty(names = {"minio.use_path_style", "use_path_style", "s3.path-style-access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "false"; @@ -115,8 +106,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { protected String forceParsingByStandardUrl = "false"; private static final Set IDENTIFIERS = ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", - "access_key", "s3.access_key", "fs.s3a.access.key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", - "endpoint", "ENDPOINT", "fs.s3a.endpoint", "fs.s3a.secret.key"); + "access_key", "s3.access_key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"); /** * Constructor to initialize the object storage properties with the provided type and original properties map. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java new file mode 100644 index 00000000000000..178d8a57f0ca43 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.storage; + +import org.apache.doris.datasource.property.ConnectorProperty; + +import com.google.common.collect.ImmutableSet; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class OzoneProperties extends AbstractS3CompatibleProperties { + + @Setter + @Getter + @ConnectorProperty(names = {"ozone.endpoint", "fs.s3a.endpoint"}, + required = false, + description = "The endpoint of Ozone S3 Gateway.") + protected String endpoint = ""; + + @Setter + @Getter + @ConnectorProperty(names = {"ozone.region", "fs.s3a.endpoint.region"}, + required = false, + description = "The region of Ozone S3 Gateway.") + protected String region = "us-east-1"; + + @Getter + @ConnectorProperty(names = {"ozone.access_key", "fs.s3a.access.key"}, + required = false, + sensitive = true, + description = "The access key of Ozone S3 Gateway.") + protected String accessKey = ""; + + @Getter + @ConnectorProperty(names = {"ozone.secret_key", "fs.s3a.secret.key"}, + required = false, + sensitive = true, + description = "The secret key of Ozone S3 Gateway.") + protected String secretKey = ""; + + @Getter + @ConnectorProperty(names = {"ozone.session_token", "fs.s3a.session.token"}, + required = false, + sensitive = true, + description = "The session token of Ozone S3 Gateway.") + protected String sessionToken = ""; + + @Getter + @ConnectorProperty(names = {"ozone.connection.maximum", "fs.s3a.connection.maximum"}, + required = false, + description = "Maximum number of connections.") + protected String maxConnections = "100"; + + @Getter + @ConnectorProperty(names = {"ozone.connection.request.timeout", "fs.s3a.connection.request.timeout"}, + required = false, + description = "Request timeout in seconds.") + protected String requestTimeoutS = "10000"; + + @Getter + @ConnectorProperty(names = {"ozone.connection.timeout", "fs.s3a.connection.timeout"}, + required = false, + description = "Connection timeout in seconds.") + protected String connectionTimeoutS = "10000"; + + @Setter + @Getter + @ConnectorProperty(names = {"ozone.use_path_style", "fs.s3a.path.style.access"}, + required = false, + description = "Whether to use path style URL for the storage.") + protected String usePathStyle = "true"; + + @Setter + @Getter + @ConnectorProperty(names = {"ozone.force_parsing_by_standard_uri"}, + required = false, + description = "Whether to use path style URL for the storage.") + protected String forceParsingByStandardUrl = "false"; + + private static final Set IDENTIFIERS = ImmutableSet.of( + "ozone.endpoint", + "ozone.access_key", + "ozone.secret_key"); + + protected OzoneProperties(Map origProps) { + super(Type.OZONE, origProps); + } + + protected static boolean guessIsMe(Map origProps) { + if (origProps == null || origProps.isEmpty()) { + return false; + } + if (IDENTIFIERS.stream().anyMatch(key -> StringUtils.isNotBlank(origProps.get(key)))) { + return true; + } + String endpoint = origProps.get("fs.s3a.endpoint"); + return StringUtils.isNotBlank(endpoint) + && (StringUtils.containsIgnoreCase(endpoint, "ozone") + || StringUtils.containsIgnoreCase(endpoint, "s3g")); + } + + @Override + protected Set endpointPatterns() { + return ImmutableSet.of(Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$")); + } + + @Override + protected void setEndpointIfPossible() { + super.setEndpointIfPossible(); + if (StringUtils.isBlank(getEndpoint())) { + throw new IllegalArgumentException("Property ozone.endpoint is required."); + } + } + + @Override + protected Set schemas() { + return ImmutableSet.of("s3", "s3a", "s3n"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index e0699f1799860e..df09803be61766 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -61,18 +61,17 @@ public class S3Properties extends AbstractS3CompatibleProperties { private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", - "aws.glue.endpoint", "fs.s3a.endpoint" + "aws.glue.endpoint" }; private static final String[] REGION_NAMES_FOR_GUESSING = { - "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region", "client.region", - "fs.s3a.endpoint.region" + "s3.region", "glue.region", "aws.glue.region", "iceberg.rest.signing-region", "client.region" }; @Setter @Getter @ConnectorProperty(names = {"s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", - "aws.glue.endpoint", "fs.s3a.endpoint"}, + "aws.glue.endpoint"}, required = false, description = "The endpoint of S3.") protected String endpoint = ""; @@ -80,7 +79,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Setter @Getter @ConnectorProperty(names = {"s3.region", "AWS_REGION", "region", "REGION", "aws.region", "glue.region", - "aws.glue.region", "iceberg.rest.signing-region", "client.region", "fs.s3a.endpoint.region"}, + "aws.glue.region", "iceberg.rest.signing-region", "client.region"}, required = false, description = "The region of S3.") protected String region = ""; @@ -88,7 +87,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY", "glue.access_key", "aws.glue.access-key", "client.credentials-provider.glue.access_key", "iceberg.rest.access-key-id", - "s3.access-key-id", "fs.s3a.access.key"}, + "s3.access-key-id"}, required = false, sensitive = true, description = "The access key of S3. Optional for anonymous access to public datasets.") @@ -97,14 +96,14 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Getter @ConnectorProperty(names = {"s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY", "glue.secret_key", "aws.glue.secret-key", "client.credentials-provider.glue.secret_key", "iceberg.rest.secret-access-key", - "s3.secret-access-key", "fs.s3a.secret.key"}, + "s3.secret-access-key"}, required = false, sensitive = true, description = "The secret key of S3. Optional for anonymous access to public datasets.") protected String secretKey = ""; @Getter - @ConnectorProperty(names = {"s3.session_token", "session_token", "s3.session-token", "fs.s3a.session.token"}, + @ConnectorProperty(names = {"s3.session_token", "session_token", "s3.session-token"}, required = false, description = "The session token of S3.") protected String sessionToken = ""; @@ -116,28 +115,29 @@ public class S3Properties extends AbstractS3CompatibleProperties { protected String sessionTokenExpiresAtMs = ""; @Getter - @ConnectorProperty(names = {"s3.connection.maximum", "AWS_MAX_CONNECTIONS", "fs.s3a.connection.maximum"}, + @ConnectorProperty(names = {"s3.connection.maximum", + "AWS_MAX_CONNECTIONS"}, required = false, description = "The maximum number of connections to S3.") protected String maxConnections = "50"; @Getter - @ConnectorProperty(names = {"s3.connection.request.timeout", "AWS_REQUEST_TIMEOUT_MS", - "fs.s3a.connection.request.timeout"}, + @ConnectorProperty(names = {"s3.connection.request.timeout", + "AWS_REQUEST_TIMEOUT_MS"}, required = false, description = "The request timeout of S3 in milliseconds,") protected String requestTimeoutS = "3000"; @Getter - @ConnectorProperty(names = {"s3.connection.timeout", "AWS_CONNECTION_TIMEOUT_MS", - "fs.s3a.connection.timeout"}, + @ConnectorProperty(names = {"s3.connection.timeout", + "AWS_CONNECTION_TIMEOUT_MS"}, required = false, description = "The connection timeout of S3 in milliseconds,") protected String connectionTimeoutS = "1000"; @Setter @Getter - @ConnectorProperty(names = {USE_PATH_STYLE, "s3.path-style-access", "fs.s3a.path.style.access"}, required = false, + @ConnectorProperty(names = {USE_PATH_STYLE, "s3.path-style-access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "false"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index 5de749f45d664c..c6fed3040baa3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -42,6 +42,7 @@ public abstract class StorageProperties extends ConnectionProperties { public static final String FS_S3_SUPPORT = "fs.s3.support"; public static final String FS_GCS_SUPPORT = "fs.gcs.support"; public static final String FS_MINIO_SUPPORT = "fs.minio.support"; + public static final String FS_OZONE_SUPPORT = "fs.ozone.support"; public static final String FS_BROKER_SUPPORT = "fs.broker.support"; public static final String FS_AZURE_SUPPORT = "fs.azure.support"; public static final String FS_OSS_SUPPORT = "fs.oss.support"; @@ -67,6 +68,7 @@ public enum Type { GCS, OSS_HDFS, MINIO, + OZONE, AZURE, BROKER, LOCAL, @@ -204,6 +206,8 @@ public static StorageProperties createPrimary(Map origProps) { || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null, props -> (isFsSupport(props, FS_MINIO_SUPPORT) || MinioProperties.guessIsMe(props)) ? new MinioProperties(props) : null, + props -> (isFsSupport(props, FS_OZONE_SUPPORT) + || OzoneProperties.guessIsMe(props)) ? new OzoneProperties(props) : null, props -> (isFsSupport(props, FS_BROKER_SUPPORT) || BrokerProperties.guessIsMe(props)) ? new BrokerProperties(props) : null, props -> (isFsSupport(props, FS_LOCAL_SUPPORT) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java index 542e1f8d483383..bd6fbf2239a17a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; import java.util.HashMap; -import java.util.List; import java.util.Map; public class MinioPropertiesTest { @@ -109,45 +108,5 @@ public void testBackendConfigProperties() { Assertions.assertEquals("minioSecretKey", backendProps.get("AWS_SECRET_KEY")); Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); } - - @Test - public void testFsS3aAliasPropertiesForMinio() { - origProps.put("fs.s3a.endpoint", "http://localhost:9878"); - origProps.put("fs.s3a.access.key", "hadoop"); - origProps.put("fs.s3a.secret.key", "hadoop"); - origProps.put("fs.s3a.endpoint.region", "us-east-1"); - origProps.put("fs.s3a.path.style.access", "true"); - - MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps); - Map backendProps = minioProperties.getBackendConfigProperties(); - - Assertions.assertEquals("http://localhost:9878", minioProperties.getEndpoint()); - Assertions.assertEquals("hadoop", minioProperties.getAccessKey()); - Assertions.assertEquals("hadoop", minioProperties.getSecretKey()); - Assertions.assertEquals("us-east-1", minioProperties.getRegion()); - Assertions.assertEquals("http://localhost:9878", backendProps.get("AWS_ENDPOINT")); - Assertions.assertEquals("hadoop", backendProps.get("AWS_ACCESS_KEY")); - Assertions.assertEquals("hadoop", backendProps.get("AWS_SECRET_KEY")); - Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); - Assertions.assertEquals("true", backendProps.get("use_path_style")); - } - - @Test - public void testFsS3aAliasPropertiesWithDefaultFs() throws UserException { - origProps.put("fs.defaultFS", "s3a://dn-data/"); - origProps.put("fs.s3a.endpoint", "http://localhost:9878"); - origProps.put("fs.s3a.access.key", "hadoop"); - origProps.put("fs.s3a.secret.key", "hadoop"); - - List allProperties = StorageProperties.createAll(origProps); - MinioProperties minioProperties = allProperties.stream() - .filter(MinioProperties.class::isInstance) - .map(MinioProperties.class::cast) - .findFirst() - .orElse(null); - - Assertions.assertNotNull(minioProperties); - Assertions.assertEquals("http://localhost:9878", - minioProperties.getBackendConfigProperties().get("AWS_ENDPOINT")); - } } + diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java new file mode 100644 index 00000000000000..3d6533a6806a64 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.storage; + +import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.LocationPath; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class OzonePropertiesTest { + private Map origProps; + + @BeforeEach + public void setup() { + origProps = new HashMap<>(); + } + + @Test + public void testValidOzoneConfiguration() { + origProps.put("ozone.endpoint", "http://ozone-s3g:9878"); + origProps.put("ozone.access_key", "hadoop"); + origProps.put("ozone.secret_key", "hadoop"); + + OzoneProperties ozoneProperties = (OzoneProperties) StorageProperties.createPrimary(origProps); + Map backendProps = ozoneProperties.getBackendConfigProperties(); + + Assertions.assertEquals(StorageProperties.Type.OZONE, ozoneProperties.getType()); + Assertions.assertEquals("http://ozone-s3g:9878", ozoneProperties.getEndpoint()); + Assertions.assertEquals("hadoop", ozoneProperties.getAccessKey()); + Assertions.assertEquals("hadoop", ozoneProperties.getSecretKey()); + Assertions.assertEquals("us-east-1", ozoneProperties.getRegion()); + Assertions.assertEquals("true", ozoneProperties.getUsePathStyle()); + + Assertions.assertEquals("http://ozone-s3g:9878", backendProps.get("AWS_ENDPOINT")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_SECRET_KEY")); + Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); + Assertions.assertEquals("true", backendProps.get("use_path_style")); + } + + @Test + public void testFsS3aPropertiesBinding() { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); + origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); + origProps.put("fs.s3a.access.key", "hadoop"); + origProps.put("fs.s3a.secret.key", "hadoop"); + origProps.put("fs.s3a.path.style.access", "true"); + origProps.put("fs.s3a.endpoint.region", "us-east-1"); + + OzoneProperties ozoneProperties = (OzoneProperties) StorageProperties.createPrimary(origProps); + Map backendProps = ozoneProperties.getBackendConfigProperties(); + + Assertions.assertEquals("http://ozone-s3g:9878", ozoneProperties.getEndpoint()); + Assertions.assertEquals("hadoop", ozoneProperties.getAccessKey()); + Assertions.assertEquals("hadoop", ozoneProperties.getSecretKey()); + Assertions.assertEquals("true", ozoneProperties.getUsePathStyle()); + + Assertions.assertEquals("http://ozone-s3g:9878", backendProps.get("AWS_ENDPOINT")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("hadoop", backendProps.get("AWS_SECRET_KEY")); + } + + @Test + public void testCreateAllWithDefaultFs() throws UserException { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); + origProps.put("fs.defaultFS", "s3a://dn-data/"); + origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); + origProps.put("fs.s3a.access.key", "hadoop"); + origProps.put("fs.s3a.secret.key", "hadoop"); + origProps.put("fs.s3a.path.style.access", "true"); + + List properties = StorageProperties.createAll(origProps); + Assertions.assertEquals(HdfsProperties.class, properties.get(0).getClass()); + Assertions.assertEquals(OzoneProperties.class, properties.get(1).getClass()); + + Map propertiesMap = properties.stream() + .collect(Collectors.toMap(StorageProperties::getType, Function.identity())); + LocationPath locationPath = LocationPath.of("s3a://dn-data/warehouse/test_table", propertiesMap); + Assertions.assertTrue(locationPath.getStorageProperties() instanceof OzoneProperties); + } + + @Test + public void testMissingAccessKeyOrSecretKey() { + origProps.put("ozone.endpoint", "http://ozone-s3g:9878"); + origProps.put("ozone.access_key", "hadoop"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Both the access key and the secret key must be set.", + () -> StorageProperties.createPrimary(origProps)); + + origProps.remove("ozone.access_key"); + origProps.put("ozone.secret_key", "hadoop"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Both the access key and the secret key must be set.", + () -> StorageProperties.createPrimary(origProps)); + } + + @Test + public void testMissingEndpoint() { + origProps.put("ozone.access_key", "hadoop"); + origProps.put("ozone.secret_key", "hadoop"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Property ozone.endpoint is required.", + () -> StorageProperties.createPrimary(origProps)); + } + + @Test + public void testGuessIsMe() { + origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); + Assertions.assertTrue(OzoneProperties.guessIsMe(origProps)); + + origProps.clear(); + origProps.put("fs.s3a.endpoint", "http://minio:9000"); + Assertions.assertFalse(OzoneProperties.guessIsMe(origProps)); + + origProps.clear(); + origProps.put("ozone.endpoint", "http://127.0.0.1:9878"); + Assertions.assertTrue(OzoneProperties.guessIsMe(origProps)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index d09f93abfea65a..aa1c95b433d142 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -495,32 +495,4 @@ public void testS3DisableHadoopCache() throws UserException { Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3.impl.disable.cache")); Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3n.impl.disable.cache")); } - - @Test - public void testFsS3aAliasPropertiesForS3() throws UserException { - Map props = Maps.newHashMap(); - props.put("fs.s3a.endpoint", "s3.us-west-2.amazonaws.com"); - props.put("fs.s3a.endpoint.region", "us-west-2"); - props.put("fs.s3a.access.key", "myS3AccessKey"); - props.put("fs.s3a.secret.key", "myS3SecretKey"); - props.put("fs.s3a.session.token", "myS3SessionToken"); - props.put("fs.s3a.path.style.access", "true"); - props.put("fs.s3a.connection.maximum", "120"); - - S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(props); - Map backendProps = s3Properties.getBackendConfigProperties(); - - Assertions.assertEquals("s3.us-west-2.amazonaws.com", s3Properties.getEndpoint()); - Assertions.assertEquals("us-west-2", s3Properties.getRegion()); - Assertions.assertEquals("myS3AccessKey", s3Properties.getAccessKey()); - Assertions.assertEquals("myS3SecretKey", s3Properties.getSecretKey()); - Assertions.assertEquals("myS3SessionToken", s3Properties.getSessionToken()); - Assertions.assertEquals("s3.us-west-2.amazonaws.com", backendProps.get("AWS_ENDPOINT")); - Assertions.assertEquals("us-west-2", backendProps.get("AWS_REGION")); - Assertions.assertEquals("myS3AccessKey", backendProps.get("AWS_ACCESS_KEY")); - Assertions.assertEquals("myS3SecretKey", backendProps.get("AWS_SECRET_KEY")); - Assertions.assertEquals("myS3SessionToken", backendProps.get("AWS_TOKEN")); - Assertions.assertEquals("true", backendProps.get("use_path_style")); - Assertions.assertEquals("120", backendProps.get("AWS_MAX_CONNECTIONS")); - } } From 318da5a9ca3d7b7217f5a3225a5368ae70bd5ee5 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Thu, 26 Feb 2026 18:27:26 +0800 Subject: [PATCH 3/4] [feature](storage) make ozone explicit and align iceberg catalog with s3.* props --- .../property/storage/OzoneProperties.java | 38 ++++--------- .../property/storage/StorageProperties.java | 6 +-- .../property/storage/OzonePropertiesTest.java | 54 ++++++++++++------- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java index 178d8a57f0ca43..8527cb73e78c46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java @@ -32,93 +32,75 @@ public class OzoneProperties extends AbstractS3CompatibleProperties { @Setter @Getter - @ConnectorProperty(names = {"ozone.endpoint", "fs.s3a.endpoint"}, + @ConnectorProperty(names = {"ozone.endpoint", "s3.endpoint"}, required = false, description = "The endpoint of Ozone S3 Gateway.") protected String endpoint = ""; @Setter @Getter - @ConnectorProperty(names = {"ozone.region", "fs.s3a.endpoint.region"}, + @ConnectorProperty(names = {"ozone.region", "s3.region"}, required = false, description = "The region of Ozone S3 Gateway.") protected String region = "us-east-1"; @Getter - @ConnectorProperty(names = {"ozone.access_key", "fs.s3a.access.key"}, + @ConnectorProperty(names = {"ozone.access_key", "s3.access_key", "s3.access-key-id"}, required = false, sensitive = true, description = "The access key of Ozone S3 Gateway.") protected String accessKey = ""; @Getter - @ConnectorProperty(names = {"ozone.secret_key", "fs.s3a.secret.key"}, + @ConnectorProperty(names = {"ozone.secret_key", "s3.secret_key", "s3.secret-access-key"}, required = false, sensitive = true, description = "The secret key of Ozone S3 Gateway.") protected String secretKey = ""; @Getter - @ConnectorProperty(names = {"ozone.session_token", "fs.s3a.session.token"}, + @ConnectorProperty(names = {"ozone.session_token", "s3.session_token", "s3.session-token"}, required = false, sensitive = true, description = "The session token of Ozone S3 Gateway.") protected String sessionToken = ""; @Getter - @ConnectorProperty(names = {"ozone.connection.maximum", "fs.s3a.connection.maximum"}, + @ConnectorProperty(names = {"ozone.connection.maximum", "s3.connection.maximum"}, required = false, description = "Maximum number of connections.") protected String maxConnections = "100"; @Getter - @ConnectorProperty(names = {"ozone.connection.request.timeout", "fs.s3a.connection.request.timeout"}, + @ConnectorProperty(names = {"ozone.connection.request.timeout", "s3.connection.request.timeout"}, required = false, description = "Request timeout in seconds.") protected String requestTimeoutS = "10000"; @Getter - @ConnectorProperty(names = {"ozone.connection.timeout", "fs.s3a.connection.timeout"}, + @ConnectorProperty(names = {"ozone.connection.timeout", "s3.connection.timeout"}, required = false, description = "Connection timeout in seconds.") protected String connectionTimeoutS = "10000"; @Setter @Getter - @ConnectorProperty(names = {"ozone.use_path_style", "fs.s3a.path.style.access"}, + @ConnectorProperty(names = {"ozone.use_path_style", "use_path_style", "s3.path-style-access"}, required = false, description = "Whether to use path style URL for the storage.") protected String usePathStyle = "true"; @Setter @Getter - @ConnectorProperty(names = {"ozone.force_parsing_by_standard_uri"}, + @ConnectorProperty(names = {"ozone.force_parsing_by_standard_uri", "force_parsing_by_standard_uri"}, required = false, description = "Whether to use path style URL for the storage.") protected String forceParsingByStandardUrl = "false"; - private static final Set IDENTIFIERS = ImmutableSet.of( - "ozone.endpoint", - "ozone.access_key", - "ozone.secret_key"); - protected OzoneProperties(Map origProps) { super(Type.OZONE, origProps); } - protected static boolean guessIsMe(Map origProps) { - if (origProps == null || origProps.isEmpty()) { - return false; - } - if (IDENTIFIERS.stream().anyMatch(key -> StringUtils.isNotBlank(origProps.get(key)))) { - return true; - } - String endpoint = origProps.get("fs.s3a.endpoint"); - return StringUtils.isNotBlank(endpoint) - && (StringUtils.containsIgnoreCase(endpoint, "ozone") - || StringUtils.containsIgnoreCase(endpoint, "s3g")); - } - @Override protected Set endpointPatterns() { return ImmutableSet.of(Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$")); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index c6fed3040baa3a..0464dabbc67446 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -205,9 +205,9 @@ public static StorageProperties createPrimary(Map origProps) { props -> (isFsSupport(props, FS_AZURE_SUPPORT) || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null, props -> (isFsSupport(props, FS_MINIO_SUPPORT) - || MinioProperties.guessIsMe(props)) ? new MinioProperties(props) : null, - props -> (isFsSupport(props, FS_OZONE_SUPPORT) - || OzoneProperties.guessIsMe(props)) ? new OzoneProperties(props) : null, + || (!isFsSupport(props, FS_OZONE_SUPPORT) + && MinioProperties.guessIsMe(props))) ? new MinioProperties(props) : null, + props -> isFsSupport(props, FS_OZONE_SUPPORT) ? new OzoneProperties(props) : null, props -> (isFsSupport(props, FS_BROKER_SUPPORT) || BrokerProperties.guessIsMe(props)) ? new BrokerProperties(props) : null, props -> (isFsSupport(props, FS_LOCAL_SUPPORT) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java index 3d6533a6806a64..64caabf2e0d16a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java @@ -41,6 +41,7 @@ public void setup() { @Test public void testValidOzoneConfiguration() { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); origProps.put("ozone.endpoint", "http://ozone-s3g:9878"); origProps.put("ozone.access_key", "hadoop"); origProps.put("ozone.secret_key", "hadoop"); @@ -63,13 +64,13 @@ public void testValidOzoneConfiguration() { } @Test - public void testFsS3aPropertiesBinding() { + public void testS3PropertiesBinding() { origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); - origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); - origProps.put("fs.s3a.access.key", "hadoop"); - origProps.put("fs.s3a.secret.key", "hadoop"); - origProps.put("fs.s3a.path.style.access", "true"); - origProps.put("fs.s3a.endpoint.region", "us-east-1"); + origProps.put("s3.endpoint", "http://ozone-s3g:9878"); + origProps.put("s3.access_key", "hadoop"); + origProps.put("s3.secret_key", "hadoop"); + origProps.put("use_path_style", "true"); + origProps.put("s3.region", "us-east-1"); OzoneProperties ozoneProperties = (OzoneProperties) StorageProperties.createPrimary(origProps); Map backendProps = ozoneProperties.getBackendConfigProperties(); @@ -85,13 +86,25 @@ public void testFsS3aPropertiesBinding() { } @Test - public void testCreateAllWithDefaultFs() throws UserException { + public void testFsS3aPropertiesAreNotSupported() { origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); - origProps.put("fs.defaultFS", "s3a://dn-data/"); origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); origProps.put("fs.s3a.access.key", "hadoop"); origProps.put("fs.s3a.secret.key", "hadoop"); - origProps.put("fs.s3a.path.style.access", "true"); + + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, + "Property ozone.endpoint is required.", + () -> StorageProperties.createPrimary(origProps)); + } + + @Test + public void testCreateAllWithDefaultFs() throws UserException { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); + origProps.put("fs.defaultFS", "s3a://dn-data/"); + origProps.put("s3.endpoint", "http://ozone-s3g:9878"); + origProps.put("s3.access_key", "hadoop"); + origProps.put("s3.secret_key", "hadoop"); + origProps.put("use_path_style", "true"); List properties = StorageProperties.createAll(origProps); Assertions.assertEquals(HdfsProperties.class, properties.get(0).getClass()); @@ -105,6 +118,7 @@ public void testCreateAllWithDefaultFs() throws UserException { @Test public void testMissingAccessKeyOrSecretKey() { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); origProps.put("ozone.endpoint", "http://ozone-s3g:9878"); origProps.put("ozone.access_key", "hadoop"); ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, @@ -120,6 +134,7 @@ public void testMissingAccessKeyOrSecretKey() { @Test public void testMissingEndpoint() { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); origProps.put("ozone.access_key", "hadoop"); origProps.put("ozone.secret_key", "hadoop"); ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, @@ -128,16 +143,19 @@ public void testMissingEndpoint() { } @Test - public void testGuessIsMe() { - origProps.put("fs.s3a.endpoint", "http://ozone-s3g:9878"); - Assertions.assertTrue(OzoneProperties.guessIsMe(origProps)); + public void testRequireExplicitFsOzoneSupport() throws UserException { + origProps.put("ozone.endpoint", "http://127.0.0.1:9878"); + origProps.put("ozone.access_key", "hadoop"); + origProps.put("ozone.secret_key", "hadoop"); - origProps.clear(); - origProps.put("fs.s3a.endpoint", "http://minio:9000"); - Assertions.assertFalse(OzoneProperties.guessIsMe(origProps)); + List propertiesWithoutFlag = StorageProperties.createAll(origProps); + Assertions.assertEquals(1, propertiesWithoutFlag.size()); + Assertions.assertEquals(HdfsProperties.class, propertiesWithoutFlag.get(0).getClass()); - origProps.clear(); - origProps.put("ozone.endpoint", "http://127.0.0.1:9878"); - Assertions.assertTrue(OzoneProperties.guessIsMe(origProps)); + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); + List propertiesWithFlag = StorageProperties.createAll(origProps); + Assertions.assertEquals(2, propertiesWithFlag.size()); + Assertions.assertEquals(HdfsProperties.class, propertiesWithFlag.get(0).getClass()); + Assertions.assertEquals(OzoneProperties.class, propertiesWithFlag.get(1).getClass()); } } From e3bcebda13c7cfb90c8a7362ead297f3aaf16751 Mon Sep 17 00:00:00 2001 From: xylaaaaa <2392805527@qq.com> Date: Fri, 27 Feb 2026 11:03:21 +0800 Subject: [PATCH 4/4] [fix][iceberg] support ozone.xxx aliases for s3a config --- .../property/storage/OzoneProperties.java | 32 +++++++++++++++++++ .../property/storage/OzonePropertiesTest.java | 22 +++++++++++++ 2 files changed, 54 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java index 8527cb73e78c46..24d1079b998df9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OzoneProperties.java @@ -101,6 +101,38 @@ protected OzoneProperties(Map origProps) { super(Type.OZONE, origProps); } + @Override + public void initNormalizeAndCheckProps() { + hydrateFromOriginalProps(); + super.initNormalizeAndCheckProps(); + hydrateFromOriginalProps(); + } + + private void hydrateFromOriginalProps() { + endpoint = StringUtils.firstNonBlank( + endpoint, + origProps.get("ozone.endpoint"), + origProps.get("s3.endpoint")); + region = StringUtils.firstNonBlank(region, origProps.get("ozone.region"), origProps.get("s3.region")); + accessKey = StringUtils.firstNonBlank( + accessKey, + origProps.get("ozone.access_key"), + origProps.get("s3.access_key"), + origProps.get("s3.access-key-id")); + secretKey = StringUtils.firstNonBlank( + secretKey, + origProps.get("ozone.secret_key"), + origProps.get("s3.secret_key"), + origProps.get("s3.secret-access-key")); + sessionToken = StringUtils.firstNonBlank(sessionToken, origProps.get("ozone.session_token"), + origProps.get("s3.session_token"), origProps.get("s3.session-token")); + usePathStyle = StringUtils.firstNonBlank(usePathStyle, origProps.get("ozone.use_path_style"), + origProps.get("use_path_style"), origProps.get("s3.path-style-access")); + forceParsingByStandardUrl = StringUtils.firstNonBlank(forceParsingByStandardUrl, + origProps.get("ozone.force_parsing_by_standard_uri"), + origProps.get("force_parsing_by_standard_uri")); + } + @Override protected Set endpointPatterns() { return ImmutableSet.of(Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java index 64caabf2e0d16a..fe6701f9163e61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OzonePropertiesTest.java @@ -116,6 +116,28 @@ public void testCreateAllWithDefaultFs() throws UserException { Assertions.assertTrue(locationPath.getStorageProperties() instanceof OzoneProperties); } + @Test + public void testCreateAllWithDefaultFsAndOzoneProperties() throws UserException { + origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true"); + origProps.put("fs.defaultFS", "s3a://dn-data/"); + origProps.put("ozone.endpoint", "http://ozone-s3g:9878"); + origProps.put("ozone.access_key", "hadoop"); + origProps.put("ozone.secret_key", "hadoop"); + origProps.put("ozone.use_path_style", "true"); + origProps.put("ozone.region", "us-east-1"); + + List properties = StorageProperties.createAll(origProps); + Assertions.assertEquals(HdfsProperties.class, properties.get(0).getClass()); + Assertions.assertEquals(OzoneProperties.class, properties.get(1).getClass()); + + OzoneProperties ozoneProperties = (OzoneProperties) properties.get(1); + Assertions.assertEquals("hadoop", ozoneProperties.getHadoopStorageConfig().get("fs.s3a.access.key")); + Assertions.assertEquals("hadoop", ozoneProperties.getHadoopStorageConfig().get("fs.s3a.secret.key")); + Assertions.assertEquals("http://ozone-s3g:9878", ozoneProperties.getHadoopStorageConfig().get("fs.s3a.endpoint")); + Assertions.assertEquals("us-east-1", ozoneProperties.getHadoopStorageConfig().get("fs.s3a.endpoint.region")); + Assertions.assertEquals("true", ozoneProperties.getHadoopStorageConfig().get("fs.s3a.path.style.access")); + } + @Test public void testMissingAccessKeyOrSecretKey() { origProps.put(StorageProperties.FS_OZONE_SUPPORT, "true");