diff --git a/ingestion/tests/unit/test_workflow_parse.py b/ingestion/tests/unit/test_workflow_parse.py index fe5b56646061..3061f66b5df3 100644 --- a/ingestion/tests/unit/test_workflow_parse.py +++ b/ingestion/tests/unit/test_workflow_parse.py @@ -404,7 +404,12 @@ def test_parsing_matillion_pipeline(self): parse_workflow_config_gracefully(config_dict) self.assertIn( - "We encountered an error parsing the configuration of your MatillionConnection.\nYou might need to review your config based on the original cause of this failure:\n\t - Missing parameter in ('connection', 'hostPort')\n\t - Missing parameter in ('connection', 'username')\n\t - Missing parameter in ('connection', 'password')", + "We encountered an error parsing the configuration of your MatillionConnection.\n" + "You might need to review your config based on the original cause of this failure:\n" + "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'hostPort')\n" + "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'username')\n" + "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'password')\n" + "\t - Invalid parameter value for ('connection', 'function-after[parse_name(), MatillionDpcAuthConfig]', 'type')", str(err.exception), ) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java index d6e0d797e15f..771516e1cc17 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java @@ -15,6 +15,7 @@ import java.util.List; import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth; import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth; import org.openmetadata.schema.utils.JsonUtils; @@ -30,7 +31,9 @@ public Object convert(Object object) { MatillionConnection matillionConnection = (MatillionConnection) JsonUtils.convertValue(object, this.clazz); - tryToConvertOrFail(matillionConnection.getConnection(), List.of(MatillionETLAuth.class)) + tryToConvertOrFail( + matillionConnection.getConnection(), + List.of(MatillionETLAuth.class, MatillionDPCAuth.class)) .ifPresent(matillionConnection::setConnection); return matillionConnection; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java index 49395183f827..3886cfe60c2d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java @@ -24,6 +24,7 @@ import org.openmetadata.schema.services.connections.database.TrinoConnection; import org.openmetadata.schema.services.connections.database.datalake.GCSConfig; import org.openmetadata.schema.services.connections.pipeline.AirflowConnection; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; import org.openmetadata.schema.services.connections.search.ElasticSearchConnection; import org.openmetadata.schema.services.connections.storage.GCSConnection; @@ -52,6 +53,7 @@ public class ClassConverterFactoryTest { Workflow.class, SalesforceConnection.class, IcebergConnection.class, + MatillionConnection.class, }) void testClassConverterIsSet(Class clazz) { assertFalse( diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverterTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverterTest.java new file mode 100644 index 000000000000..585ebaa7bd51 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverterTest.java @@ -0,0 +1,55 @@ +package org.openmetadata.service.secrets.converter; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.services.connections.pipeline.MatillionConnection; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth; +import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth; +import org.openmetadata.schema.utils.JsonUtils; + +class MatillionConnectionClassConverterTest { + + private final ClassConverter converter = + ClassConverterFactory.getConverter(MatillionConnection.class); + + @Test + void testConvertsETLAuth() { + MatillionETLAuth etlAuth = new MatillionETLAuth().withHostPort("https://matillion.example.com"); + + MatillionConnection input = new MatillionConnection().withConnection(etlAuth); + Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class); + + MatillionConnection result = (MatillionConnection) converter.convert(rawInput); + + assertNotNull(result); + assertInstanceOf(MatillionETLAuth.class, result.getConnection()); + } + + @Test + void testConvertsDPCAuth() { + MatillionDPCAuth dpcAuth = + new MatillionDPCAuth().withClientId("client-id").withClientSecret("secret"); + + MatillionConnection input = new MatillionConnection().withConnection(dpcAuth); + Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class); + + MatillionConnection result = (MatillionConnection) converter.convert(rawInput); + + assertNotNull(result); + assertInstanceOf(MatillionDPCAuth.class, result.getConnection()); + } + + @Test + void testNullConnectionDoesNotThrow() { + MatillionConnection input = new MatillionConnection(); + Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class); + + MatillionConnection result = (MatillionConnection) converter.convert(rawInput); + + assertNotNull(result); + assertNull(result.getConnection()); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionDPC.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionDPC.json new file mode 100644 index 000000000000..1a2f64086bee --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionDPC.json @@ -0,0 +1,46 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/matillion/matillionDPC.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Matillion DPC Auth Config", + "description": "Matillion Data Productivity Cloud Auth Config.", + "javaType": "org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth", + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "MatillionDPC" + ], + "default": "MatillionDPC" + }, + "clientId": { + "title": "Client ID", + "description": "OAuth2 Client ID for Matillion DPC authentication.", + "type": "string" + }, + "clientSecret": { + "title": "Client Secret", + "description": "OAuth2 Client Secret for Matillion DPC authentication.", + "type": "string", + "format": "password" + }, + "region": { + "title": "Region", + "description": "Matillion DPC region. Determines the API base URL.", + "type": "string", + "enum": [ + "us1", + "eu1" + ], + "default": "us1" + }, + "personalAccessToken": { + "title": "Personal Access Token", + "description": "Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials.", + "type": "string", + "format": "password" + } + }, + "required": [], + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json index b2eaa8bdfdfe..c283df8b9470 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json @@ -28,6 +28,9 @@ "oneOf": [ { "$ref": "matillion/matillionETL.json" + }, + { + "$ref": "matillion/matillionDPC.json" } ] }, @@ -36,10 +39,18 @@ "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern", "title": "Default Pipeline Filter Pattern" }, + "lineageLookbackDays": { + "title": "Lineage Lookback Days", + "description": "Number of days to look back when fetching lineage events from Matillion DPC OpenLineage API.", + "type": "integer", + "default": 30, + "minimum": 1, + "maximum": 365 + }, "supportsMetadataExtraction": { "title": "Supports Metadata Extraction", "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" } }, "additionalProperties": false -} \ No newline at end of file +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts index 22ec533896bf..92388a9a585d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts @@ -1958,6 +1958,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -3647,6 +3652,8 @@ export interface GCPImpersonateServiceAccountValues { * * Matillion ETL Auth Config. * + * Matillion Data Productivity Cloud Auth Config. + * * Choose between mysql and postgres connection for alation database */ export interface ConfigConnection { @@ -3828,6 +3835,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -3944,6 +3967,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -4108,6 +4139,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts index ffb3498ca06d..4949d470ff75 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts @@ -122,7 +122,7 @@ export interface ConfigObject { * * Matillion Auth Configuration */ - connection?: AirflowConnection; + connection?: ConnectionClass; /** * Pipeline Service Management/UI URI. * @@ -312,6 +312,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * Available sources to fetch metadata. */ @@ -706,8 +711,10 @@ export interface AzureCredentials { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ -export interface AirflowConnection { +export interface ConnectionClass { /** * Airflow REST API version. */ @@ -840,6 +847,22 @@ export interface AirflowConnection { */ password?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -1072,6 +1095,14 @@ export interface FilterPattern { includes?: string[]; } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -1189,6 +1220,7 @@ export enum SSLMode { */ export enum Type { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts index 76594db8b148..330a91571b4d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts @@ -4719,6 +4719,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -5937,6 +5942,8 @@ export interface ConfigSourceConnection { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ export interface ConfigConnection { /** @@ -6117,6 +6124,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -6233,6 +6256,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -6397,6 +6428,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts index 61b850467fc6..16abd12a51a4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/testServiceConnection.ts @@ -1840,6 +1840,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -3529,6 +3534,8 @@ export interface GCPImpersonateServiceAccountValues { * * Matillion ETL Auth Config. * + * Matillion Data Productivity Cloud Auth Config. + * * Choose between mysql and postgres connection for alation database */ export interface ConfigConnection { @@ -3710,6 +3717,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -3826,6 +3849,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -3990,6 +4021,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts index 05790271f0c8..ad6ee477b8a5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/automations/workflow.ts @@ -2493,6 +2493,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -4041,6 +4046,8 @@ export interface GCPImpersonateServiceAccountValues { * * Matillion ETL Auth Config. * + * Matillion Data Productivity Cloud Auth Config. + * * Choose between mysql and postgres connection for alation database */ export interface ConfigConnection { @@ -4222,6 +4229,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -4338,6 +4361,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -4502,6 +4533,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillion/matillionDPC.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillion/matillionDPC.ts new file mode 100644 index 000000000000..56ec8ecda8d6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillion/matillionDPC.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Matillion Data Productivity Cloud Auth Config. + */ +export interface MatillionDPC { + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; + type?: Type; +} + +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + +export enum Type { + MatillionDPC = "MatillionDPC", +} diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillionConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillionConnection.ts index 22f4e857fab1..1ae5cfea736a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillionConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/pipeline/matillionConnection.ts @@ -17,7 +17,12 @@ export interface MatillionConnection { /** * Matillion Auth Configuration */ - connection?: Matillion; + connection?: MatillionConnectionClass; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * Regex exclude pipelines. */ @@ -33,23 +38,49 @@ export interface MatillionConnection { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ -export interface Matillion { +export interface MatillionConnectionClass { /** * Matillion Host */ - hostPort: string; + hostPort?: string; /** * Password to connect to the Matillion. */ - password: string; + password?: string; sslConfig?: Config; type?: Type; /** * Username to connect to the Matillion. This user should have privileges to read all the * metadata in Matillion. */ - username: string; + username?: string; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; +} + +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", } /** @@ -73,6 +104,7 @@ export interface Config { } export enum Type { + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", } diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts index fb6304a98191..8bee6a7f07fb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/connections/serviceConnection.ts @@ -2003,6 +2003,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -3513,6 +3518,8 @@ export interface GCPImpersonateServiceAccountValues { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ export interface ConfigConnection { /** @@ -3693,6 +3700,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -3809,6 +3832,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -3973,6 +4004,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts index d518a821830e..e839c61f4b14 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/ingestionPipelines/ingestionPipeline.ts @@ -5302,6 +5302,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -6462,6 +6467,8 @@ export interface ConfigSourceConnection { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ export interface ConfigConnection { /** @@ -6642,6 +6649,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -6758,6 +6781,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -6922,6 +6953,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts index 61a44afd70b5..0c29dee2534e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/services/pipelineService.ts @@ -240,7 +240,7 @@ export interface ConfigObject { * * Matillion Auth Configuration */ - connection?: AirflowConnection; + connection?: ConnectionClass; /** * Pipeline Service Management/UI URI. * @@ -430,6 +430,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * Available sources to fetch metadata. */ @@ -824,8 +829,10 @@ export interface AzureCredentials { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ -export interface AirflowConnection { +export interface ConnectionClass { /** * Airflow REST API version. */ @@ -958,6 +965,22 @@ export interface AirflowConnection { */ password?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -1190,6 +1213,14 @@ export interface FilterPattern { includes?: string[]; } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -1307,6 +1338,7 @@ export enum SSLMode { */ export enum Type { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts index 8bba7dc87b99..b9a127b11f4f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts @@ -2047,6 +2047,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -3557,6 +3562,8 @@ export interface GCPImpersonateServiceAccountValues { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ export interface ConfigConnection { /** @@ -3737,6 +3744,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -3853,6 +3876,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -4017,6 +4048,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts index c3cb2cb4ea71..21e8441b3b3a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/workflow.ts @@ -2092,6 +2092,11 @@ export interface ConfigObject { * List of IDs of your DBT cloud projects seperated by comma `,` */ projectIds?: string[]; + /** + * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage + * API. + */ + lineageLookbackDays?: number; /** * The name of your azure data factory. */ @@ -3615,6 +3620,8 @@ export interface GCPImpersonateServiceAccountValues { * Matillion Auth Configuration * * Matillion ETL Auth Config. + * + * Matillion Data Productivity Cloud Auth Config. */ export interface ConfigConnection { /** @@ -3795,6 +3802,22 @@ export interface ConfigConnection { */ databaseMode?: string; supportsViewLineageExtraction?: boolean; + /** + * OAuth2 Client ID for Matillion DPC authentication. + */ + clientId?: string; + /** + * OAuth2 Client Secret for Matillion DPC authentication. + */ + clientSecret?: string; + /** + * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials. + */ + personalAccessToken?: string; + /** + * Matillion DPC region. Determines the API base URL. + */ + region?: Region; } /** @@ -3911,6 +3934,14 @@ export enum Provider { LDAP = "ldap", } +/** + * Matillion DPC region. Determines the API base URL. + */ +export enum Region { + Eu1 = "eu1", + Us1 = "us1", +} + /** * Storage config to store sample data */ @@ -4075,6 +4106,7 @@ export enum SSLMode { */ export enum ConnectionType { Backend = "Backend", + MatillionDPC = "MatillionDPC", MatillionETL = "MatillionETL", Mysql = "Mysql", Postgres = "Postgres",