feat: Add matillion data cloud support

ingestion/tests/unit/test_workflow_parse.py

@@ -404,7 +404,12 @@ def test_parsing_matillion_pipeline(self):
             parse_workflow_config_gracefully(config_dict)
 
         self.assertIn(
-            "We encountered an error parsing the configuration of your MatillionConnection.\nYou might need to review your config based on the original cause of this failure:\n\t - Missing parameter in ('connection', 'hostPort')\n\t - Missing parameter in ('connection', 'username')\n\t - Missing parameter in ('connection', 'password')",
+            "We encountered an error parsing the configuration of your MatillionConnection.\n"
+            "You might need to review your config based on the original cause of this failure:\n"
+            "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'hostPort')\n"
+            "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'username')\n"
+            "\t - Missing parameter in ('connection', 'function-after[parse_name(), MatillionEtlAuthConfig]', 'password')\n"
+            "\t - Invalid parameter value for ('connection', 'function-after[parse_name(), MatillionDpcAuthConfig]', 'type')",
             str(err.exception),
         )
 

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverter.java

@@ -15,6 +15,7 @@
 
 import java.util.List;
 import org.openmetadata.schema.services.connections.pipeline.MatillionConnection;
+import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth;
 import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth;
 import org.openmetadata.schema.utils.JsonUtils;
 
@@ -30,7 +31,9 @@ public Object convert(Object object) {
     MatillionConnection matillionConnection =
         (MatillionConnection) JsonUtils.convertValue(object, this.clazz);
 
-    tryToConvertOrFail(matillionConnection.getConnection(), List.of(MatillionETLAuth.class))
+    tryToConvertOrFail(
+            matillionConnection.getConnection(),
+            List.of(MatillionETLAuth.class, MatillionDPCAuth.class))
         .ifPresent(matillionConnection::setConnection);
 
     return matillionConnection;

openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/ClassConverterFactoryTest.java

@@ -24,6 +24,7 @@
 import org.openmetadata.schema.services.connections.database.TrinoConnection;
 import org.openmetadata.schema.services.connections.database.datalake.GCSConfig;
 import org.openmetadata.schema.services.connections.pipeline.AirflowConnection;
+import org.openmetadata.schema.services.connections.pipeline.MatillionConnection;
 import org.openmetadata.schema.services.connections.search.ElasticSearchConnection;
 import org.openmetadata.schema.services.connections.storage.GCSConnection;
 
@@ -52,6 +53,7 @@ public class ClassConverterFactoryTest {
         Workflow.class,
         SalesforceConnection.class,
         IcebergConnection.class,
+        MatillionConnection.class,
       })
   void testClassConverterIsSet(Class<?> clazz) {
     assertFalse(

openmetadata-service/src/test/java/org/openmetadata/service/secrets/converter/MatillionConnectionClassConverterTest.java

@@ -0,0 +1,55 @@
+package org.openmetadata.service.secrets.converter;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import org.junit.jupiter.api.Test;
+import org.openmetadata.schema.services.connections.pipeline.MatillionConnection;
+import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth;
+import org.openmetadata.schema.services.connections.pipeline.matillion.MatillionETLAuth;
+import org.openmetadata.schema.utils.JsonUtils;
+
+class MatillionConnectionClassConverterTest {
+
+  private final ClassConverter converter =
+      ClassConverterFactory.getConverter(MatillionConnection.class);
+
+  @Test
+  void testConvertsETLAuth() {
+    MatillionETLAuth etlAuth = new MatillionETLAuth().withHostPort("https://matillion.example.com");
+
+    MatillionConnection input = new MatillionConnection().withConnection(etlAuth);
+    Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class);
+
+    MatillionConnection result = (MatillionConnection) converter.convert(rawInput);
+
+    assertNotNull(result);
+    assertInstanceOf(MatillionETLAuth.class, result.getConnection());
+  }
+
+  @Test
+  void testConvertsDPCAuth() {
+    MatillionDPCAuth dpcAuth =
+        new MatillionDPCAuth().withClientId("client-id").withClientSecret("secret");
+
+    MatillionConnection input = new MatillionConnection().withConnection(dpcAuth);
+    Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class);
+
+    MatillionConnection result = (MatillionConnection) converter.convert(rawInput);
+
+    assertNotNull(result);
+    assertInstanceOf(MatillionDPCAuth.class, result.getConnection());
+  }
+
+  @Test
+  void testNullConnectionDoesNotThrow() {
+    MatillionConnection input = new MatillionConnection();
+    Object rawInput = JsonUtils.readValue(JsonUtils.pojoToJson(input), Object.class);
+
+    MatillionConnection result = (MatillionConnection) converter.convert(rawInput);
+
+    assertNotNull(result);
+    assertNull(result.getConnection());
+  }
+}

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillion/matillionDPC.json

@@ -0,0 +1,46 @@
+{
+    "$id": "https://open-metadata.org/schema/entity/services/connections/pipeline/matillion/matillionDPC.json",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Matillion DPC Auth Config",
+    "description": "Matillion Data Productivity Cloud Auth Config.",
+    "javaType": "org.openmetadata.schema.services.connections.pipeline.matillion.MatillionDPCAuth",
+    "type": "object",
+    "properties": {
+        "type": {
+            "type": "string",
+            "enum": [
+                "MatillionDPC"
+            ],
+            "default": "MatillionDPC"
+        },
+        "clientId": {
+            "title": "Client ID",
+            "description": "OAuth2 Client ID for Matillion DPC authentication.",
+            "type": "string"
+        },
+        "clientSecret": {
+            "title": "Client Secret",
+            "description": "OAuth2 Client Secret for Matillion DPC authentication.",
+            "type": "string",
+            "format": "password"
+        },
+        "region": {
+            "title": "Region",
+            "description": "Matillion DPC region. Determines the API base URL.",
+            "type": "string",
+            "enum": [
+                "us1",
+                "eu1"
+            ],
+            "default": "us1"
+        },
+        "personalAccessToken": {
+            "title": "Personal Access Token",
+            "description": "Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials.",
+            "type": "string",
+            "format": "password"
+        }
+    },
+    "required": [],
+    "additionalProperties": false
+}

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json

@@ -28,6 +28,9 @@
       "oneOf": [
         {
           "$ref": "matillion/matillionETL.json"
+        },
+        {
+          "$ref": "matillion/matillionDPC.json"
         }
       ]
     },
@@ -36,10 +39,18 @@
       "$ref": "../../../../type/filterPattern.json#/definitions/filterPattern",
       "title": "Default Pipeline Filter Pattern"
     },
+    "lineageLookbackDays": {
+      "title": "Lineage Lookback Days",
+      "description": "Number of days to look back when fetching lineage events from Matillion DPC OpenLineage API.",
+      "type": "integer",
+      "default": 30,
+      "minimum": 1,
+      "maximum": 365
+    },
     "supportsMetadataExtraction": {
       "title": "Supports Metadata Extraction",
       "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction"
     }
   },
   "additionalProperties": false
-}
+}

openmetadata-ui/src/main/resources/ui/src/generated/api/automations/createWorkflow.ts

@@ -1959,6 +1959,11 @@ export interface ConfigObject {
      * List of IDs of your DBT cloud projects seperated by comma `,`
      */
     projectIds?: string[];
+    /**
+     * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage
+     * API.
+     */
+    lineageLookbackDays?: number;
     /**
      * The name of your azure data factory.
      */
@@ -3637,6 +3642,8 @@ export interface GCPImpersonateServiceAccountValues {
  *
  * Matillion ETL Auth Config.
  *
+ * Matillion Data Productivity Cloud Auth Config.
+ *
  * Choose between mysql and postgres connection for alation database
  */
 export interface ConfigConnection {
@@ -3806,6 +3813,22 @@ export interface ConfigConnection {
      */
     databaseMode?:                  string;
     supportsViewLineageExtraction?: boolean;
+    /**
+     * OAuth2 Client ID for Matillion DPC authentication.
+     */
+    clientId?: string;
+    /**
+     * OAuth2 Client Secret for Matillion DPC authentication.
+     */
+    clientSecret?: string;
+    /**
+     * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials.
+     */
+    personalAccessToken?: string;
+    /**
+     * Matillion DPC region. Determines the API base URL.
+     */
+    region?: Region;
 }
 
 /**
@@ -3836,6 +3859,14 @@ export enum Provider {
     LDAP = "ldap",
 }
 
+/**
+ * Matillion DPC region. Determines the API base URL.
+ */
+export enum Region {
+    Eu1 = "eu1",
+    Us1 = "us1",
+}
+
 /**
  * Storage config to store sample data
  */
@@ -3998,6 +4029,7 @@ export enum SSLMode {
  */
 export enum ConnectionType {
     Backend = "Backend",
+    MatillionDPC = "MatillionDPC",
     MatillionETL = "MatillionETL",
     Mysql = "Mysql",
     Postgres = "Postgres",

openmetadata-ui/src/main/resources/ui/src/generated/api/services/createPipelineService.ts

@@ -123,7 +123,7 @@ export interface ConfigObject {
      *
      * Matillion Auth Configuration
      */
-    connection?: MetadataDatabaseConnection;
+    connection?: ConnectionClass;
     /**
      * Pipeline Service Management/UI URI.
      *
@@ -313,6 +313,11 @@ export interface ConfigObject {
      * List of IDs of your DBT cloud projects seperated by comma `,`
      */
     projectIds?: string[];
+    /**
+     * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage
+     * API.
+     */
+    lineageLookbackDays?: number;
     /**
      * Available sources to fetch metadata.
      */
@@ -704,8 +709,10 @@ export interface AzureCredentials {
  * Matillion Auth Configuration
  *
  * Matillion ETL Auth Config.
+ *
+ * Matillion Data Productivity Cloud Auth Config.
  */
-export interface MetadataDatabaseConnection {
+export interface ConnectionClass {
     /**
      * Regex exclude pipelines.
      */
@@ -825,6 +832,22 @@ export interface MetadataDatabaseConnection {
      */
     password?:                      string;
     supportsViewLineageExtraction?: boolean;
+    /**
+     * OAuth2 Client ID for Matillion DPC authentication.
+     */
+    clientId?: string;
+    /**
+     * OAuth2 Client Secret for Matillion DPC authentication.
+     */
+    clientSecret?: string;
+    /**
+     * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials.
+     */
+    personalAccessToken?: string;
+    /**
+     * Matillion DPC region. Determines the API base URL.
+     */
+    region?: Region;
 }
 
 /**
@@ -875,6 +898,14 @@ export interface FilterPattern {
     includes?: string[];
 }
 
+/**
+ * Matillion DPC region. Determines the API base URL.
+ */
+export enum Region {
+    Eu1 = "eu1",
+    Us1 = "us1",
+}
+
 /**
  * Storage config to store sample data
  */
@@ -990,6 +1021,7 @@ export enum SSLMode {
  */
 export enum Type {
     Backend = "Backend",
+    MatillionDPC = "MatillionDPC",
     MatillionETL = "MatillionETL",
     Mysql = "Mysql",
     Postgres = "Postgres",

openmetadata-ui/src/main/resources/ui/src/generated/api/services/ingestionPipelines/createIngestionPipeline.ts

@@ -4716,6 +4716,11 @@ export interface ConfigObject {
      * List of IDs of your DBT cloud projects seperated by comma `,`
      */
     projectIds?: string[];
+    /**
+     * Number of days to look back when fetching lineage events from Matillion DPC OpenLineage
+     * API.
+     */
+    lineageLookbackDays?: number;
     /**
      * The name of your azure data factory.
      */
@@ -5927,6 +5932,8 @@ export interface ConfigSourceConnection {
  * Matillion Auth Configuration
  *
  * Matillion ETL Auth Config.
+ *
+ * Matillion Data Productivity Cloud Auth Config.
  */
 export interface ConfigConnection {
     /**
@@ -6095,6 +6102,22 @@ export interface ConfigConnection {
      */
     databaseMode?:                  string;
     supportsViewLineageExtraction?: boolean;
+    /**
+     * OAuth2 Client ID for Matillion DPC authentication.
+     */
+    clientId?: string;
+    /**
+     * OAuth2 Client Secret for Matillion DPC authentication.
+     */
+    clientSecret?: string;
+    /**
+     * Personal Access Token for Matillion DPC. Alternative to OAuth2 Client Credentials.
+     */
+    personalAccessToken?: string;
+    /**
+     * Matillion DPC region. Determines the API base URL.
+     */
+    region?: Region;
 }
 
 /**
@@ -6125,6 +6148,14 @@ export enum Provider {
     LDAP = "ldap",
 }
 
+/**
+ * Matillion DPC region. Determines the API base URL.
+ */
+export enum Region {
+    Eu1 = "eu1",
+    Us1 = "us1",
+}
+
 /**
  * Storage config to store sample data
  */
@@ -6287,6 +6318,7 @@ export enum SSLMode {
  */
 export enum ConnectionType {
     Backend = "Backend",
+    MatillionDPC = "MatillionDPC",
     MatillionETL = "MatillionETL",
     Mysql = "Mysql",
     Postgres = "Postgres",