ci: add SynapseML-Internal compatibility check to OSS pipeline

pipeline.yaml

@@ -1,5 +1,11 @@
 resources:
-- repo: self
+  repositories:
+  - repository: self
+    type: self
+  - repository: SynapseML-Internal
+    type: git
+    name: A365/SynapseML-Internal
+    ref: master
 
 trigger:
   branches:
@@ -803,3 +809,132 @@ jobs:
     - template: templates/kv.yml
     - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.SourceBranch'], 'refs/heads/master'), startsWith(variables['Build.SourceBranch'], 'refs/tags/')) }}:
       - template: templates/codecov.yml
+
+- job: InternalCompat
+  displayName: 'SynapseML-Internal Compatibility Check'
+  cancelTimeoutInMinutes: 0
+  timeoutInMinutes: 90
+  continueOnError: true
+  pool:
+    vmImage: $(UBUNTU_VERSION)
+  steps:
+    - checkout: self
+    - checkout: SynapseML-Internal
+    - task: AzureCLI@2
+      displayName: 'Publish OSS to local Maven'
+      timeoutInMinutes: 20
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          OSS_VERSION=$(sbt -no-colors "core/version" 2>&1 | grep '^\[info\] [0-9]' | tail -1 | sed 's/\[info\] //')
+          echo "OSS version: $OSS_VERSION"
+          if [ -z "$OSS_VERSION" ]; then
+            echo "##vso[task.logissue type=error]Failed to determine OSS version"
+            exit 1
+          fi
+          echo "##vso[task.setvariable variable=OSS_VERSION]$OSS_VERSION"
+          sbt publishM2
+    - bash: |
+        set -e
+        cd $(Build.SourcesDirectory)/SynapseML-Internal
+
+        echo "=== Retargeting Internal to OSS version $(OSS_VERSION) ==="
+        sed -i 's|val synapseMLVersion = ".*"|val synapseMLVersion = "$(OSS_VERSION)"|' build.sbt
+        sed -i '/^resolvers ++= Seq(/a\  Resolver.mavenLocal,' build.sbt
+
+        echo "=== Modified build.sbt ==="
+        grep -n 'synapseMLVersion' build.sbt
+        grep -n -A4 'resolvers ++=' build.sbt
+      displayName: 'Retarget Internal to this build'
+    - task: AzureCLI@2
+      displayName: 'Compile Internal against OSS'
+      timeoutInMinutes: 15
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML-Internal
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          echo "Compiling SynapseML-Internal against OSS $(OSS_VERSION)..."
+          sbt compile Test/compile
+    - bash: |
+        echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: 'Add conda to PATH'
+    - bash: |
+        sudo chown -R $(whoami):$(id -ng) $(CONDA_CACHE_DIR)
+      displayName: 'Fix conda directory permissions'
+    - task: PipAuthenticate@1
+      displayName: 'Private Conda Feed Authentication'
+      inputs:
+        artifactFeeds: 'A365/Synapse-Conda'
+    - bash: |
+        conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
+        conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
+      displayName: 'Accept Anaconda TOS'
+    - bash: |
+        echo "=== Disk space BEFORE cleanup ==="
+        df -h / | grep -E 'Filesystem|/$'
+        echo "Removing unused pre-installed SDKs and runtimes..."
+        sudo rm -rf /usr/local/lib/android || true
+        sudo rm -rf /usr/share/dotnet || true
+        sudo rm -rf /opt/ghc || true
+        sudo rm -rf /usr/local/share/boost || true
+        docker system prune -af --volumes 2>/dev/null || true
+        echo "=== Disk space AFTER cleanup ==="
+        df -h / | grep -E 'Filesystem|/$'
+      displayName: 'Free disk space (remove unused SDKs)'
+    - bash: |
+        set -e
+        conda env create --yes -f $(Build.SourcesDirectory)/SynapseML-Internal/environment.yaml -v || \
+          conda env create --yes -f $(Build.SourcesDirectory)/SynapseML-Internal/environment.yaml -v
+        conda clean --all -y
+        pip cache purge
+      displayName: 'Create Internal conda env'
+    - task: AzureKeyVault@2
+      displayName: 'Fetch AI service secrets'
+      retryCountOnTaskFailure: 3
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        keyVaultName: mmlspark-keys
+    - task: AzureCLI@2
+      displayName: 'Run Internal Scala tests (non-Fabric)'
+      timeoutInMinutes: 60
+      inputs:
+        azureSubscription: 'SynapseML Build'
+        scriptLocation: inlineScript
+        scriptType: bash
+        inlineScript: |
+          set -e
+          cd $(Build.SourcesDirectory)/SynapseML-Internal
+          source activate synapseml-internal
+          export CREATE_SEMPY_WRITER=false
+          export SBT_OPTS="-Xmx4G -Xss2M -Duser.timezone=GMT"
+          echo "Running non-Fabric Internal tests against OSS $(OSS_VERSION)..."
+          # Only spark.aifunc is runnable without Fabric creds.
+          # powerbi/ebm/predict extend HasSparkSession which eagerly inits
+          # FabricTestConstants (requires INTEGRATION_ACCOUNT from fabrictest-cert-admin-kv).
+          FAILURES=0
+          for pkg in spark.aifunc; do
+            echo "=== Testing $pkg ==="
+            if ! sbt "testOnly com.microsoft.azure.synapse.ml.$pkg.**"; then
+              echo "##vso[task.logissue type=warning]$pkg tests failed"
+              FAILURES=$((FAILURES + 1))
+            fi
+          done
+          if [ $FAILURES -gt 0 ]; then
+            echo "##vso[task.logissue type=warning]$FAILURES test package(s) failed"
+            exit 1
+          fi
+    - task: PublishTestResults@2
+      displayName: 'Publish Internal Test Results'
+      inputs:
+        testResultsFiles: '**/test-reports/TEST-*.xml'
+        failTaskOnFailedTests: false
+      condition: succeededOrFailed()