Skip to content

OADP-6675: Update Azure registry configuration for workload identity support #350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kaovilai wants to merge 1 commit into
base: oadp-dev
Choose a base branch
from
Draft

OADP-6675: Update Azure registry configuration for workload identity support #350

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 41 additions & 32 deletions velero-plugins/imagestream/registry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@ const (
RegistryStorageS3RootdirectoryEnvVarKey = "REGISTRY_STORAGE_S3_ROOTDIRECTORY"
RegistryStorageS3SkipverifyEnvVarKey = "REGISTRY_STORAGE_S3_SKIPVERIFY"
// Azure registry env vars
RegistryStorageAzureContainerEnvVarKey = "REGISTRY_STORAGE_AZURE_CONTAINER"
RegistryStorageAzureAccountnameEnvVarKey = "REGISTRY_STORAGE_AZURE_ACCOUNTNAME"
RegistryStorageAzureAccountkeyEnvVarKey = "REGISTRY_STORAGE_AZURE_ACCOUNTKEY"
RegistryStorageAzureSPNClientIDEnvVarKey = "REGISTRY_STORAGE_AZURE_SPN_CLIENT_ID"
RegistryStorageAzureSPNClientSecretEnvVarKey = "REGISTRY_STORAGE_AZURE_SPN_CLIENT_SECRET"
RegistryStorageAzureSPNTenantIDEnvVarKey = "REGISTRY_STORAGE_AZURE_SPN_TENANT_ID"
RegistryStorageAzureAADEndpointEnvVarKey = "REGISTRY_STORAGE_AZURE_AAD_ENDPOINT"
RegistryStorageAzureContainerEnvVarKey = "REGISTRY_STORAGE_AZURE_CONTAINER"
RegistryStorageAzureAccountnameEnvVarKey = "REGISTRY_STORAGE_AZURE_ACCOUNTNAME"
RegistryStorageAzureAccountkeyEnvVarKey = "REGISTRY_STORAGE_AZURE_ACCOUNTKEY"
RegistryStorageAzureCredentialsTypeEnvVarKey = "REGISTRY_STORAGE_AZURE_CREDENTIALS_TYPE"
RegistryStorageAzureCredentialsClientIDEnvVarKey = "REGISTRY_STORAGE_AZURE_CREDENTIALS_CLIENTID"
RegistryStorageAzureCredentialsSecretEnvVarKey = "REGISTRY_STORAGE_AZURE_CREDENTIALS_SECRET"
RegistryStorageAzureCredentialsTenantIDEnvVarKey = "REGISTRY_STORAGE_AZURE_CREDENTIALS_TENANTID"
// GCP registry env vars
RegistryStorageGCSBucket = "REGISTRY_STORAGE_GCS_BUCKET"
RegistryStorageGCSKeyfile = "REGISTRY_STORAGE_GCS_KEYFILE"
Expand Down Expand Up @@ -81,19 +81,19 @@ var cloudProviderEnvVarMap = map[string][]corev1.EnvVar{
Value: "",
},
{
Name: RegistryStorageAzureAADEndpointEnvVarKey,
Name: RegistryStorageAzureCredentialsTypeEnvVarKey,
Value: "",
},
{
Name: RegistryStorageAzureSPNClientIDEnvVarKey,
Name: RegistryStorageAzureCredentialsClientIDEnvVarKey,
Value: "",
},
{
Name: RegistryStorageAzureSPNClientSecretEnvVarKey,
Name: RegistryStorageAzureCredentialsSecretEnvVarKey,
Value: "",
},
{
Name: RegistryStorageAzureSPNTenantIDEnvVarKey,
Name: RegistryStorageAzureCredentialsTenantIDEnvVarKey,
Value: "",
},
},
Expand Down Expand Up @@ -126,9 +126,9 @@ func getAWSRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar
if bsl.Spec.Config == nil {
bsl.Spec.Config = make(map[string]string)
}
if bsl.Spec.Config[S3URL] == "" && bsl.Spec.Config[Region] == "" {
if bsl.Spec.Config[S3URL] == "" && bsl.Spec.Config[Region] == "" {
var err error
bsl.Spec.Config[Region], err = GetBucketRegion(bsl.Spec.StorageType.ObjectStorage.Bucket)
bsl.Spec.Config[Region], err = GetBucketRegion(bsl.Spec.ObjectStorage.Bucket)
if err != nil {
return nil, errors.Wrap(err, "failed to get bucket region")
}
Expand All @@ -146,7 +146,7 @@ func getAWSRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar
},
{
Name: RegistryStorageS3BucketEnvVarKey,
Value: bsl.Spec.StorageType.ObjectStorage.Bucket,
Value: bsl.Spec.ObjectStorage.Bucket,
},
{
Name: RegistryStorageS3RegionEnvVarKey,
Expand Down Expand Up @@ -199,7 +199,7 @@ func getAWSRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar
func getBslSecretPath(bsl *velerov1.BackupStorageLocation) string {
var secretName, secretKey string
if bsl.Spec.Credential != nil {
secretName = bsl.Spec.Credential.LocalObjectReference.Name
secretName = bsl.Spec.Credential.Name
secretKey = bsl.Spec.Credential.Key
}
// if secretName or secretKey is not set, inherit from OADP defaults for each provider
Expand All @@ -216,44 +216,53 @@ func getAzureRegistryEnvVars(bsl *velerov1.BackupStorageLocation, azureEnvVars [
if bsl.Spec.Config == nil {
bsl.Spec.Config = make(map[string]string)
}
secretName := "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"

for i := range azureEnvVars {
if azureEnvVars[i].Name == RegistryStorageAzureContainerEnvVarKey {
azureEnvVars[i].Value = bsl.Spec.StorageType.ObjectStorage.Bucket
}
switch azureEnvVars[i].Name {
case RegistryStorageAzureContainerEnvVarKey:
azureEnvVars[i].Value = bsl.Spec.ObjectStorage.Bucket

if azureEnvVars[i].Name == RegistryStorageAzureAccountnameEnvVarKey {
case RegistryStorageAzureAccountnameEnvVarKey:
azureEnvVars[i].Value = bsl.Spec.Config[StorageAccount]
}

if azureEnvVars[i].Name == RegistryStorageAzureAccountkeyEnvVarKey {
case RegistryStorageAzureAccountkeyEnvVarKey:
azureEnvVars[i].ValueFrom = &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
Key: "storage_account_key",
},
}
}
if azureEnvVars[i].Name == RegistryStorageAzureSPNClientIDEnvVarKey {

case RegistryStorageAzureCredentialsTypeEnvVarKey:
// Get credentials type from secret
azureEnvVars[i].ValueFrom = &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
Key: "credentials_type",
},
}

case RegistryStorageAzureCredentialsClientIDEnvVarKey:
azureEnvVars[i].ValueFrom = &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
Key: "client_id_key",
},
}
}

if azureEnvVars[i].Name == RegistryStorageAzureSPNClientSecretEnvVarKey {
case RegistryStorageAzureCredentialsSecretEnvVarKey:
azureEnvVars[i].ValueFrom = &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
Key: "client_secret_key",
},
}
}
if azureEnvVars[i].Name == RegistryStorageAzureSPNTenantIDEnvVarKey {

case RegistryStorageAzureCredentialsTenantIDEnvVarKey:
azureEnvVars[i].ValueFrom = &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + bsl.Name + "-" + bsl.Spec.Provider + "-registry-secret"},
LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
Key: "tenant_id_key",
},
}
Expand All @@ -270,7 +279,7 @@ func getGCPRegistryEnvVars(bsl *velerov1.BackupStorageLocation) ([]corev1.EnvVar
},
{
Name: RegistryStorageGCSBucket,
Value: bsl.Spec.StorageType.ObjectStorage.Bucket,
Value: bsl.Spec.ObjectStorage.Bucket,
},
{
Name: RegistryStorageGCSKeyfile,
Expand Down
143 changes: 133 additions & 10 deletions velero-plugins/imagestream/registry_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,22 +118,36 @@ var (
web_identity_token_file=/var/run/secrets/some/path
`),
}
// Azure registry secret data with shared_key authentication
azureRegistrySecretData = map[string][]byte{
"credentials_type": []byte("shared_key"), // Valid values: shared_key, client_secret, default_credentials
"client_id_key": []byte(""),
"client_secret_key": []byte(""),
"resource_group_key": []byte(""),
"storage_account_key": []byte(testStoragekey),
"subscription_id_key": []byte(""),
"tenant_id_key": []byte(""),
}
// Azure registry secret data with service principal (client_secret) authentication
azureRegistrySPSecretData = map[string][]byte{
"credentials_type": []byte("client_secret"), // Service principal authentication
"client_id_key": []byte(testClientID),
"client_secret_key": []byte(testClientSecret),
"resource_group_key": []byte(testResourceGroup),
"storage_account_key": []byte(testStoragekey),
"subscription_id_key": []byte(testSubscriptionID),
"tenant_id_key": []byte(testTenantID),
}
// Azure registry secret data with managed identity (default_credentials) authentication
azureRegistryManagedIdentitySecretData = map[string][]byte{
"credentials_type": []byte("default_credentials"), // Managed identity/workload identity
"client_id_key": []byte(testClientID),
"client_secret_key": []byte(""),
"resource_group_key": []byte(testResourceGroup),
"storage_account_key": []byte(testStoragekey),
"subscription_id_key": []byte(testSubscriptionID),
"tenant_id_key": []byte(""),
}
)

var testAWSEnvVar = cloudProviderEnvVarMap["aws"]
Expand Down Expand Up @@ -632,6 +646,44 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
wantProfile: "test-sp-profile",
matchProfile: true,
},
{
name: "given azure bsl & managed identity credentials, appropriate env var for the container are returned",
bsl: &velerov1.BackupStorageLocation{
ObjectMeta: metav1.ObjectMeta{
Name: "test-bsl",
Namespace: "test-ns",
},
Spec: velerov1.BackupStorageLocationSpec{
Provider: AzureProvider,
StorageType: velerov1.StorageType{
ObjectStorage: &velerov1.ObjectStorageLocation{
Bucket: "azure-bucket",
},
},
Config: map[string]string{
StorageAccount: "velero-azure-account",
ResourceGroup: testResourceGroup,
"subscriptionId": testSubscriptionID,
},
},
},
secret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "cloud-credentials-azure",
Namespace: "test-ns",
},
Data: secretAzureData,
},
registrySecret: &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "oadp-test-bsl-azure-registry-secret",
Namespace: "test-ns",
},
Data: azureRegistryManagedIdentitySecretData,
},
wantProfile: "test-mi-profile",
matchProfile: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand All @@ -658,11 +710,16 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureAADEndpointEnvVarKey,
Value: "",
Name: RegistryStorageAzureCredentialsTypeEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "credentials_type",
},
},
},
{
Name: RegistryStorageAzureSPNClientIDEnvVarKey,
Name: RegistryStorageAzureCredentialsClientIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand All @@ -671,7 +728,7 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureSPNClientSecretEnvVarKey,
Name: RegistryStorageAzureCredentialsSecretEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand All @@ -680,7 +737,7 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureSPNTenantIDEnvVarKey,
Name: RegistryStorageAzureCredentialsTenantIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand Down Expand Up @@ -713,11 +770,77 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureAADEndpointEnvVarKey,
Value: "",
Name: RegistryStorageAzureCredentialsTypeEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "credentials_type",
},
},
},
{
Name: RegistryStorageAzureCredentialsClientIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "client_id_key",
},
},
},
{
Name: RegistryStorageAzureCredentialsSecretEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "client_secret_key",
},
},
},
{
Name: RegistryStorageAzureCredentialsTenantIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "tenant_id_key",
},
},
},
}
}
if tt.wantProfile == "test-mi-profile" {
tt.wantRegistryContainerEnvVar = []corev1.EnvVar{
{
Name: RegistryStorageEnvVarKey,
Value: Azure,
},
{
Name: RegistryStorageAzureContainerEnvVarKey,
Value: "azure-bucket",
},
{
Name: RegistryStorageAzureAccountnameEnvVarKey,
Value: "velero-azure-account",
},
{
Name: RegistryStorageAzureAccountkeyEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "storage_account_key",
},
},
},
{
Name: RegistryStorageAzureCredentialsTypeEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Key: "credentials_type",
},
},
},
{
Name: RegistryStorageAzureSPNClientIDEnvVarKey,
Name: RegistryStorageAzureCredentialsClientIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand All @@ -726,7 +849,7 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureSPNClientSecretEnvVarKey,
Name: RegistryStorageAzureCredentialsSecretEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand All @@ -735,7 +858,7 @@ func Test_getAzureRegistryEnvVars(t *testing.T) {
},
},
{
Name: RegistryStorageAzureSPNTenantIDEnvVarKey,
Name: RegistryStorageAzureCredentialsTenantIDEnvVarKey,
ValueFrom: &corev1.EnvVarSource{
SecretKeyRef: &corev1.SecretKeySelector{
LocalObjectReference: corev1.LocalObjectReference{Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry-secret"},
Expand Down