deployment failed - exclusive access to storage account

[mjudeikis]

Code="DeploymentFailed" Message="Deployment failed." Details=[{"message":"{"code":"DeploymentFailed","message":"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/DeployOperations for usage details.","target":null,"details":[{"code":"Conflict","message":"{\r\n \"error\": {\r\n \"code\": \"StorageAccountOperationInProgress\",\r\n \"message\": \"An operation is currently performing on this storage account that requires exclusive access.\"\r\n }\r\n}"}],"innererror":null,"additionalInfo":null}"}]
/home/cloud-user/agent/_work/4/go/src/github.com/Azure/ARO-RP/test/e2e/setup.go:199

time="2020-11-26T11:44:24Z" level=info msg="running step [Action github.com/Azure/ARO-RP/pkg/cluster.(*manager).createDNS-fm]" func="steps.Run()" file="pkg/util/steps/runner.go:30" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:26Z" level=info msg="running step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/cluster.(*manager).Install.func1]]" func="steps.Run()" file="pkg/util/steps/runner.go:30" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:26Z" level=info msg="creating resource group" func="cluster.(*manager).deployStorageTemplate()" file="pkg/cluster/deploystorage.go:113" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:27Z" level=info msg="development mode: applying resource group role assignment" func="env.(*dev).CreateARMResourceGroupRoleAssignment()" file="pkg/env/dev.go:123"
time="2020-11-26T11:44:29Z" level=info msg="deploying storage template" func="cluster.(*manager).deployARMTemplate()" file="pkg/cluster/arm.go:21" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:29Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=00400095-e30c-446f-8210-c32de5873c48 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:30Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.028769124 request_id=00400095-e30c-446f-8210-c32de5873c48 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:40Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=80f4f762-64d2-40e2-94ed-84f48d997733 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:40Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.015406098 request_id=80f4f762-64d2-40e2-94ed-84f48d997733 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:50Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=617e27ad-6125-4f2d-9cc8-349211b8e2fb request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:44:50Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.013804763 request_id=617e27ad-6125-4f2d-9cc8-349211b8e2fb request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:00Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=97607312-a6b2-4431-b142-d86b4151aa32 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:00Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.018789783 request_id=97607312-a6b2-4431-b142-d86b4151aa32 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:10Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=2bf7ed02-97e5-4e44-b675-8ff76aa6814d request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:10Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.014176795 request_id=2bf7ed02-97e5-4e44-b675-8ff76aa6814d request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:20Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=5fa22732-1d5e-4a0f-9402-d691ead2b111 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:20Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.041206372 request_id=5fa22732-1d5e-4a0f-9402-d691ead2b111 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:30Z" level=info msg="read request" func="middleware.Log.func1.1()" file="pkg/frontend/middleware/log.go:97" client_principal_name= client_request_id= component=access correlation_id= request_id=eb1384c8-a720-45ef-89c8-d68d937c95a8 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:30Z" level=info msg="sent response" func="middleware.Log.func1.1.1()" file="pkg/frontend/middleware/log.go:105" body_read_bytes=0 body_written_bytes=304 client_principal_name= client_request_id= component=access correlation_id= duration=0.011944306 request_id=eb1384c8-a720-45ef-89c8-d68d937c95a8 request_method=GET request_path=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/providers/microsoft.redhatopenshift/locations/eastus/operationsstatus/129216ca-1a2c-483b-a432-66288fbb0c29 request_proto=HTTP/1.1 request_remote_addr="127.0.0.1:44874" request_user_agent="Go/go1.14.9 (amd64-linux) go-autorest/v14.2.1 Azure-SDK-For-Go/0.0.0 redhatopenshift/2021-01-31-preview" response_status_code=200 subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=error msg="step [AuthorizationRefreshingAction [Action github.com/Azure/ARO-RP/pkg/cluster.(*manager).Install.func1]] encountered error: 400: DeploymentFailed: : Deployment failed. Details: : : {\"code\":\"DeploymentFailed\",\"message\":\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/DeployOperations for usage details.\",\"target\":null,\"details\":[{\"code\":\"Conflict\",\"message\":\"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"StorageAccountOperationInProgress\\\",\\r\\n    \\\"message\\\": \\\"An operation is currently performing on this storage account that requires exclusive access.\\\"\\r\\n  }\\r\\n}\"}],\"innererror\":null,\"additionalInfo\":null}" func="steps.Run()" file="pkg/util/steps/runner.go:34" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg="github.com/Azure/ARO-RP/pkg/cluster.(*manager).logClusterVersion-fm: null" func="cluster.(*manager).gatherFailureLogs()" file="pkg/cluster/gatherlogs.go:34" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg="github.com/Azure/ARO-RP/pkg/cluster.(*manager).logNodes-fm: null" func="cluster.(*manager).gatherFailureLogs()" file="pkg/cluster/gatherlogs.go:34" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg="github.com/Azure/ARO-RP/pkg/cluster.(*manager).logClusterOperators-fm: null" func="cluster.(*manager).gatherFailureLogs()" file="pkg/cluster/gatherlogs.go:34" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg="github.com/Azure/ARO-RP/pkg/cluster.(*manager).logIngressControllers-fm: null" func="cluster.(*manager).gatherFailureLogs()" file="pkg/cluster/gatherlogs.go:34" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg="400: DeploymentFailed: : Deployment failed. Details: : : {\"code\":\"DeploymentFailed\",\"message\":\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/DeployOperations for usage details.\",\"target\":null,\"details\":[{\"code\":\"Conflict\",\"message\":\"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"StorageAccountOperationInProgress\\\",\\r\\n    \\\"message\\\": \\\"An operation is currently performing on this storage account that requires exclusive access.\\\"\\r\\n  }\\r\\n}\"}],\"innererror\":null,\"additionalInfo\":null}" func="backend.(*openShiftClusterBackend).updateAsyncOperation.func1()" file="pkg/backend/openshiftcluster.go:218" client_principal_name= client_request_id= component=backend correlation_id= request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb
time="2020-11-26T11:45:39Z" level=info msg=done func="backend.(*openShiftClusterBackend).try.func1.1()" file="pkg/backend/openshiftcluster.go:73" client_principal_name= client_request_id= component=backend correlation_id= duration=89.691648148 request_id=7640f3dd-05ba-4b3c-8ab0-d90b86bea6f7 resource_group=v4-e2e-v37130758-eastus resource_id=/subscriptions/46626fc5-476d-41ad-8c76-2ec49c6994eb/resourcegroups/v4-e2e-v37130758-eastus/providers/microsoft.redhatopenshift/openshiftclusters/v4-e2e-v37130758-eastus resource_name=v4-e2e-v37130758-eastus subscription_id=46626fc5-476d-41ad-8c76-2ec49c6994eb