From ef7c12885e21d7fd84b487ae2faa983cff83bb11 Mon Sep 17 00:00:00 2001 From: Les Orchard Date: Tue, 9 Nov 2021 16:24:24 -0800 Subject: [PATCH] EXP-1865 add isRollout boolean property to Experiment type Because: * we want to implement support for an isRollout flag in experiments This commit: * adds an isRollout boolean to the NimbusExperiment type * introduces example recipes for testing * adds a test for boolean property schema validation for experiments --- .../android-96-rollout.json | 54 ++++++++++++ .../desktop-97-rollout.json | 81 ++++++++++++++++++ .../ios-39-rollout.json | 85 +++++++++++++++++++ test/test-experiment-recipe-firefox.ts | 8 ++ types/experiments.ts | 8 ++ 5 files changed, 236 insertions(+) create mode 100644 data/experiment-recipe-samples/android-96-rollout.json create mode 100644 data/experiment-recipe-samples/desktop-97-rollout.json create mode 100644 data/experiment-recipe-samples/ios-39-rollout.json diff --git a/data/experiment-recipe-samples/android-96-rollout.json b/data/experiment-recipe-samples/android-96-rollout.json new file mode 100644 index 00000000..7df40e53 --- /dev/null +++ b/data/experiment-recipe-samples/android-96-rollout.json @@ -0,0 +1,54 @@ +{ + "appId": "org.mozilla.firefox", + "appName": "fenix", + "application": "org.mozilla.firefox", + "arguments": {}, + "branches": [ + { + "feature": { + "enabled": true, + "featureId": "homescreen", + "value": { + "sections-enabled": { + "jumpBackIn": true, + "pocket": true, + "recentExplorations": true, + "recentlySaved": true, + "topSites": true + } + } + }, + "ratio": 1, + "slug": "control" + } + ], + "bucketConfig": { + "count": 9000, + "namespace": "fenix-homescreen-release-2-rollout", + "randomizationUnit": "nimbus_id", + "start": 0, + "total": 10000 + }, + "channel": "release", + "endDate": null, + "featureIds": ["homescreen"], + "id": "mobile-rollout-example-all-users-home-screen-android-production", + "isEnrollmentPaused": false, + "isRollout": true, + "outcomes": [ + { + "priority": "secondary", + "slug": "default-browser" + } + ], + "probeSets": [], + "proposedDuration": 45, + "proposedEnrollment": 28, + "referenceBranch": "control", + "schemaVersion": "1.6.2", + "slug": "mobile-rollout-example-all-users-home-screen-android-production", + "startDate": null, + "targeting": "true", + "userFacingDescription": "We want to understand the impact of the new Home screen on our users, to be measured by increased usage.", + "userFacingName": "Mobile Rollout Example All Users Home Screen Android Production" +} diff --git a/data/experiment-recipe-samples/desktop-97-rollout.json b/data/experiment-recipe-samples/desktop-97-rollout.json new file mode 100644 index 00000000..e709d416 --- /dev/null +++ b/data/experiment-recipe-samples/desktop-97-rollout.json @@ -0,0 +1,81 @@ +{ + "appId": "firefox-desktop", + "appName": "firefox_desktop", + "application": "firefox-desktop", + "arguments": {}, + "branches": [ + { + "feature": { + "featureId": "unused-feature-id-for-legacy-support", + "enabled": false, + "value": {} + }, + "features": [ + { + "enabled": true, + "featureId": "pocketNewtab", + "value": {} + }, + { + "featureId": "upgradeDialog", + "value": { + "enabled": false + } + } + ], + "ratio": 1, + "slug": "control" + }, + { + "feature": { + "featureId": "unused-feature-id-for-legacy-support", + "enabled": false, + "value": {} + }, + "features": [ + { + "enabled": true, + "featureId": "pocketNewtab", + "value": { + "compactLayout": true, + "lastCardMessageEnabled": true, + "loadMore": true, + "newFooterSection": true + } + }, + { + "featureId": "upgradeDialog", + "value": { + "enabled": true + } + } + ], + "ratio": 1, + "slug": "treatment" + } + ], + "bucketConfig": { + "count": 10000, + "namespace": "firefox-desktop-rollout-test-namespace-rollout", + "randomizationUnit": "normandy_id", + "start": 0, + "total": 10000 + }, + "channel": "nightly", + "endDate": null, + "featureIds": ["upgradeDialog", "pocketNewtab"], + "id": "firefox-desktop-rollout-test", + "isEnrollmentPaused": false, + "isRollout": true, + "outcomes": [], + "probeSets": [], + "proposedDuration": 63, + "proposedEnrollment": 7, + "referenceBranch": "control", + "schemaVersion": "1.6.2", + "slug": "firefox-desktop-rollout-test", + "startDate": "2021-10-26", + "targeting": "true", + "userFacingDescription": "Experimenting on onboarding content when you upgrade Firefox.", + "userFacingName": "Firefox Desktop Rollout Test" +} diff --git a/data/experiment-recipe-samples/ios-39-rollout.json b/data/experiment-recipe-samples/ios-39-rollout.json new file mode 100644 index 00000000..58e3b88c --- /dev/null +++ b/data/experiment-recipe-samples/ios-39-rollout.json @@ -0,0 +1,85 @@ +{ + "appId": "org.mozilla.ios.Firefox", + "appName": "firefox_ios", + "application": "org.mozilla.ios.Firefox", + "arguments": {}, + "branches": [ + { + "features": [ + { + "enabled": true, + "featureId": "onboarding-default-browser", + "value": { + "should-hide-title-image": false + } + }, + { + "enabled": true, + "featureId": "homescreen", + "value": { + "sections-enabled": { + "jumpBackIn": false, + "libraryShortcuts": true, + "recentlySaved": false + } + } + } + ], + "ratio": 1, + "slug": "control" + }, + { + "features": [ + { + "enabled": true, + "featureId": "onboarding-default-browser", + "value": { + "should-hide-title-image": true + } + }, + { + "enabled": true, + "featureId": "homescreen", + "value": { + "sections-enabled": { + "jumpBackIn": true, + "libraryShortcuts": false, + "recentlySaved": true + } + } + } + ], + "ratio": 1, + "slug": "treatment" + } + ], + "bucketConfig": { + "count": 10000, + "namespace": "firefox-ios-rollout-test-rollout", + "randomizationUnit": "nimbus_id", + "start": 0, + "total": 10000 + }, + "channel": "release", + "endDate": null, + "featureIds": ["onboarding-default-browser", "homescreen"], + "id": "firefox-ios-rollout-test", + "isEnrollmentPaused": false, + "isRollout": true, + "outcomes": [ + { + "priority": "secondary", + "slug": "default_browser" + } + ], + "probeSets": [], + "proposedDuration": 28, + "proposedEnrollment": 7, + "referenceBranch": "control", + "schemaVersion": "1.6.2", + "slug": "firefox-ios-rollout-test", + "startDate": "2021-11-01", + "targeting": "true", + "userFacingDescription": "firefox-ios-rollout-test", + "userFacingName": "firefox-ios-rollout-test" +} diff --git a/test/test-experiment-recipe-firefox.ts b/test/test-experiment-recipe-firefox.ts index 96e9d966..dc40f0cb 100644 --- a/test/test-experiment-recipe-firefox.ts +++ b/test/test-experiment-recipe-firefox.ts @@ -14,6 +14,14 @@ describe("experiment schemas", () => { assert.equal(result.ok, false, "validation should fail"); assert.propertyVal(result.errors[0], "message", 'should match format "date"'); }); + it("should fail on a non-boolean for a boolean", async () => { + const result = typeGuards.experiments_checkNimbusExperiment({ + ...TEST_EXPERIMENT, + isRollout: "foo", + }); + assert.equal(result.ok, false, "validation should fail"); + assert.propertyVal(result.errors[0], "message", "should be boolean"); + }); it("should fail on a non-integer value for a number", async () => { const result = typeGuards.experiments_checkNimbusExperiment({ ...TEST_EXPERIMENT, diff --git a/types/experiments.ts b/types/experiments.ts index 94703e57..bb03e16d 100644 --- a/types/experiments.ts +++ b/types/experiments.ts @@ -47,6 +47,14 @@ export interface NimbusExperiment { */ isEnrollmentPaused: boolean; + /** + * When this property is set to true, treat this experiment as a rollout. + * Rollouts are currently handled as single-branch experiments separated + * from the bucketing namespace for normal experiments. + * See also: https://mozilla-hub.atlassian.net/browse/SDK-405 + */ + isRollout?: boolean; + /** Bucketing configuration */ bucketConfig: BucketConfig;