Merge branch 'main' into fcappelli/breakage_improvement_mac

# Conflicts:
#	Package.swift

.github/workflows/codeql.yml

@@ -36,7 +36,7 @@ jobs:
         languages: "swift"
 
     - name: Select Xcode
-      run: sudo xcode-select -s /Applications/Xcode_14.3.app/Contents/Developer
+      run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
 
     - run: swift build
 

.github/workflows/pr.yml

@@ -58,7 +58,7 @@ jobs:
             ${{ runner.os }}-spm-
 
       - name: Select Xcode
-        run: sudo xcode-select -s /Applications/Xcode_15.0.app/Contents/Developer
+        run: sudo xcode-select -s /Applications/Xcode_$(<.xcode-version).app/Contents/Developer
 
       - name: Install xcbeautify
         continue-on-error: true

.xcode-version

@@ -0,0 +1 @@
+15.1

Package.resolved

@@ -14,8 +14,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/content-scope-scripts",
       "state" : {
-        "revision" : "b7ad9843e70cede0c2ca9c4260d970f62cb28156",
-        "version" : "4.52.0"
+        "revision" : "bb027f14bec7fbb1a85d308139e7a66686da160e",
+        "version" : "4.59.0"
       }
     },
     {

Package.swift

@@ -40,7 +40,7 @@ let package = Package(
         .package(url: "https://github.com/duckduckgo/sync_crypto", exact: "0.2.0"),
         .package(url: "https://github.com/gumob/PunycodeSwift.git", exact: "2.1.0"),
         .package(url: "https://github.com/duckduckgo/privacy-dashboard", branch: "12-14-feat_macos_allow_direct_navigation_to_breakage_form"),
-        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "4.52.0"),
+        .package(url: "https://github.com/duckduckgo/content-scope-scripts", exact: "4.59.0"),
         .package(url: "https://github.com/httpswift/swifter.git", exact: "1.5.0"),
         .package(url: "https://github.com/duckduckgo/bloom_cpp.git", exact: "3.0.0"),
         .package(url: "https://github.com/duckduckgo/wireguard-apple", exact: "1.1.1"),
@@ -127,6 +127,7 @@ let package = Package(
         .target(
             name: "DDGSync",
             dependencies: [
+                "BrowserServicesKit",
                 "Common",
                 .product(name: "DDGSyncCrypto", package: "sync_crypto"),
                 "Networking"

Sources/BrowserServicesKit/LinkProtection/LinkProtection.swift

@@ -36,10 +36,26 @@ public struct LinkProtection {
                                              errorReporting: errorReporting)
     }
 
+    private func makeNewRequest(changingUrl url: URL, inRequest request: URLRequest) -> URLRequest {
+        var newRequest = request
+        newRequest.url = url
+        return newRequest
+    }
+
     public mutating func setMainFrameUrl(_ url: URL?) {
         mainFrameUrl = url
     }
 
+    public func getCleanURLRequest(from urlRequest: URLRequest,
+                                   onStartExtracting: () -> Void,
+                                   onFinishExtracting: @escaping () -> Void,
+                                   completion: @escaping (URLRequest) -> Void) {
+        getCleanURL(from: urlRequest.url!, onStartExtracting: onStartExtracting, onFinishExtracting: onFinishExtracting) { newUrl in
+            let newRequest = makeNewRequest(changingUrl: newUrl, inRequest: urlRequest)
+            completion(newRequest)
+        }
+    }
+
     public func getCleanURL(from url: URL,
                             onStartExtracting: () -> Void,
                             onFinishExtracting: @escaping () -> Void,
@@ -77,11 +93,12 @@ public struct LinkProtection {
 
     // swiftlint:disable function_parameter_count
     public func requestTrackingLinkRewrite(initiatingURL: URL?,
-                                           destinationURL: URL,
+                                           destinationRequest: URLRequest,
                                            onStartExtracting: () -> Void,
                                            onFinishExtracting: @escaping () -> Void,
-                                           onLinkRewrite: @escaping (URL) -> Void,
+                                           onLinkRewrite: @escaping (URLRequest) -> Void,
                                            policyDecisionHandler: @escaping (Bool) -> Void) -> Bool {
+        let destinationURL = destinationRequest.url
         if let mainFrameUrl = mainFrameUrl, destinationURL != mainFrameUrl {
             // If mainFrameUrl is set and is different from destinationURL we will assume this is a redirect
             // We do not rewrite redirects due to breakage concerns
@@ -91,7 +108,7 @@ public struct LinkProtection {
         var didRewriteLink = false
         if let newURL = linkCleaner.extractCanonicalFromAMPLink(initiator: initiatingURL, destination: destinationURL) {
             policyDecisionHandler(false)
-            onLinkRewrite(newURL)
+            onLinkRewrite(makeNewRequest(changingUrl: newURL, inRequest: destinationRequest))
             didRewriteLink = true
         } else if ampExtractor.urlContainsAMPKeyword(destinationURL) {
             onStartExtracting()
@@ -103,13 +120,13 @@ public struct LinkProtection {
                 }
 
                 policyDecisionHandler(false)
-                onLinkRewrite(canonical)
+                onLinkRewrite(makeNewRequest(changingUrl: canonical, inRequest: destinationRequest))
             }
             didRewriteLink = true
         } else if let newURL = linkCleaner.cleanTrackingParameters(initiator: initiatingURL, url: destinationURL) {
             if newURL != destinationURL {
                 policyDecisionHandler(false)
-                onLinkRewrite(newURL)
+                onLinkRewrite(makeNewRequest(changingUrl: newURL, inRequest: destinationRequest))
                 didRewriteLink = true
             }
         }
@@ -121,10 +138,10 @@ public struct LinkProtection {
                                            navigationAction: WKNavigationAction,
                                            onStartExtracting: () -> Void,
                                            onFinishExtracting: @escaping () -> Void,
-                                           onLinkRewrite: @escaping (URL, WKNavigationAction) -> Void,
+                                           onLinkRewrite: @escaping (URLRequest, WKNavigationAction) -> Void,
                                            policyDecisionHandler: @escaping (WKNavigationActionPolicy) -> Void) -> Bool {
         requestTrackingLinkRewrite(initiatingURL: initiatingURL,
-                                   destinationURL: navigationAction.request.url!,
+                                   destinationRequest: navigationAction.request,
                                    onStartExtracting: onStartExtracting,
                                    onFinishExtracting: onFinishExtracting,
                                    onLinkRewrite: { onLinkRewrite($0, navigationAction) },
@@ -134,13 +151,13 @@ public struct LinkProtection {
 
     @MainActor
     public func requestTrackingLinkRewrite(initiatingURL: URL?,
-                                           destinationURL: URL,
+                                           destinationRequest: URLRequest,
                                            onStartExtracting: () -> Void,
                                            onFinishExtracting: @escaping () -> Void,
-                                           onLinkRewrite: @escaping (URL) -> Void) async -> Bool? {
+                                           onLinkRewrite: @escaping (URLRequest) -> Void) async -> Bool? {
         await withCheckedContinuation { continuation in
             let didRewriteLink = requestTrackingLinkRewrite(initiatingURL: initiatingURL,
-                                                            destinationURL: destinationURL,
+                                                            destinationRequest: destinationRequest,
                                                             onStartExtracting: onStartExtracting,
                                                             onFinishExtracting: onFinishExtracting,
                                                             onLinkRewrite: onLinkRewrite) { navigationActionPolicy in

Sources/BrowserServicesKit/PrivacyConfig/Features/PrivacyFeature.swift

@@ -41,6 +41,7 @@ public enum PrivacyFeature: String {
     case newTabContinueSetUp
     case networkProtection
     case dbp
+    case sync
 }
 
 /// An abstraction to be implemented by any "subfeature" of a given `PrivacyConfiguration` feature.
@@ -82,3 +83,14 @@ public enum DBPSubfeature: String, Equatable, PrivacySubfeature {
     case waitlist
     case waitlistBetaActive
 }
+
+public enum SyncSubfeature: String, PrivacySubfeature {
+    public var parent: PrivacyFeature {
+        .sync
+    }
+
+    case level0ShowSync
+    case level1AllowDataSyncing
+    case level2AllowSetupFlows
+    case level3AllowCreateAccount
+}

Sources/BrowserServicesKit/PrivacyConfig/PrivacyConfigurationManager.swift

@@ -31,6 +31,7 @@ public protocol PrivacyConfigurationManaging: AnyObject {
     var currentConfig: Data { get }
     var updatesPublisher: AnyPublisher<Void, Never> { get }
     var privacyConfig: PrivacyConfiguration { get }
+    var internalUserDecider: InternalUserDecider { get }
 
     @discardableResult func reload(etag: String?, data: Data?) -> PrivacyConfigurationManager.ReloadResult
 }
@@ -53,7 +54,7 @@ public class PrivacyConfigurationManager: PrivacyConfigurationManaging {
     private let embeddedDataProvider: EmbeddedDataProvider
     private let localProtection: DomainsProtectionStore
     private let errorReporting: EventMapping<ContentBlockerDebugEvents>?
-    private let internalUserDecider: InternalUserDecider
+    public let internalUserDecider: InternalUserDecider
     private let installDate: Date?
 
     private let updatesSubject = PassthroughSubject<Void, Never>()

Sources/BrowserServicesKit/SecureVault/SecureVaultManager.swift

@@ -229,9 +229,11 @@ extension SecureVaultManager: AutofillSecureVaultDelegate {
                 autosaveAccountCreatedInSession = false
             }
 
-            // Do not autosave anything if user has requested to never be prompted to save credentials for this domain
+            // Do not autosave anything if user has requested to never be prompted to save credentials for this domain and are using DDG as password manager
             if let neverPrompt = try vault?.hasNeverPromptWebsitesFor(domain: domain), neverPrompt {
-                return
+                guard let passwordManager = passwordManager, passwordManager.isEnabled else {
+                    return
+                }
             }
 
             // Validate the existing account exists and matches the domain and fetch the credentials

Sources/Common/Extensions/URLExtension.swift

@@ -17,6 +17,7 @@
 //
 
 import Foundation
+import Network
 
 extension URL {
 
@@ -193,6 +194,12 @@ extension URL {
                     // could be a local domain but user needs to use the protocol to specify that
                     return nil
                 }
+                if IPv4Address(String(hostname)) != nil {
+                    // Require 4 octets specified explicitly for an IPv4 address (avoid 1.4 -> 1.0.0.4 expansion)
+                    guard hostname.split(separator: ".").count == 4 else {
+                        return nil
+                    }
+                }
             } else {
                 return nil
             }

Sources/Configuration/ConfigurationFetching.swift

@@ -22,7 +22,7 @@ import Networking
 
 protocol ConfigurationFetching {
 
-    func fetch(_ configuration: Configuration) async throws
+    func fetch(_ configuration: Configuration, isDebug: Bool) async throws
     func fetch(all configurations: [Configuration]) async throws
 
 }
@@ -74,8 +74,9 @@ public final class ConfigurationFetcher: ConfigurationFetching {
     - Throws:
       An error of type Error is thrown if the configuration fails to fetch or validate.
     */
-    public func fetch(_ configuration: Configuration) async throws {
-        let fetchResult = try await fetch(from: configuration.url, withEtag: etag(for: configuration), requirements: .default)
+    public func fetch(_ configuration: Configuration, isDebug: Bool = false) async throws {
+        let requirements: APIResponseRequirements = isDebug ? .requireNonEmptyData : .default
+        let fetchResult = try await fetch(from: configuration.url, withEtag: etag(for: configuration), requirements: requirements)
         if let data = fetchResult.data {
             try validator.validate(data, for: configuration)
         }
@@ -134,7 +135,7 @@ public final class ConfigurationFetcher: ConfigurationFetching {
         let log = log
         let request = APIRequest(configuration: configuration, requirements: requirements, urlSession: urlSession, log: log)
         let (data, response) = try await request.fetch()
-        return (response.etag!, data)
+        return (response.etag ?? "", data)
     }
 
     private func store(_ result: ConfigurationFetchResult, for configuration: Configuration) throws {

Sources/DDGSync/DDGSync.swift

@@ -16,15 +16,21 @@
 //  limitations under the License.
 //
 
-import Foundation
+import BrowserServicesKit
 import Combine
-import DDGSyncCrypto
 import Common
+import DDGSyncCrypto
+import Foundation
 
 public class DDGSync: DDGSyncing {
 
     public static let bundle = Bundle.module
 
+    @Published public private(set) var featureFlags: SyncFeatureFlags = .all
+    public var featureFlagsPublisher: AnyPublisher<SyncFeatureFlags, Never> {
+        $featureFlags.eraseToAnyPublisher()
+    }
+
     enum Constants {
         public static let syncEnabledKey = "com.duckduckgo.sync.enabled"
     }
@@ -55,9 +61,15 @@ public class DDGSync: DDGSyncing {
     /// This is the constructor intended for use by app clients.
     public convenience init(dataProvidersSource: DataProvidersSource,
                             errorEvents: EventMapping<SyncError>,
+                            privacyConfigurationManager: PrivacyConfigurationManaging,
                             log: @escaping @autoclosure () -> OSLog = .disabled,
                             environment: ServerEnvironment = .production) {
-        let dependencies = ProductionDependencies(serverEnvironment: environment, errorEvents: errorEvents, log: log())
+        let dependencies = ProductionDependencies(
+            serverEnvironment: environment,
+            privacyConfigurationManager: privacyConfigurationManager,
+            errorEvents: errorEvents,
+            log: log()
+        )
         self.init(dataProvidersSource: dataProvidersSource, dependencies: dependencies)
     }
 
@@ -189,6 +201,18 @@ public class DDGSync: DDGSyncing {
     init(dataProvidersSource: DataProvidersSource, dependencies: SyncDependencies) {
         self.dataProvidersSource = dataProvidersSource
         self.dependencies = dependencies
+
+        featureFlagsCancellable = Publishers.Merge(
+            self.dependencies.privacyConfigurationManager.updatesPublisher,
+            self.dependencies.privacyConfigurationManager.internalUserDecider.isInternalUserPublisher.map { _ in () })
+            .compactMap { [weak self] in
+                self?.dependencies.privacyConfigurationManager.privacyConfig
+            }
+            .prepend(dependencies.privacyConfigurationManager.privacyConfig)
+            .map(SyncFeatureFlags.init)
+            .removeDuplicates()
+            .receive(on: DispatchQueue.main)
+            .assign(to: \.featureFlags, onWeaklyHeld: self)
     }
 
     public func initializeIfNeeded() {
@@ -229,6 +253,7 @@ public class DDGSync: DDGSyncing {
             dependencies.scheduler.isEnabled = false
             startSyncCancellable?.cancel()
             syncQueueCancellable?.cancel()
+            isDataSyncingFeatureFlagEnabledCancellable?.cancel()
             try syncQueue?.dataProviders.forEach { try $0.deregisterFeature() }
             syncQueue = nil
             authState = .inactive
@@ -291,6 +316,9 @@ public class DDGSync: DDGSyncing {
                 self?.syncQueue?.resumeQueue()
             }
 
+        isDataSyncingFeatureFlagEnabledCancellable = featureFlagsPublisher.prepend(featureFlags).map { $0.contains(.dataSyncing) }
+            .assign(to: \.isDataSyncingFeatureFlagEnabled, onWeaklyHeld: syncQueue)
+
         dependencies.scheduler.isEnabled = true
         self.syncQueue = syncQueue
     }
@@ -317,6 +345,8 @@ public class DDGSync: DDGSyncing {
     private var startSyncCancellable: AnyCancellable?
     private var cancelSyncCancellable: AnyCancellable?
     private var resumeSyncCancellable: AnyCancellable?
+    private var featureFlagsCancellable: AnyCancellable?
+    private var isDataSyncingFeatureFlagEnabledCancellable: AnyCancellable?
 
     private var syncQueue: SyncQueue?
     private var syncQueueCancellable: AnyCancellable?

Sources/DDGSync/DDGSyncing.swift

@@ -16,9 +16,10 @@
 //  limitations under the License.
 //
 
-import Foundation
-import DDGSyncCrypto
+import BrowserServicesKit
 import Combine
+import DDGSyncCrypto
+import Foundation
 
 public enum SyncAuthState: String, Sendable, Codable {
     /// Sync engine is not initialized.
@@ -48,6 +49,16 @@ public protocol DDGSyncing: DDGSyncingDebuggingSupport {
 
     var dataProvidersSource: DataProvidersSource? { get set }
 
+    /**
+     Describes current availability of sync features.
+     */
+    var featureFlags: SyncFeatureFlags { get }
+
+    /**
+     Emits changes to current availability of sync features
+     */
+    var featureFlagsPublisher: AnyPublisher<SyncFeatureFlags, Never> { get }
+
     /**
      Describes current state of sync account.