From aacde2095f5b7e442086e7f2ce94179cc57c61e1 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Fri, 7 Jun 2024 17:49:51 -0400 Subject: [PATCH 01/14] Delete more Events in database cleanup and avoid recreating EventReferences --- Nos/Controller/PersistenceController.swift | 4 +- .../EventReference+CoreDataClass.swift | 7 ++ Nos/Service/DatabaseCleaner.swift | 41 ++----- NosTests/Service/DatabaseCleanerTests.swift | 107 ++++++++++++++++++ NosTests/Test Helpers/EventFixture.swift | 8 ++ 5 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 NosTests/Test Helpers/EventFixture.swift diff --git a/Nos/Controller/PersistenceController.swift b/Nos/Controller/PersistenceController.swift index 391997ace..514f9e594 100644 --- a/Nos/Controller/PersistenceController.swift +++ b/Nos/Controller/PersistenceController.swift @@ -198,7 +198,9 @@ class PersistenceController { let context = newBackgroundContext() do { - try await DatabaseCleaner.cleanupEntities(before: Date.now, for: authorKey, in: context) + // We don't want to delete any events downloaded after app boot, so we subtract 60 seconds + let cleanupBeforeDate = Date.now.addingTimeInterval(-60) // Subtract 60 seconds + try await DatabaseCleaner.cleanupEntities(before: cleanupBeforeDate, for: authorKey, in: context) } catch { Log.optional(error) crashReporting.report("Error in database cleanup: \(error.localizedDescription)") diff --git a/Nos/Models/CoreData/EventReference+CoreDataClass.swift b/Nos/Models/CoreData/EventReference+CoreDataClass.swift index 4af0b18d8..da4ce020c 100644 --- a/Nos/Models/CoreData/EventReference+CoreDataClass.swift +++ b/Nos/Models/CoreData/EventReference+CoreDataClass.swift @@ -26,6 +26,13 @@ public class EventReference: NosManagedObject { marker.unwrap { EventReferenceMarker(rawValue: $0) } } + /// Retreives all the EventReferences + static func all() -> NSFetchRequest { + let fetchRequest = NSFetchRequest(entityName: "EventReference") + fetchRequest.sortDescriptors = [NSSortDescriptor(keyPath: \EventReference.eventId, ascending: false)] + return fetchRequest + } + /// Retreives all the EventReferences whose referencing Event has been deleted. static func orphanedRequest() -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "EventReference") diff --git a/Nos/Service/DatabaseCleaner.swift b/Nos/Service/DatabaseCleaner.swift index fd2abe6a7..3d25c0cf0 100644 --- a/Nos/Service/DatabaseCleaner.swift +++ b/Nos/Service/DatabaseCleaner.swift @@ -17,7 +17,7 @@ enum DatabaseCleaner { /// - delete any other models that are orphaned by the previous deletions /// - fix EventReferences whose referencedEvent was deleted by createing a stubbed Event static func cleanupEntities( - before date: Date, + before deleteBefore: Date, for authorKey: RawAuthorID, in context: NSManagedObjectContext ) async throws { @@ -29,31 +29,25 @@ enum DatabaseCleaner { Log.info("Database statistics: \(try await PersistenceController.databaseStatistics(from: context))") - // Delete all but the most recent n events - let eventsToKeep = 1000 - let fetchFirstEventToDelete = Event.allEventsRequest() - fetchFirstEventToDelete.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: false)] - fetchFirstEventToDelete.fetchLimit = 1 - fetchFirstEventToDelete.fetchOffset = eventsToKeep - fetchFirstEventToDelete.predicate = NSPredicate(format: "receivedAt != nil") - var deleteBefore = Date.distantPast try await context.perform { guard let currentAuthor = try? Author.find(by: authorKey, context: context) else { return } - if let firstEventToDelete = try context.fetch(fetchFirstEventToDelete).first, - let receivedAt = firstEventToDelete.receivedAt { - deleteBefore = receivedAt - } + let oldEventReferencesRequest = EventReference.all() + oldEventReferencesRequest.predicate = NSPredicate( + format: "referencedEvent.receivedAt < %@ AND referencingEvent.receivedAt < %@", + deleteBefore as CVarArg, + deleteBefore as CVarArg + ) let oldStoryCutoff = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now // Delete events older than `deleteBefore` let oldEventsRequest = NSFetchRequest(entityName: "Event") oldEventsRequest.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] - let oldEventClause = "(receivedAt <= %@ OR receivedAt == nil)" + let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" let notOwnEventClause = "(author.hexadecimalPublicKey != %@)" let readStoryClause = "(isRead = 1 AND receivedAt > %@)" let userReportClause = "(kind == \(EventKind.report.rawValue) AND " + @@ -70,6 +64,7 @@ enum DatabaseCleaner { ) let deleteRequests: [NSPersistentStoreRequest] = [ + oldEventReferencesRequest, oldEventsRequest, Event.expiredRequest(), EventReference.orphanedRequest(), @@ -94,25 +89,7 @@ enum DatabaseCleaner { } } - // Heal EventReferences - let brokenEventReferencesRequest = NSFetchRequest(entityName: "EventReference") - brokenEventReferencesRequest.sortDescriptors = [ - NSSortDescriptor(keyPath: \EventReference.eventId, ascending: false) - ] - brokenEventReferencesRequest.predicate = NSPredicate(format: "referencedEvent = nil") - let brokenEventReferences = try context.fetch(brokenEventReferencesRequest) - Log.info("Healing \(brokenEventReferences.count) EventReferences") - for eventReference in brokenEventReferences { - guard let eventID = eventReference.eventId else { - Log.error("Found an EventReference with no eventID") - continue - } - let referencedEvent = try Event.findOrCreateStubBy(id: eventID, context: context) - eventReference.referencedEvent = referencedEvent - } - try context.saveIfNeeded() - context.refreshAllObjects() } let newStatistics = try await PersistenceController.databaseStatistics(from: context) diff --git a/NosTests/Service/DatabaseCleanerTests.swift b/NosTests/Service/DatabaseCleanerTests.swift index 7bdee06da..3ad0ef532 100644 --- a/NosTests/Service/DatabaseCleanerTests.swift +++ b/NosTests/Service/DatabaseCleanerTests.swift @@ -1,8 +1,115 @@ import XCTest +import CoreData final class DatabaseCleanerTests: CoreDataTestCase { func test_emptyDatabase() async throws { try await DatabaseCleaner.cleanupEntities(before: Date.now, for: KeyFixture.alice.publicKeyHex, in: testContext) } + + func test_cleanup_deletesCorrectEventReferences() async throws { + // Arrange + let deleteBeforeDate = Date(timeIntervalSince1970: 10) + let user = KeyFixture.alice + _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + let oldEventOne = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let oldEventTwo = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 8)) + let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + + // Create a reference with two old events that should be deleted + let eventReferenceToBeDeleted = EventReference(context: testContext) + eventReferenceToBeDeleted.referencedEvent = oldEventOne + eventReferenceToBeDeleted.referencingEvent = oldEventTwo + + // Create references with a new event that should not be deleted + let eventReferenceToBeKept = EventReference(context: testContext) + eventReferenceToBeDeleted.referencingEvent = newEvent + eventReferenceToBeDeleted.referencedEvent = oldEventTwo + + try testContext.save() + + // Act + try await DatabaseCleaner.cleanupEntities( + before: deleteBeforeDate, + for: KeyFixture.alice.publicKeyHex, + in: testContext + ) + + // Assert + let eventReferences = try testContext.fetch(EventReference.all()) + XCTAssertEqual(eventReferences.count, 1) + XCTAssertEqual(eventReferences.first?.referencingEvent, newEvent) + XCTAssertEqual(eventReferences.first?.referencedEvent, oldEventTwo) + } + + func test_cleanup_deletesOldEvents() async throws { + let deleteBeforeDate = Date(timeIntervalSince1970: 10) + let user = KeyFixture.alice + _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + let oldEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + + try testContext.save() + + // Act + try await DatabaseCleaner.cleanupEntities( + before: deleteBeforeDate, + for: KeyFixture.alice.publicKeyHex, + in: testContext + ) + + // Assert + let events = try testContext.fetch(Event.allEventsRequest()) + XCTAssertEqual(events, [newEvent]) + } + + func test_cleanup_savesReferencedEvents() async throws { + let deleteBeforeDate = Date(timeIntervalSince1970: 10) + let user = KeyFixture.alice + _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + + // Create an old event that is referenced by a newer event + let oldEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + + let eventReferenceToBeDeleted = EventReference(context: testContext) + eventReferenceToBeDeleted.referencedEvent = oldEvent + eventReferenceToBeDeleted.referencingEvent = newEvent + + try testContext.save() + + // Act + try await DatabaseCleaner.cleanupEntities( + before: deleteBeforeDate, + for: KeyFixture.alice.publicKeyHex, + in: testContext + ) + + // Assert + let events = try testContext.fetch(Event.allEventsRequest()) + XCTAssertEqual(events.count, 2) + } + + // MARK: - Helpers + + private func createTestEvent( + in context: NSManagedObjectContext, + publicKey: RawAuthorID = KeyFixture.pubKeyHex, + receivedAt: Date = .now + ) throws -> Event { + let event = Event(context: context) + event.identifier = UUID().uuidString + event.createdAt = Date.now + event.receivedAt = receivedAt + event.content = "Testing nos #[0]" + event.kind = 1 + + let author = Author(context: context) + author.hexadecimalPublicKey = publicKey + event.author = author + + let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] + event.allTags = tags as NSObject + return event + } } diff --git a/NosTests/Test Helpers/EventFixture.swift b/NosTests/Test Helpers/EventFixture.swift new file mode 100644 index 000000000..f5186320d --- /dev/null +++ b/NosTests/Test Helpers/EventFixture.swift @@ -0,0 +1,8 @@ +// +// EventFixture.swift +// NosTests +// +// Created by Matthew Lorentz on 6/7/24. +// + +import Foundation From 09c9eddac6269932bdafb4241bbe417871c4fef4 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Fri, 7 Jun 2024 18:31:19 -0400 Subject: [PATCH 02/14] Refactored event creation code into EventFixture --- Nos.xcodeproj/project.pbxproj | 4 ++ NosTests/Model/AuthorTests.swift | 24 +------ NosTests/Model/EventTests.swift | 74 ++++++++------------- NosTests/Service/DatabaseCleanerTests.swift | 39 +++-------- NosTests/Test Helpers/EventFixture.swift | 38 +++++++++-- 5 files changed, 72 insertions(+), 107 deletions(-) diff --git a/Nos.xcodeproj/project.pbxproj b/Nos.xcodeproj/project.pbxproj index 27a689c15..5d321b617 100644 --- a/Nos.xcodeproj/project.pbxproj +++ b/Nos.xcodeproj/project.pbxproj @@ -241,6 +241,7 @@ C96D39272B61B6D200D3D0A1 /* RawNostrID.swift in Sources */ = {isa = PBXBuildFile; fileRef = C96D39262B61B6D200D3D0A1 /* RawNostrID.swift */; }; C96D39282B61B6D200D3D0A1 /* RawNostrID.swift in Sources */ = {isa = PBXBuildFile; fileRef = C96D39262B61B6D200D3D0A1 /* RawNostrID.swift */; }; C973364F2A7968220012D8B8 /* SetUpUNSBanner.swift in Sources */ = {isa = PBXBuildFile; fileRef = C973364E2A7968220012D8B8 /* SetUpUNSBanner.swift */; }; + C9736E5E2C13B718005BCE70 /* EventFixture.swift in Sources */ = {isa = PBXBuildFile; fileRef = C9736E5D2C13B718005BCE70 /* EventFixture.swift */; }; C973AB5B2A323167002AED16 /* Follow+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = C973AB552A323167002AED16 /* Follow+CoreDataProperties.swift */; }; C973AB5C2A323167002AED16 /* Follow+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = C973AB552A323167002AED16 /* Follow+CoreDataProperties.swift */; }; C973AB5D2A323167002AED16 /* Event+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = C973AB562A323167002AED16 /* Event+CoreDataProperties.swift */; }; @@ -677,6 +678,7 @@ C96D391A2B61AFD500D3D0A1 /* RawNostrIDTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RawNostrIDTests.swift; sourceTree = ""; }; C96D39262B61B6D200D3D0A1 /* RawNostrID.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RawNostrID.swift; sourceTree = ""; }; C973364E2A7968220012D8B8 /* SetUpUNSBanner.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SetUpUNSBanner.swift; sourceTree = ""; }; + C9736E5D2C13B718005BCE70 /* EventFixture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventFixture.swift; sourceTree = ""; }; C973AB552A323167002AED16 /* Follow+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Follow+CoreDataProperties.swift"; sourceTree = ""; }; C973AB562A323167002AED16 /* Event+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Event+CoreDataProperties.swift"; sourceTree = ""; }; C973AB572A323167002AED16 /* AuthorReference+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AuthorReference+CoreDataProperties.swift"; sourceTree = ""; }; @@ -1235,6 +1237,7 @@ C91400232B2A3894009B13B4 /* SQLiteStoreTestCase.swift */, C9C9444129F6F0E2002F2C7A /* XCTest+Eventually.swift */, 0373CE982C0910250027C856 /* XCTestCase+JSONData.swift */, + C9736E5D2C13B718005BCE70 /* EventFixture.swift */, ); path = "Test Helpers"; sourceTree = ""; @@ -2187,6 +2190,7 @@ 035729B92BE416A6005FEE85 /* GiftWrapperTests.swift in Sources */, 032634702C10C40B00E489B5 /* NostrBuildAPIClientTests.swift in Sources */, C9646EAA29B7A506007239A4 /* Analytics.swift in Sources */, + C9736E5E2C13B718005BCE70 /* EventFixture.swift in Sources */, 035729AF2BE4167E005FEE85 /* KeyPairTests.swift in Sources */, 035729AE2BE4167E005FEE85 /* FollowTests.swift in Sources */, 5BFBB2952BD9D7EB002E909F /* URLParserTests.swift in Sources */, diff --git a/NosTests/Model/AuthorTests.swift b/NosTests/Model/AuthorTests.swift index 2d8328ec9..f6692d2ce 100644 --- a/NosTests/Model/AuthorTests.swift +++ b/NosTests/Model/AuthorTests.swift @@ -216,7 +216,7 @@ final class AuthorTests: CoreDataTestCase { func test_allPostsRequest_onlyRootPosts() throws { // Arrange let publicKey = "test" - _ = try createTestEvent(in: testContext, publicKey: publicKey, deletedOn: [Relay(context: testContext)]) + _ = try EventFixture.build(in: testContext, publicKey: publicKey, deletedOn: [Relay(context: testContext)]) let author = try XCTUnwrap(Author.find(by: publicKey, context: testContext)) // Act @@ -226,26 +226,4 @@ final class AuthorTests: CoreDataTestCase { // Assert XCTAssertEqual(events.count, 0) } - - // MARK: - Helpers - - private func createTestEvent( - in context: NSManagedObjectContext, - publicKey: RawAuthorID = KeyFixture.pubKeyHex, - deletedOn: Set = [] - ) throws -> Event { - let event = Event(context: context) - event.createdAt = Date(timeIntervalSince1970: TimeInterval(1_675_264_762)) - event.content = "Testing nos #[0]" - event.deletedOn = deletedOn - event.kind = 1 - - let author = Author(context: context) - author.hexadecimalPublicKey = publicKey - event.author = author - - let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] - event.allTags = tags as NSObject - return event - } } diff --git a/NosTests/Model/EventTests.swift b/NosTests/Model/EventTests.swift index c29338df3..56506c519 100644 --- a/NosTests/Model/EventTests.swift +++ b/NosTests/Model/EventTests.swift @@ -9,7 +9,9 @@ final class EventTests: CoreDataTestCase { // MARK: - Serialization func testSerializedEventForSigning() throws { // Arrange - let event = try createTestEvent(in: testContext) + let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] + let content = "Testing nos #[0]" + let event = try EventFixture.build(in: testContext, content: content, tags: tags) // swiftlint:disable line_length let expectedString = """ [0,"32730e9dfcab797caf8380d096e548d9ef98f3af3000542f9271a91a9e3b0001",1675264762,1,[["p","d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]],"Testing nos #[0]"] @@ -28,7 +30,9 @@ final class EventTests: CoreDataTestCase { func testIdentifierCalculation() throws { // Arrange - let event = try createTestEvent(in: testContext) + let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] + let content = "Testing nos #[0]" + let event = try EventFixture.build(in: testContext, content: content, tags: tags) // Act XCTAssertEqual( @@ -39,7 +43,21 @@ final class EventTests: CoreDataTestCase { func testIdentifierCalculationWithNoTags() throws { // Arrange - let event = try createTestEventWithNoTags(in: testContext) + let content = "Testing nos #[0]" + let event = try EventFixture.build(in: testContext, content: content) + + // Act + XCTAssertEqual( + try event.calculateIdentifier(), + "bc45c3ac53de113e1400fca956048a816ad1c2e6ecceba6b1372ca597066fa9a" + ) + } + + func testIdentifierCalculationWithEmptyTags() throws { + // Arrange + let content = "Testing nos #[0]" + let event = try EventFixture.build(in: testContext, content: content, tags: nil) + // Act XCTAssertEqual( try event.calculateIdentifier(), @@ -54,7 +72,7 @@ final class EventTests: CoreDataTestCase { /// does is verify that we are internally consistent in our signature logic. func testSigningAndVerification() throws { // Arrange - let event = try createTestEvent(in: testContext) + let event = try EventFixture.build(in: testContext) // Act try event.sign(withKey: KeyFixture.keyPair) @@ -65,7 +83,7 @@ final class EventTests: CoreDataTestCase { func testVerificationOnBadId() throws { // Arrange - let event = try createTestEvent(in: testContext) + let event = try EventFixture.build(in: testContext) // Act try event.sign(withKey: KeyFixture.keyPair) @@ -77,7 +95,7 @@ final class EventTests: CoreDataTestCase { func testVerificationOnBadSignature() throws { // Arrange - let event = try createTestEvent(in: testContext) + let event = try EventFixture.build(in: testContext) event.identifier = try event.calculateIdentifier() // Act @@ -89,7 +107,7 @@ final class EventTests: CoreDataTestCase { } func testFetchEventByIDPerformance() throws { - let testEvent = try createTestEvent(in: testContext) + let testEvent = try EventFixture.build(in: testContext) testEvent.identifier = try testEvent.calculateIdentifier() let eventID = testEvent.identifier! try testContext.save() @@ -103,7 +121,7 @@ final class EventTests: CoreDataTestCase { // MARK: - Replies func testReferencedNoteGivenMentionMarker() throws { - let testEvent = try createTestEvent(in: testContext) + let testEvent = try EventFixture.build(in: testContext) let mention = try EventReference( jsonTag: ["e", "646daa2f5d2d990dc98fb50a6ce8de65d77419cee689d7153c912175e85ca95d", "", "mention"], @@ -115,7 +133,7 @@ final class EventTests: CoreDataTestCase { } func testRepostedNote() throws { - let testEvent = try createTestEvent(in: testContext) + let testEvent = try EventFixture.build(in: testContext) testEvent.kind = 6 let mention = try EventReference( @@ -131,7 +149,7 @@ final class EventTests: CoreDataTestCase { } func testRepostedNoteGivenNonRepost() throws { - let testEvent = try createTestEvent(in: testContext) + let testEvent = try EventFixture.build(in: testContext) testEvent.kind = 1 let mention = try EventReference( @@ -188,40 +206,4 @@ final class EventTests: CoreDataTestCase { // Assert XCTAssertEqual(events.count, 0) } - - // MARK: - Helpers - - private func createTestEvent( - in context: NSManagedObjectContext, - publicKey: RawAuthorID = KeyFixture.pubKeyHex - ) throws -> Event { - let event = Event(context: context) - event.createdAt = Date(timeIntervalSince1970: TimeInterval(1_675_264_762)) - event.content = "Testing nos #[0]" - event.kind = 1 - - let author = Author(context: context) - author.hexadecimalPublicKey = publicKey - event.author = author - - let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] - event.allTags = tags as NSObject - return event - } - - private func createTestEventWithNoTags( - in context: NSManagedObjectContext, - publicKey: RawAuthorID = KeyFixture.pubKeyHex - ) throws -> Event { - let event = Event(context: context) - event.createdAt = Date(timeIntervalSince1970: TimeInterval(1_675_264_762)) - event.content = "Testing nos #[0]" - event.kind = 1 - - let author = Author(context: context) - author.hexadecimalPublicKey = publicKey - event.author = author - - return event - } } diff --git a/NosTests/Service/DatabaseCleanerTests.swift b/NosTests/Service/DatabaseCleanerTests.swift index 3ad0ef532..b44482a81 100644 --- a/NosTests/Service/DatabaseCleanerTests.swift +++ b/NosTests/Service/DatabaseCleanerTests.swift @@ -1,7 +1,7 @@ import XCTest import CoreData -final class DatabaseCleanerTests: CoreDataTestCase { +@MainActor final class DatabaseCleanerTests: CoreDataTestCase { func test_emptyDatabase() async throws { try await DatabaseCleaner.cleanupEntities(before: Date.now, for: KeyFixture.alice.publicKeyHex, in: testContext) @@ -12,9 +12,9 @@ final class DatabaseCleanerTests: CoreDataTestCase { let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) - let oldEventOne = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) - let oldEventTwo = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 8)) - let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + let oldEventOne = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let oldEventTwo = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 8)) + let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) // Create a reference with two old events that should be deleted let eventReferenceToBeDeleted = EventReference(context: testContext) @@ -46,8 +46,8 @@ final class DatabaseCleanerTests: CoreDataTestCase { let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) - let oldEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) - let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) try testContext.save() @@ -69,8 +69,8 @@ final class DatabaseCleanerTests: CoreDataTestCase { _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) // Create an old event that is referenced by a newer event - let oldEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) - let newEvent = try createTestEvent(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) let eventReferenceToBeDeleted = EventReference(context: testContext) eventReferenceToBeDeleted.referencedEvent = oldEvent @@ -89,27 +89,4 @@ final class DatabaseCleanerTests: CoreDataTestCase { let events = try testContext.fetch(Event.allEventsRequest()) XCTAssertEqual(events.count, 2) } - - // MARK: - Helpers - - private func createTestEvent( - in context: NSManagedObjectContext, - publicKey: RawAuthorID = KeyFixture.pubKeyHex, - receivedAt: Date = .now - ) throws -> Event { - let event = Event(context: context) - event.identifier = UUID().uuidString - event.createdAt = Date.now - event.receivedAt = receivedAt - event.content = "Testing nos #[0]" - event.kind = 1 - - let author = Author(context: context) - author.hexadecimalPublicKey = publicKey - event.author = author - - let tags = [["p", "d0a1ffb8761b974cec4a3be8cbcb2e96a7090dcf465ffeac839aa4ca20c9a59e"]] - event.allTags = tags as NSObject - return event - } } diff --git a/NosTests/Test Helpers/EventFixture.swift b/NosTests/Test Helpers/EventFixture.swift index f5186320d..f4f4bf9fc 100644 --- a/NosTests/Test Helpers/EventFixture.swift +++ b/NosTests/Test Helpers/EventFixture.swift @@ -1,8 +1,32 @@ -// -// EventFixture.swift -// NosTests -// -// Created by Matthew Lorentz on 6/7/24. -// - import Foundation +import CoreData + +enum EventFixture { + + /// Builds an event with sensible defaults. Just pass the values you need to specify. + static func build( + in context: NSManagedObjectContext, + publicKey: RawAuthorID = KeyFixture.pubKeyHex, + content: String = UUID().uuidString, + createdAt: Date = Date(timeIntervalSince1970: TimeInterval(1_675_264_762)), + receivedAt: Date = .now, + tags: [[String]]? = [], + deletedOn: Set = [] + ) throws -> Event { + let event = Event(context: context) + event.createdAt = createdAt + event.receivedAt = receivedAt + event.content = content + event.kind = 1 + event.allTags = tags as? NSObject + + let author = try Author.findOrCreate(by: publicKey, context: context) + event.author = author + + event.identifier = try event.calculateIdentifier() + + event.deletedOn = deletedOn + + return event + } +} From f853d6feded827c0e0a3ef631f9e666722301bb6 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 10:52:51 -0400 Subject: [PATCH 03/14] Wrote some bonus tests --- .../CoreData/Follow+CoreDataClass.swift | 9 ++ NosTests/Model/AuthorTests.swift | 23 +++++ NosTests/Service/DatabaseCleanerTests.swift | 93 ++++++++++++++++--- 3 files changed, 111 insertions(+), 14 deletions(-) diff --git a/Nos/Models/CoreData/Follow+CoreDataClass.swift b/Nos/Models/CoreData/Follow+CoreDataClass.swift index ce763de98..0fef90701 100644 --- a/Nos/Models/CoreData/Follow+CoreDataClass.swift +++ b/Nos/Models/CoreData/Follow+CoreDataClass.swift @@ -108,6 +108,15 @@ public class Follow: NosManagedObject { fetchRequest.predicate = NSPredicate(format: "destination IN %@", authors) return fetchRequest } + + @nonobjc public class func allFollowsRequest() -> NSFetchRequest { + let fetchRequest = NSFetchRequest(entityName: "Follow") + fetchRequest.sortDescriptors = [ + NSSortDescriptor(keyPath: \Follow.source?.hexadecimalPublicKey, ascending: false), + NSSortDescriptor(keyPath: \Follow.destination?.hexadecimalPublicKey, ascending: false), + ] + return fetchRequest + } /// Retreives all the Follows whose source Author has been deleted. static func orphanedRequest() -> NSFetchRequest { diff --git a/NosTests/Model/AuthorTests.swift b/NosTests/Model/AuthorTests.swift index f6692d2ce..ff0724765 100644 --- a/NosTests/Model/AuthorTests.swift +++ b/NosTests/Model/AuthorTests.swift @@ -226,4 +226,27 @@ final class AuthorTests: CoreDataTestCase { // Assert XCTAssertEqual(events.count, 0) } + + func test_outOfNetwork_givenCircleOfFollows() throws { + // Arrange + let alice = try Author.findOrCreate(by: "alice", context: testContext) + let bob = try Author.findOrCreate(by: "bob", context: testContext) + let carl = try Author.findOrCreate(by: "carl", context: testContext) + let eve = try Author.findOrCreate(by: "eve", context: testContext) + + // Act + // Create a circle of follows alice -> bob -> carl -> eve -> alice + _ = try Follow.findOrCreate(source: alice, destination: bob, context: testContext) + _ = try Follow.findOrCreate(source: bob, destination: carl, context: testContext) + _ = try Follow.findOrCreate(source: carl, destination: eve, context: testContext) + _ = try Follow.findOrCreate(source: eve, destination: alice, context: testContext) + + try testContext.saveIfNeeded() + + // Act + let authors = try testContext.fetch(Author.outOfNetwork(for: alice)) + + // Assert + XCTAssertEqual(authors, [eve]) + } } diff --git a/NosTests/Service/DatabaseCleanerTests.swift b/NosTests/Service/DatabaseCleanerTests.swift index 92980d11d..822dfae03 100644 --- a/NosTests/Service/DatabaseCleanerTests.swift +++ b/NosTests/Service/DatabaseCleanerTests.swift @@ -1,7 +1,7 @@ import XCTest import CoreData -@MainActor final class DatabaseCleanerTests: CoreDataTestCase { +final class DatabaseCleanerTests: CoreDataTestCase { func test_emptyDatabase() async throws { // Act @@ -18,6 +18,8 @@ import CoreData } } + // MARK: - EventReferences + func test_cleanup_deletesCorrectEventReferences() async throws { // Arrange let deleteBeforeDate = Date(timeIntervalSince1970: 10) @@ -34,8 +36,8 @@ import CoreData // Create references with a new event that should not be deleted let eventReferenceToBeKept = EventReference(context: testContext) - eventReferenceToBeDeleted.referencingEvent = newEvent - eventReferenceToBeDeleted.referencedEvent = oldEventTwo + eventReferenceToBeKept.referencingEvent = newEvent + eventReferenceToBeKept.referencedEvent = oldEventTwo try testContext.save() @@ -53,13 +55,19 @@ import CoreData XCTAssertEqual(eventReferences.first?.referencedEvent, oldEventTwo) } - func test_cleanup_deletesOldEvents() async throws { + func test_cleanup_savesReferencedEvents() async throws { let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + + // Create an old event that is referenced by a newer event let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + let eventReferenceToBeDeleted = EventReference(context: testContext) + eventReferenceToBeDeleted.referencedEvent = oldEvent + eventReferenceToBeDeleted.referencingEvent = newEvent + try testContext.save() // Act @@ -71,22 +79,18 @@ import CoreData // Assert let events = try testContext.fetch(Event.allEventsRequest()) - XCTAssertEqual(events, [newEvent]) + XCTAssertEqual(events.count, 2) } - func test_cleanup_savesReferencedEvents() async throws { + // MARK: - Events + + func test_cleanup_deletesOldEvents() async throws { let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) - - // Create an old event that is referenced by a newer event - let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + _ = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) // old event let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) - let eventReferenceToBeDeleted = EventReference(context: testContext) - eventReferenceToBeDeleted.referencedEvent = oldEvent - eventReferenceToBeDeleted.referencingEvent = newEvent - try testContext.save() // Act @@ -98,6 +102,67 @@ import CoreData // Assert let events = try testContext.fetch(Event.allEventsRequest()) - XCTAssertEqual(events.count, 2) + XCTAssertEqual(events, [newEvent]) + } + + // MARK: - Authors + + func test_cleanup_keepsInNetworkAuthors() async throws { + // Arrange + let alice = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) + let bob = try Author.findOrCreate(by: "bob", context: testContext) + let carl = try Author.findOrCreate(by: "carl", context: testContext) + let eve = try Author.findOrCreate(by: "eve", context: testContext) + + // Act + // Create a circle of follows alice -> bob -> carl -> eve -> alice + _ = try Follow.findOrCreate(source: alice, destination: bob, context: testContext) + _ = try Follow.findOrCreate(source: bob, destination: carl, context: testContext) + _ = try Follow.findOrCreate(source: carl, destination: eve, context: testContext) + _ = try Follow.findOrCreate(source: eve, destination: alice, context: testContext) + + try testContext.saveIfNeeded() + + // Act + try await DatabaseCleaner.cleanupEntities( + before: Date.now, + for: KeyFixture.alice.publicKeyHex, + in: testContext + ) + + // Assert + let authors = try testContext.fetch(Author.allAuthorsRequest()) + // Eve is out of network and should be deleted. The others should be kept. + XCTAssertEqual(authors, [carl, bob, alice]) + } + + // MARK: - Follows + + func test_cleanup_keepsInNetworkFollows() async throws { + // Arrange + let alice = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) + let bob = try Author.findOrCreate(by: "bob", context: testContext) + let carl = try Author.findOrCreate(by: "carl", context: testContext) + let eve = try Author.findOrCreate(by: "eve", context: testContext) + + // Act + // Create a circle of follows alice -> bob -> carl -> eve -> alice + let followOne = try Follow.findOrCreate(source: alice, destination: bob, context: testContext) + let followTwo = try Follow.findOrCreate(source: bob, destination: carl, context: testContext) + let followThree = try Follow.findOrCreate(source: carl, destination: eve, context: testContext) + _ = try Follow.findOrCreate(source: eve, destination: alice, context: testContext) + + try testContext.saveIfNeeded() + + // Act + try await DatabaseCleaner.cleanupEntities( + before: Date.now, + for: KeyFixture.alice.publicKeyHex, + in: testContext + ) + + // Assert + let follows = try testContext.fetch(Follow.allFollowsRequest()) + XCTAssertEqual(follows, [followThree, followTwo, followOne]) } } From 5dd6bbc02beb040017829945e85bcb1cb7445504 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 11:31:05 -0400 Subject: [PATCH 04/14] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fc1564dd..1ed4a7db2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- Fixed a bug where infinite spinners would be shown on reposted notes. - Added support for opening njump.me content in Nos. - Fixed a crash on logout - Fixed a bug where some profiles wouldn't load old notes. From 1c8a9133c7d58b2bf110fc5a625fb3e13d347b48 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 15:46:25 -0400 Subject: [PATCH 05/14] Refactor some fetch requests out of DatabaseCleaner --- Nos/Models/CoreData/Event+CoreDataClass.swift | 31 +++++++++++++++ .../EventReference+CoreDataClass.swift | 12 ++++++ Nos/Service/DatabaseCleaner.swift | 39 ++----------------- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Nos/Models/CoreData/Event+CoreDataClass.swift b/Nos/Models/CoreData/Event+CoreDataClass.swift index dfa3e0ca7..131a95677 100644 --- a/Nos/Models/CoreData/Event+CoreDataClass.swift +++ b/Nos/Models/CoreData/Event+CoreDataClass.swift @@ -342,6 +342,37 @@ public class Event: NosManagedObject, VerifiableEvent { return fetchRequest } + /// A fetch requests for all the events that should be cleared out of the database by + /// `DatabaseCleaner.cleanupEntities(...)`. + /// + /// It will save the events for the given `user`, as well as other important events matching various other + /// criteria. + /// - Parameter before: The date before which events will be considered for cleanup. + /// - Parameter user: The Author record for the currently logged in user. Special treatment is given to their data. + @nonobjc public class func cleanupRequest(before date: Date, for user: Author) -> NSFetchRequest { + let oldStoryCutoff = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now + + let request = NSFetchRequest(entityName: "Event") + request.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] + let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" + let notOwnEventClause = "(author != %@)" + let readStoryClause = "(isRead = 1 AND receivedAt > %@)" + let userReportClause = "(kind == \(EventKind.report.rawValue) AND " + + "authorReferences.@count > 0 AND eventReferences.@count == 0)" + let clauses = "\(oldEventClause) AND" + + "\(notOwnEventClause) AND " + + "NOT \(readStoryClause) AND " + + "NOT \(userReportClause)" + request.predicate = NSPredicate( + format: clauses, + date as CVarArg, + user, + oldStoryCutoff as CVarArg + ) + + return request + } + @nonobjc public class func expiredRequest() -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "Event") fetchRequest.predicate = NSPredicate(format: "expirationDate <= %@", Date.now as CVarArg) diff --git a/Nos/Models/CoreData/EventReference+CoreDataClass.swift b/Nos/Models/CoreData/EventReference+CoreDataClass.swift index da4ce020c..0326841d8 100644 --- a/Nos/Models/CoreData/EventReference+CoreDataClass.swift +++ b/Nos/Models/CoreData/EventReference+CoreDataClass.swift @@ -33,6 +33,18 @@ public class EventReference: NosManagedObject { return fetchRequest } + /// A request for all the references whose associated `Events` were received before the given date. + static func all(before date: Date) -> NSFetchRequest { + let fetchRequest = NSFetchRequest(entityName: "EventReference") + fetchRequest.sortDescriptors = [NSSortDescriptor(keyPath: \EventReference.eventId, ascending: false)] + fetchRequest.predicate = NSPredicate( + format: "referencedEvent.receivedAt < %@ AND referencingEvent.receivedAt < %@", + date as CVarArg, + date as CVarArg + ) + return fetchRequest + } + /// Retreives all the EventReferences whose referencing Event has been deleted. static func orphanedRequest() -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "EventReference") diff --git a/Nos/Service/DatabaseCleaner.swift b/Nos/Service/DatabaseCleaner.swift index 3d25c0cf0..1e63bcf9c 100644 --- a/Nos/Service/DatabaseCleaner.swift +++ b/Nos/Service/DatabaseCleaner.swift @@ -5,8 +5,6 @@ import Dependencies enum DatabaseCleaner { - // swiftlint:disable function_body_length - /// Deletes unneeded entities from Core Data. /// /// This should only be called once right at app launch. @@ -31,45 +29,17 @@ enum DatabaseCleaner { try await context.perform { - guard let currentAuthor = try? Author.find(by: authorKey, context: context) else { + guard let currentUser = try? Author.find(by: authorKey, context: context) else { return } - let oldEventReferencesRequest = EventReference.all() - oldEventReferencesRequest.predicate = NSPredicate( - format: "referencedEvent.receivedAt < %@ AND referencingEvent.receivedAt < %@", - deleteBefore as CVarArg, - deleteBefore as CVarArg - ) - - let oldStoryCutoff = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now - - // Delete events older than `deleteBefore` - let oldEventsRequest = NSFetchRequest(entityName: "Event") - oldEventsRequest.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] - let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" - let notOwnEventClause = "(author.hexadecimalPublicKey != %@)" - let readStoryClause = "(isRead = 1 AND receivedAt > %@)" - let userReportClause = "(kind == \(EventKind.report.rawValue) AND " + - "authorReferences.@count > 0 AND eventReferences.@count == 0)" - let clauses = "\(oldEventClause) AND" + - "\(notOwnEventClause) AND " + - "NOT \(readStoryClause) AND " + - "NOT \(userReportClause)" - oldEventsRequest.predicate = NSPredicate( - format: clauses, - deleteBefore as CVarArg, - authorKey, - oldStoryCutoff as CVarArg - ) - let deleteRequests: [NSPersistentStoreRequest] = [ - oldEventReferencesRequest, - oldEventsRequest, + EventReference.all(before: deleteBefore), + Event.cleanupRequest(before: deleteBefore, for: currentUser), Event.expiredRequest(), EventReference.orphanedRequest(), AuthorReference.orphanedRequest(), - Author.outOfNetwork(for: currentAuthor), + Author.outOfNetwork(for: currentUser), Follow.orphanedRequest(), Relay.orphanedRequest(), NosNotification.oldNotificationsRequest(), @@ -99,5 +69,4 @@ enum DatabaseCleaner { let elapsedTime = Date.now.timeIntervalSince1970 - startTime.timeIntervalSince1970 Log.info("Finished Core Data cleanup in \(elapsedTime) seconds.") } - // swiftlint:enable function_body_length } From e6b9d74d629a74e8e17f249536bdedc88b524eef Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 15:51:40 -0400 Subject: [PATCH 06/14] Fix build warning --- NosTests/Service/FileStorage/NostrBuildAPIClientTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NosTests/Service/FileStorage/NostrBuildAPIClientTests.swift b/NosTests/Service/FileStorage/NostrBuildAPIClientTests.swift index 0ff316ad7..fdbd25ae2 100644 --- a/NosTests/Service/FileStorage/NostrBuildAPIClientTests.swift +++ b/NosTests/Service/FileStorage/NostrBuildAPIClientTests.swift @@ -107,7 +107,7 @@ class NostrBuildAPIClientTests: XCTestCase { let expectedAuthorization = "Nostr \(requestData.base64EncodedString())" // Act - let (uploadRequest, data) = try subject.uploadRequest(fileAt: fileURL) + let (uploadRequest, _) = try subject.uploadRequest(fileAt: fileURL) // Assert XCTAssertEqual(uploadRequest.value(forHTTPHeaderField: "Authorization"), expectedAuthorization) From 92724453e85e4ee0b5adbc34c1e668ba13a0e6cd Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 17:29:48 -0400 Subject: [PATCH 07/14] Update Nos/Models/CoreData/Event+CoreDataClass.swift Co-authored-by: Martin Dutra --- Nos/Models/CoreData/Event+CoreDataClass.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nos/Models/CoreData/Event+CoreDataClass.swift b/Nos/Models/CoreData/Event+CoreDataClass.swift index 131a95677..ef68e7b62 100644 --- a/Nos/Models/CoreData/Event+CoreDataClass.swift +++ b/Nos/Models/CoreData/Event+CoreDataClass.swift @@ -342,7 +342,7 @@ public class Event: NosManagedObject, VerifiableEvent { return fetchRequest } - /// A fetch requests for all the events that should be cleared out of the database by + /// A fetch request for all the events that should be cleared out of the database by /// `DatabaseCleaner.cleanupEntities(...)`. /// /// It will save the events for the given `user`, as well as other important events matching various other From ba92754dff07099887e089c162e3b0e46c3185fd Mon Sep 17 00:00:00 2001 From: mplorentz Date: Mon, 10 Jun 2024 17:33:36 -0400 Subject: [PATCH 08/14] Simplify date arithmetic --- Nos/Controller/PersistenceController.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Nos/Controller/PersistenceController.swift b/Nos/Controller/PersistenceController.swift index 514f9e594..b1d50c358 100644 --- a/Nos/Controller/PersistenceController.swift +++ b/Nos/Controller/PersistenceController.swift @@ -199,7 +199,7 @@ class PersistenceController { let context = newBackgroundContext() do { // We don't want to delete any events downloaded after app boot, so we subtract 60 seconds - let cleanupBeforeDate = Date.now.addingTimeInterval(-60) // Subtract 60 seconds + let cleanupBeforeDate = Date.now - 60 try await DatabaseCleaner.cleanupEntities(before: cleanupBeforeDate, for: authorKey, in: context) } catch { Log.optional(error) From ced69618ef696c1a60b50ebbdc18c15a2142333d Mon Sep 17 00:00:00 2001 From: mplorentz Date: Wed, 12 Jun 2024 12:55:18 -0400 Subject: [PATCH 09/14] Add back logic to keep 1000 events on disk. --- Nos/Controller/PersistenceController.swift | 4 +- Nos/Service/DatabaseCleaner.swift | 33 +++++++++++++- NosTests/Service/DatabaseCleanerTests.swift | 48 +++++++++++++++------ 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/Nos/Controller/PersistenceController.swift b/Nos/Controller/PersistenceController.swift index b1d50c358..4897bd1f8 100644 --- a/Nos/Controller/PersistenceController.swift +++ b/Nos/Controller/PersistenceController.swift @@ -198,9 +198,7 @@ class PersistenceController { let context = newBackgroundContext() do { - // We don't want to delete any events downloaded after app boot, so we subtract 60 seconds - let cleanupBeforeDate = Date.now - 60 - try await DatabaseCleaner.cleanupEntities(before: cleanupBeforeDate, for: authorKey, in: context) + try await DatabaseCleaner.cleanupEntities(for: authorKey, in: context) } catch { Log.optional(error) crashReporting.report("Error in database cleanup: \(error.localizedDescription)") diff --git a/Nos/Service/DatabaseCleaner.swift b/Nos/Service/DatabaseCleaner.swift index 1e63bcf9c..2784976ce 100644 --- a/Nos/Service/DatabaseCleaner.swift +++ b/Nos/Service/DatabaseCleaner.swift @@ -15,9 +15,9 @@ enum DatabaseCleaner { /// - delete any other models that are orphaned by the previous deletions /// - fix EventReferences whose referencedEvent was deleted by createing a stubbed Event static func cleanupEntities( - before deleteBefore: Date, for authorKey: RawAuthorID, - in context: NSManagedObjectContext + in context: NSManagedObjectContext, + keeping eventsToKeep: Int = 1000 ) async throws { // this function was written in a hurry and probably should be refactored and tested thorougly. @Dependency(\.analytics) var analytics @@ -33,6 +33,8 @@ enum DatabaseCleaner { return } + let deleteBefore = try computeDeleteBeforeDate(keeping: eventsToKeep, context: context) + let deleteRequests: [NSPersistentStoreRequest] = [ EventReference.all(before: deleteBefore), Event.cleanupRequest(before: deleteBefore, for: currentUser), @@ -69,4 +71,31 @@ enum DatabaseCleaner { let elapsedTime = Date.now.timeIntervalSince1970 - startTime.timeIntervalSince1970 Log.info("Finished Core Data cleanup in \(elapsedTime) seconds.") } + + /// Takes the number of events we want to keep in our database and computes a date after which we can safely delete + /// events. We use a date because you can't tell Core Data to just delete events after a certain index. Also the + /// date is used for other fetch requests, i.e. to avoid deleting older events that are referenced by newer events. + /// + /// This must be called inside a `NSManagedObjectContext.perform` block. + private static func computeDeleteBeforeDate( + keeping eventsToKeep: Int, + context: NSManagedObjectContext + ) throws -> Date { + // Delete all but the most recent n events + var deleteBefore = Date.now + guard eventsToKeep > 0 else { + return deleteBefore + } + let fetchFirstEventToDelete = Event.allEventsRequest() + fetchFirstEventToDelete.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: false)] + fetchFirstEventToDelete.fetchLimit = 1 + fetchFirstEventToDelete.fetchOffset = eventsToKeep - 1 + fetchFirstEventToDelete.predicate = NSPredicate(format: "receivedAt != nil") + if let firstEventToDelete = try context.fetch(fetchFirstEventToDelete).first, + let receivedAt = firstEventToDelete.receivedAt { + deleteBefore = receivedAt + } + + return deleteBefore + } } diff --git a/NosTests/Service/DatabaseCleanerTests.swift b/NosTests/Service/DatabaseCleanerTests.swift index 822dfae03..b8c44dc79 100644 --- a/NosTests/Service/DatabaseCleanerTests.swift +++ b/NosTests/Service/DatabaseCleanerTests.swift @@ -5,7 +5,7 @@ final class DatabaseCleanerTests: CoreDataTestCase { func test_emptyDatabase() async throws { // Act - try await DatabaseCleaner.cleanupEntities(before: Date.now, for: KeyFixture.alice.publicKeyHex, in: testContext) + try await DatabaseCleaner.cleanupEntities(for: KeyFixture.alice.publicKeyHex, in: testContext) // Assert that the database is still empty let managedObjectModel = try XCTUnwrap(testContext.persistentStoreCoordinator?.managedObjectModel) @@ -22,7 +22,6 @@ final class DatabaseCleanerTests: CoreDataTestCase { func test_cleanup_deletesCorrectEventReferences() async throws { // Arrange - let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) let oldEventOne = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) @@ -43,9 +42,9 @@ final class DatabaseCleanerTests: CoreDataTestCase { // Act try await DatabaseCleaner.cleanupEntities( - before: deleteBeforeDate, for: KeyFixture.alice.publicKeyHex, - in: testContext + in: testContext, + keeping: 1 ) // Assert @@ -56,7 +55,6 @@ final class DatabaseCleanerTests: CoreDataTestCase { } func test_cleanup_savesReferencedEvents() async throws { - let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) @@ -72,9 +70,9 @@ final class DatabaseCleanerTests: CoreDataTestCase { // Act try await DatabaseCleaner.cleanupEntities( - before: deleteBeforeDate, for: KeyFixture.alice.publicKeyHex, - in: testContext + in: testContext, + keeping: 1 ) // Assert @@ -84,8 +82,36 @@ final class DatabaseCleanerTests: CoreDataTestCase { // MARK: - Events + func test_cleanup_keepsNEvents() async throws { + let user = KeyFixture.alice + _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + var events = [Event]() + for i in 0..<10 { + let date = Date(timeIntervalSince1970: TimeInterval(i)) + events.append( + try EventFixture.build( + in: testContext, + createdAt: date, + receivedAt: date + ) + ) + } + + try testContext.save() + + // Act + try await DatabaseCleaner.cleanupEntities( + for: KeyFixture.alice.publicKeyHex, + in: testContext, + keeping: 5 + ) + + // Assert + let fetchedEvents = try testContext.fetch(Event.allEventsRequest()) + XCTAssertEqual(fetchedEvents.map { $0.identifier }, events.suffix(5).map { $0.identifier }) + } + func test_cleanup_deletesOldEvents() async throws { - let deleteBeforeDate = Date(timeIntervalSince1970: 10) let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) _ = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) // old event @@ -95,9 +121,9 @@ final class DatabaseCleanerTests: CoreDataTestCase { // Act try await DatabaseCleaner.cleanupEntities( - before: deleteBeforeDate, for: KeyFixture.alice.publicKeyHex, - in: testContext + in: testContext, + keeping: 1 ) // Assert @@ -125,7 +151,6 @@ final class DatabaseCleanerTests: CoreDataTestCase { // Act try await DatabaseCleaner.cleanupEntities( - before: Date.now, for: KeyFixture.alice.publicKeyHex, in: testContext ) @@ -156,7 +181,6 @@ final class DatabaseCleanerTests: CoreDataTestCase { // Act try await DatabaseCleaner.cleanupEntities( - before: Date.now, for: KeyFixture.alice.publicKeyHex, in: testContext ) From ae9c33dc6bb17ff2e996023263d5c0bed90edadd Mon Sep 17 00:00:00 2001 From: mplorentz Date: Thu, 20 Jun 2024 11:36:55 -0400 Subject: [PATCH 10/14] Fixed issue where chains of event references could keep too many events around --- Nos.xcodeproj/project.pbxproj | 70 ++++++------ Nos/Models/CoreData/Event+CoreDataClass.swift | 100 ++++++++++++++++- .../EventReference+CoreDataClass.swift | 22 +++- Nos/Service/DatabaseCleaner.swift | 103 ++++++++++++++---- NosTests/Service/DatabaseCleanerTests.swift | 69 +++++++++--- 5 files changed, 286 insertions(+), 78 deletions(-) diff --git a/Nos.xcodeproj/project.pbxproj b/Nos.xcodeproj/project.pbxproj index 932813311..37cf2356b 100644 --- a/Nos.xcodeproj/project.pbxproj +++ b/Nos.xcodeproj/project.pbxproj @@ -1688,19 +1688,19 @@ ); mainGroup = C9DEBFC5298941000078B43A; packageReferences = ( - C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream.git" */, + C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream" */, C94D855D29914D2300749478 /* XCRemoteSwiftPackageReference "swiftui-navigation" */, - C9ADB139299299570075E7F8 /* XCRemoteSwiftPackageReference "bech32.git" */, + C9ADB139299299570075E7F8 /* XCRemoteSwiftPackageReference "bech32" */, C9646E9829B79E04007239A4 /* XCRemoteSwiftPackageReference "logger-ios" */, - C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios.git" */, + C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios" */, C9646EA529B7A3DD007239A4 /* XCRemoteSwiftPackageReference "swift-dependencies" */, - C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections.git" */, - C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa.git" */, + C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections" */, + C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa" */, C99DBF7C2A9E81CF00F7068F /* XCRemoteSwiftPackageReference "SDWebImageSwiftUI" */, C9B737702AB24D5F00398BE7 /* XCRemoteSwiftPackageReference "SwiftGenPlugin" */, C91565BF2B2368FA0068EECA /* XCRemoteSwiftPackageReference "ViewInspector" */, - 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin.git" */, - C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1.swift" */, + 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin" */, + C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1" */, C9FD35112BCED5A6008F8D95 /* XCRemoteSwiftPackageReference "nostr-sdk-ios" */, ); productRefGroup = C9DEBFCF298941000078B43A /* Products */; @@ -2225,11 +2225,11 @@ /* Begin PBXTargetDependency section */ 3AD3185D2B294E9000026B07 /* PBXTargetDependency */ = { isa = PBXTargetDependency; - productRef = 3AD3185C2B294E9000026B07 /* plugin:XCStringsToolPlugin */; + productRef = 3AD3185C2B294E9000026B07 /* XCStringsToolPlugin */; }; 3AEABEF32B2BF806001BC933 /* PBXTargetDependency */ = { isa = PBXTargetDependency; - productRef = 3AEABEF22B2BF806001BC933 /* plugin:XCStringsToolPlugin */; + productRef = 3AEABEF22B2BF806001BC933 /* XCStringsToolPlugin */; }; C90862C229E9804B00C35A71 /* PBXTargetDependency */ = { isa = PBXTargetDependency; @@ -2238,11 +2238,11 @@ }; C9A6C7442AD83F7A001F9500 /* PBXTargetDependency */ = { isa = PBXTargetDependency; - productRef = C9A6C7432AD83F7A001F9500 /* plugin:SwiftGenPlugin */; + productRef = C9A6C7432AD83F7A001F9500 /* SwiftGenPlugin */; }; C9D573402AB24A3700E06BB4 /* PBXTargetDependency */ = { isa = PBXTargetDependency; - productRef = C9D5733F2AB24A3700E06BB4 /* plugin:SwiftGenPlugin */; + productRef = C9D5733F2AB24A3700E06BB4 /* SwiftGenPlugin */; }; C9DEBFE6298941020078B43A /* PBXTargetDependency */ = { isa = PBXTargetDependency; @@ -2965,7 +2965,7 @@ /* End XCConfigurationList section */ /* Begin XCRemoteSwiftPackageReference section */ - 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin.git" */ = { + 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/liamnichols/xcstrings-tool-plugin.git"; requirement = { @@ -2997,7 +2997,7 @@ minimumVersion = 1.1.0; }; }; - C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios.git" */ = { + C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/PostHog/posthog-ios.git"; requirement = { @@ -3013,7 +3013,7 @@ minimumVersion = 0.1.4; }; }; - C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections.git" */ = { + C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/apple/swift-collections.git"; requirement = { @@ -3029,7 +3029,7 @@ minimumVersion = 2.0.0; }; }; - C9ADB139299299570075E7F8 /* XCRemoteSwiftPackageReference "bech32.git" */ = { + C9ADB139299299570075E7F8 /* XCRemoteSwiftPackageReference "bech32" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/0xdeadp00l/bech32.git"; requirement = { @@ -3037,7 +3037,7 @@ kind = branch; }; }; - C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa.git" */ = { + C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/getsentry/sentry-cocoa.git"; requirement = { @@ -3061,7 +3061,7 @@ minimumVersion = 6.6.2; }; }; - C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream.git" */ = { + C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/daltoniam/Starscream.git"; requirement = { @@ -3069,7 +3069,7 @@ minimumVersion = 4.0.0; }; }; - C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1.swift" */ = { + C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/GigaBitcoin/secp256k1.swift"; requirement = { @@ -3088,19 +3088,19 @@ /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ - 3AD3185C2B294E9000026B07 /* plugin:XCStringsToolPlugin */ = { + 3AD3185C2B294E9000026B07 /* XCStringsToolPlugin */ = { isa = XCSwiftPackageProductDependency; - package = 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin.git" */; + package = 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin" */; productName = "plugin:XCStringsToolPlugin"; }; - 3AEABEF22B2BF806001BC933 /* plugin:XCStringsToolPlugin */ = { + 3AEABEF22B2BF806001BC933 /* XCStringsToolPlugin */ = { isa = XCSwiftPackageProductDependency; - package = 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin.git" */; + package = 3AD3185B2B294E6200026B07 /* XCRemoteSwiftPackageReference "xcstrings-tool-plugin" */; productName = "plugin:XCStringsToolPlugin"; }; C905B0742A619367009B8A78 /* DequeModule */ = { isa = XCSwiftPackageProductDependency; - package = C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections.git" */; + package = C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections" */; productName = DequeModule; }; C91565C02B2368FA0068EECA /* ViewInspector */ = { @@ -3129,7 +3129,7 @@ }; C9646EA329B7A24A007239A4 /* PostHog */ = { isa = XCSwiftPackageProductDependency; - package = C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios.git" */; + package = C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios" */; productName = PostHog; }; C9646EA629B7A3DD007239A4 /* Dependencies */ = { @@ -3139,7 +3139,7 @@ }; C9646EA829B7A4F2007239A4 /* PostHog */ = { isa = XCSwiftPackageProductDependency; - package = C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios.git" */; + package = C9646EA229B7A24A007239A4 /* XCRemoteSwiftPackageReference "posthog-ios" */; productName = PostHog; }; C9646EAB29B7A520007239A4 /* Dependencies */ = { @@ -3149,7 +3149,7 @@ }; C96CB98B2A6040C500498C4E /* DequeModule */ = { isa = XCSwiftPackageProductDependency; - package = C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections.git" */; + package = C96CB98A2A6040C500498C4E /* XCRemoteSwiftPackageReference "swift-collections" */; productName = DequeModule; }; C99DBF7D2A9E81CF00F7068F /* SDWebImageSwiftUI */ = { @@ -3167,44 +3167,44 @@ package = C99DBF7C2A9E81CF00F7068F /* XCRemoteSwiftPackageReference "SDWebImageSwiftUI" */; productName = SDWebImageSwiftUI; }; - C9A6C7432AD83F7A001F9500 /* plugin:SwiftGenPlugin */ = { + C9A6C7432AD83F7A001F9500 /* SwiftGenPlugin */ = { isa = XCSwiftPackageProductDependency; package = C9B737702AB24D5F00398BE7 /* XCRemoteSwiftPackageReference "SwiftGenPlugin" */; productName = "plugin:SwiftGenPlugin"; }; C9B71DBD2A8E9BAD0031ED9F /* Sentry */ = { isa = XCSwiftPackageProductDependency; - package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa.git" */; + package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa" */; productName = Sentry; }; C9B71DBF2A8E9BAD0031ED9F /* SentrySwiftUI */ = { isa = XCSwiftPackageProductDependency; - package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa.git" */; + package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa" */; productName = SentrySwiftUI; }; C9B71DC42A9008300031ED9F /* Sentry */ = { isa = XCSwiftPackageProductDependency; - package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa.git" */; + package = C9B71DBC2A8E9BAD0031ED9F /* XCRemoteSwiftPackageReference "sentry-cocoa" */; productName = Sentry; }; - C9D5733F2AB24A3700E06BB4 /* plugin:SwiftGenPlugin */ = { + C9D5733F2AB24A3700E06BB4 /* SwiftGenPlugin */ = { isa = XCSwiftPackageProductDependency; package = C9C8450C2AB249DB00654BC1 /* XCRemoteSwiftPackageReference "SwiftGenPlugin" */; productName = "plugin:SwiftGenPlugin"; }; C9DEC067298965270078B43A /* Starscream */ = { isa = XCSwiftPackageProductDependency; - package = C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream.git" */; + package = C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream" */; productName = Starscream; }; C9FD34F52BCEC89C008F8D95 /* secp256k1 */ = { isa = XCSwiftPackageProductDependency; - package = C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1.swift" */; + package = C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1" */; productName = secp256k1; }; C9FD34F72BCEC8B5008F8D95 /* secp256k1 */ = { isa = XCSwiftPackageProductDependency; - package = C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1.swift" */; + package = C9FD34F42BCEC89C008F8D95 /* XCRemoteSwiftPackageReference "secp256k1" */; productName = secp256k1; }; C9FD35122BCED5A6008F8D95 /* NostrSDK */ = { @@ -3214,7 +3214,7 @@ }; CDDA1F7A29A527650047ACD8 /* Starscream */ = { isa = XCSwiftPackageProductDependency; - package = C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream.git" */; + package = C9DEC066298965270078B43A /* XCRemoteSwiftPackageReference "Starscream" */; productName = Starscream; }; CDDA1F7C29A527650047ACD8 /* SwiftUINavigation */ = { diff --git a/Nos/Models/CoreData/Event+CoreDataClass.swift b/Nos/Models/CoreData/Event+CoreDataClass.swift index 9fed89701..07ebb25e8 100644 --- a/Nos/Models/CoreData/Event+CoreDataClass.swift +++ b/Nos/Models/CoreData/Event+CoreDataClass.swift @@ -66,7 +66,10 @@ public class Event: NosManagedObject, VerifiableEvent { @nonobjc public class func allEventsRequest() -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "Event") - fetchRequest.sortDescriptors = [NSSortDescriptor(keyPath: \Event.createdAt, ascending: true)] + fetchRequest.sortDescriptors = [ + NSSortDescriptor(keyPath: \Event.createdAt, ascending: true), + NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true) + ] return fetchRequest } @@ -354,7 +357,7 @@ public class Event: NosManagedObject, VerifiableEvent { let request = NSFetchRequest(entityName: "Event") request.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] - let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" + let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" let notOwnEventClause = "(author != %@)" let readStoryClause = "(isRead = 1 AND receivedAt > %@)" let userReportClause = "(kind == \(EventKind.report.rawValue) AND " + @@ -373,6 +376,81 @@ public class Event: NosManagedObject, VerifiableEvent { return request } + /// This constructs a predicate for events that should be protected from deletion when we are purging the database. + /// - Parameter before: The date before which events will be considered for cleanup. + /// - Parameter user: The Author record for the currently logged in user. Special treatment is given to their data. + @nonobjc public class func protectedFromCleanupPredicate(for user: Author) -> NSPredicate { + // NOTE: This code is pretty much the same as that of `protectedFromCleanupSubqueryPredicate` but I can't + // figure out how to share it. If you change any of this you should probably make equivalent changes there. + guard let userKey = user.hexadecimalPublicKey else { + return NSPredicate.false + } + + // protect all events authored by the current user + let userEventsPredicate = NSPredicate(format: "author.hexadecimalPublicKey = '\(userKey)'") + + // protect stories that were read recently, so we don't redownload and show them as unread again + let oldStoryCutoffDate = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now + let recentlyReadStoriesPredicate = NSPredicate( + format: "(isRead = 1 AND receivedAt > %@)", // TODO: stubbed events? + oldStoryCutoffDate as CVarArg + ) + + // keep author reports from people we follow + let userReportPredicate = NSPredicate( + format: "(kind == \(EventKind.report.rawValue) AND " + + "SUBQUERY(authorReferences, $references, TRUEPREDICATE).@count > 0 AND " + + "SUBQUERY(eventReferences, $references, TRUEPREDICATE).@count == 0 AND " + + "ANY author.followers.source.hexadecimalPublicKey == %@)", + userKey + ) + + return NSCompoundPredicate( + orPredicateWithSubpredicates: [ + userEventsPredicate, + recentlyReadStoriesPredicate, + userReportPredicate + ] + ) + } + + /// Same as `protectedFromCleanupPredicate(for:)` but constructs a predicate designed to be used in the SUBQUERY + /// of another predicate. + @nonobjc public class func protectedFromCleanupSubqueryPredicate(for user: Author) -> NSPredicate { + // NOTE: This code is pretty much the same as that of `protectedFromCleanupPredicate` but I can't figure out + // how to share it. If you change any of this you should probably make equivalent changes there. + guard let userKey = user.hexadecimalPublicKey else { + return NSPredicate.false + } + + // protect all events authored by the current user + let userEventsPredicate = NSPredicate(format: "$event.author.hexadecimalPublicKey = '\(userKey)'") + + // protect stories that were read recently, so we don't redownload and show them as unread again + let oldStoryCutoffDate = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now + let recentlyReadStoriesPredicate = NSPredicate( + format: "($event.isRead = 1 AND $event.receivedAt > %@)", // TODO: stubbed events? + oldStoryCutoffDate as CVarArg + ) + + // keep author reports from people we follow + let userReportPredicate = NSPredicate( + format: "($event.kind == \(EventKind.report.rawValue) AND " + + "SUBQUERY($event.authorReferences, $references, TRUEPREDICATE).@count > 0 AND " + + "SUBQUERY($event.eventReferences, $references, TRUEPREDICATE).@count == 0 AND " + + "ANY $event.author.followers.source.hexadecimalPublicKey == %@)", + userKey + ) + + return NSCompoundPredicate( + orPredicateWithSubpredicates: [ + userEventsPredicate, + recentlyReadStoriesPredicate, + userReportPredicate + ] + ) + } + @nonobjc public class func expiredRequest() -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "Event") fetchRequest.predicate = NSPredicate(format: "expirationDate <= %@", Date.now as CVarArg) @@ -1198,6 +1276,24 @@ public class Event: NosManagedObject, VerifiableEvent { return "https://njump.me" } } + + /// Converts an event back to a stubbed event by deleting all data except the `identifier`. + func stub() { + allTags = nil + content = nil + createdAt = nil + isVerified = false + receivedAt = nil + sendAttempts = 0 + signature = nil + author = nil + authorReferences = NSOrderedSet() + deletedOn = Set() + eventReferences = NSOrderedSet() + publishedTo = Set() + seenOnRelays = Set() + shouldBePublishedTo = Set() + } } // swiftlint:enable type_body_length // swiftlint:enable file_length diff --git a/Nos/Models/CoreData/EventReference+CoreDataClass.swift b/Nos/Models/CoreData/EventReference+CoreDataClass.swift index 0326841d8..33b644abc 100644 --- a/Nos/Models/CoreData/EventReference+CoreDataClass.swift +++ b/Nos/Models/CoreData/EventReference+CoreDataClass.swift @@ -33,15 +33,31 @@ public class EventReference: NosManagedObject { return fetchRequest } - /// A request for all the references whose associated `Events` were received before the given date. - static func all(before date: Date) -> NSFetchRequest { + /// This fetches all the references that can be deleted during the `DatabaseCleaner` routine. It takes care + /// to only select references before a given date that are not referenced by events we are keeping, and it also + /// accounts for "protected events" from `Event.protectedFromCleanupSubqueryPredicate()` to make sure we keep + /// events published by the current user etc. + static func cleanupRequest(before date: Date, user: Author) -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "EventReference") fetchRequest.sortDescriptors = [NSSortDescriptor(keyPath: \EventReference.eventId, ascending: false)] - fetchRequest.predicate = NSPredicate( + let protectedEventsPredicate = Event.protectedFromCleanupSubqueryPredicate(for: user) + let referencedEventIsNotProtected = NSPredicate( + format: "SUBQUERY(referencedEvent, $event, \(protectedEventsPredicate.predicateFormat)).@count == 0" + ) + let referencingEventIsNotProtected = NSPredicate( + format: "SUBQUERY(referencingEvent, $event, \(protectedEventsPredicate.predicateFormat)).@count == 0" + ) + let eventsAreOld = NSPredicate( format: "referencedEvent.receivedAt < %@ AND referencingEvent.receivedAt < %@", date as CVarArg, date as CVarArg ) + fetchRequest.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [ + referencedEventIsNotProtected, + referencingEventIsNotProtected, + eventsAreOld + ]) + return fetchRequest } diff --git a/Nos/Service/DatabaseCleaner.swift b/Nos/Service/DatabaseCleaner.swift index 2784976ce..97d3b2fbd 100644 --- a/Nos/Service/DatabaseCleaner.swift +++ b/Nos/Service/DatabaseCleaner.swift @@ -33,33 +33,40 @@ enum DatabaseCleaner { return } + // This is a delicate dance to get rid of events without breaking the consistency of the object graph. + // The app expects that certain post-processing has been done during parsing i.e. every "e" tag should + // have an EventReference and the EventReference should have at least a stubbed Event. So we can't just + // delete events before a certain date or we would leave dangling references around. + // + // The generic strategy is to pick a date and delete stuff received before then. However there are complex + // exceptions to i.e. keep events the current user has published. Most of these are defined in + // `Event.protectedFromCleanupPredicate(for: user)` which is used in several fetch requests. let deleteBefore = try computeDeleteBeforeDate(keeping: eventsToKeep, context: context) - let deleteRequests: [NSPersistentStoreRequest] = [ - EventReference.all(before: deleteBefore), - Event.cleanupRequest(before: deleteBefore, for: currentUser), - Event.expiredRequest(), - EventReference.orphanedRequest(), - AuthorReference.orphanedRequest(), - Author.outOfNetwork(for: currentUser), - Follow.orphanedRequest(), - Relay.orphanedRequest(), - NosNotification.oldNotificationsRequest(), - ] + // Get rid of all event references where 1) neither event is protected and 2) both events are old + try batchDelete( + objectsMatching: [EventReference.cleanupRequest(before: deleteBefore, user: currentUser)], + in: context + ) - for request in deleteRequests { - guard let fetchRequest = request as? NSFetchRequest else { - Log.error("Bad fetch request: \(request)") - continue - } - - let batchDeleteRequest = NSBatchDeleteRequest(fetchRequest: fetchRequest) - batchDeleteRequest.resultType = .resultTypeCount - let deleteResult = try context.execute(batchDeleteRequest) as? NSBatchDeleteResult - if let entityName = fetchRequest.entityName { - Log.info("Deleted \(deleteResult?.result ?? 0) of type \(entityName)") - } - } + // stub all events that aren't in a protected class before deleteBefore but are still referenced by events + // we are keeping + try stubReferencedOldEvents(before: deleteBefore, user: currentUser, in: context) + + try batchDelete( + objectsMatching: [ + // delete all events before deleteBefore that aren't protected or referenced + Event.cleanupRequest(before: deleteBefore, for: currentUser), + Event.expiredRequest(), + EventReference.orphanedRequest(), + AuthorReference.orphanedRequest(), + Author.outOfNetwork(for: currentUser), + Follow.orphanedRequest(), + Relay.orphanedRequest(), + NosNotification.oldNotificationsRequest(), + ], + in: context + ) try context.saveIfNeeded() } @@ -72,6 +79,54 @@ enum DatabaseCleaner { Log.info("Finished Core Data cleanup in \(elapsedTime) seconds.") } + /// This converts old hydrated events back to stubs. We do this because EventReferences can form long chains + /// of events that we can't delete. By stubbing an event we can delete its eventReferences and also the + /// referencedEvents. + private static func stubReferencedOldEvents( + before deleteBefore: Date, + user: Author, + in context: NSManagedObjectContext + ) throws { + let request = NSFetchRequest(entityName: "Event") + request.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] + let oldEventPredicate = NSPredicate(format: "(receivedAt < %@)", deleteBefore as CVarArg) + let referencedEventsPredicate = NSPredicate(format: "referencingEvents.@count > 0") + let nonProtectedEventsPredicate = NSCompoundPredicate( + notPredicateWithSubpredicate: Event.protectedFromCleanupPredicate(for: user) + ) + request.predicate = NSCompoundPredicate(andPredicateWithSubpredicates: [ + oldEventPredicate, + referencedEventsPredicate, + nonProtectedEventsPredicate + ]) + + let events = try context.fetch(request) + Log.info("Stubbing \(events.count) old Events that are still referenced by newer events") + for event in events { + event.stub() + } + } + + /// Performs a batch delete request using the given `fetchRequests` with nice logging. + private static func batchDelete( + objectsMatching fetchRequests: [NSPersistentStoreRequest], + in context: NSManagedObjectContext + ) throws { + for request in fetchRequests { + guard let fetchRequest = request as? NSFetchRequest else { + Log.error("Bad fetch request: \(request)") + continue + } + + let batchDeleteRequest = NSBatchDeleteRequest(fetchRequest: fetchRequest) + batchDeleteRequest.resultType = .resultTypeCount + let deleteResult = try context.execute(batchDeleteRequest) as? NSBatchDeleteResult + if let entityName = fetchRequest.entityName { + Log.info("Deleted \(deleteResult?.result ?? 0) of type \(entityName)") + } + } + } + /// Takes the number of events we want to keep in our database and computes a date after which we can safely delete /// events. We use a date because you can't tell Core Data to just delete events after a certain index. Also the /// date is used for other fetch requests, i.e. to avoid deleting older events that are referenced by newer events. diff --git a/NosTests/Service/DatabaseCleanerTests.swift b/NosTests/Service/DatabaseCleanerTests.swift index b8c44dc79..d0be44a06 100644 --- a/NosTests/Service/DatabaseCleanerTests.swift +++ b/NosTests/Service/DatabaseCleanerTests.swift @@ -3,7 +3,7 @@ import CoreData final class DatabaseCleanerTests: CoreDataTestCase { - func test_emptyDatabase() async throws { + @MainActor func test_emptyDatabase() async throws { // Act try await DatabaseCleaner.cleanupEntities(for: KeyFixture.alice.publicKeyHex, in: testContext) @@ -20,10 +20,10 @@ final class DatabaseCleanerTests: CoreDataTestCase { // MARK: - EventReferences - func test_cleanup_deletesCorrectEventReferences() async throws { + @MainActor func test_cleanup_deletesCorrectEventReferences() async throws { // Arrange - let user = KeyFixture.alice - _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + // Create the signed in user or the DatabaseCleaner will exit early. + _ = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) let oldEventOne = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) let oldEventTwo = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 8)) let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) @@ -54,10 +54,10 @@ final class DatabaseCleanerTests: CoreDataTestCase { XCTAssertEqual(eventReferences.first?.referencedEvent, oldEventTwo) } - func test_cleanup_savesReferencedEvents() async throws { - let user = KeyFixture.alice - _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) - + @MainActor func test_cleanup_savesReferencedEvents() async throws { + // Arrange + // Create the signed in user or the DatabaseCleaner will exit early. + _ = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) // Create an old event that is referenced by a newer event let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) @@ -80,11 +80,50 @@ final class DatabaseCleanerTests: CoreDataTestCase { XCTAssertEqual(events.count, 2) } + @MainActor func test_cleanup_givenEventReferenceChain_thenOldEventsStubbed() async throws { + // Arrange + // Create the signed in user or the DatabaseCleaner will exit early. + _ = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) + // Create a chain of events that reference one another. Two of them are old enough to be deleted. + try testContext.save() + let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) + let oldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) + let reallyOldEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 0)) + try testContext.save() + + let eventReferenceToBeKept = EventReference(context: testContext) + eventReferenceToBeKept.referencingEvent = newEvent + eventReferenceToBeKept.referencedEvent = oldEvent + + let eventReferenceToBeDeleted = EventReference(context: testContext) + eventReferenceToBeDeleted.referencingEvent = oldEvent + eventReferenceToBeDeleted.referencedEvent = reallyOldEvent + + try testContext.save() + + // Act + try await DatabaseCleaner.cleanupEntities( + for: KeyFixture.alice.publicKeyHex, + in: testContext, + keeping: 1 + ) + + // Assert + // We expect the really old event to be deleted entirely and the old event should be changed to a stub. + let events = try testContext.fetch(Event.allEventsRequest()) + XCTAssertEqual(events.count, 2) + XCTAssertEqual(events.first?.identifier, oldEvent.identifier) + XCTAssertEqual(events.first?.isStub, true) + XCTAssertEqual(events.last?.identifier, newEvent.identifier) + XCTAssertEqual(events.last?.isStub, false) + } + // MARK: - Events - func test_cleanup_keepsNEvents() async throws { - let user = KeyFixture.alice - _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + @MainActor func test_cleanup_keepsNEvents() async throws { + // Create the signed in user or the DatabaseCleaner will exit early. + _ = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) + var events = [Event]() for i in 0..<10 { let date = Date(timeIntervalSince1970: TimeInterval(i)) @@ -111,9 +150,11 @@ final class DatabaseCleanerTests: CoreDataTestCase { XCTAssertEqual(fetchedEvents.map { $0.identifier }, events.suffix(5).map { $0.identifier }) } - func test_cleanup_deletesOldEvents() async throws { + @MainActor func test_cleanup_deletesOldEvents() async throws { + // Create the signed in user or the DatabaseCleaner will exit early. let user = KeyFixture.alice _ = try Author.findOrCreate(by: user.publicKeyHex, context: testContext) + _ = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 9)) // old event let newEvent = try EventFixture.build(in: testContext, receivedAt: Date(timeIntervalSince1970: 11)) @@ -133,7 +174,7 @@ final class DatabaseCleanerTests: CoreDataTestCase { // MARK: - Authors - func test_cleanup_keepsInNetworkAuthors() async throws { + @MainActor func test_cleanup_keepsInNetworkAuthors() async throws { // Arrange let alice = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) let bob = try Author.findOrCreate(by: "bob", context: testContext) @@ -163,7 +204,7 @@ final class DatabaseCleanerTests: CoreDataTestCase { // MARK: - Follows - func test_cleanup_keepsInNetworkFollows() async throws { + @MainActor func test_cleanup_keepsInNetworkFollows() async throws { // Arrange let alice = try Author.findOrCreate(by: KeyFixture.alice.publicKeyHex, context: testContext) let bob = try Author.findOrCreate(by: "bob", context: testContext) From f6d1ef5e84dd63814ffab457ec4b3c10dc229b32 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Thu, 20 Jun 2024 11:39:31 -0400 Subject: [PATCH 11/14] Combine two tests to make contrast clearer --- NosTests/Model/EventTests.swift | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/NosTests/Model/EventTests.swift b/NosTests/Model/EventTests.swift index 56506c519..f96efe63c 100644 --- a/NosTests/Model/EventTests.swift +++ b/NosTests/Model/EventTests.swift @@ -41,27 +41,20 @@ final class EventTests: CoreDataTestCase { ) } - func testIdentifierCalculationWithNoTags() throws { + func testIdentifierCalculationWithEmptyAndNoTags() throws { // Arrange let content = "Testing nos #[0]" - let event = try EventFixture.build(in: testContext, content: content) + let nilTagsEvent = try EventFixture.build(in: testContext, content: content, tags: nil) + let emptyTagsEvent = try EventFixture.build(in: testContext, content: content, tags: []) // Act XCTAssertEqual( - try event.calculateIdentifier(), - "bc45c3ac53de113e1400fca956048a816ad1c2e6ecceba6b1372ca597066fa9a" + try nilTagsEvent.calculateIdentifier(), + "9b906de1db4ae84bda4b61b94724f8dfddd6fd9e6acddfe7ed79accb50052570" ) - } - - func testIdentifierCalculationWithEmptyTags() throws { - // Arrange - let content = "Testing nos #[0]" - let event = try EventFixture.build(in: testContext, content: content, tags: nil) - - // Act XCTAssertEqual( - try event.calculateIdentifier(), - "9b906de1db4ae84bda4b61b94724f8dfddd6fd9e6acddfe7ed79accb50052570" + try emptyTagsEvent.calculateIdentifier(), + "bc45c3ac53de113e1400fca956048a816ad1c2e6ecceba6b1372ca597066fa9a" ) } From 24f7e5722011f2084b54904bc3e10421d0f07b0d Mon Sep 17 00:00:00 2001 From: mplorentz Date: Thu, 20 Jun 2024 14:30:00 -0400 Subject: [PATCH 12/14] Fix Core Data threading warning --- Nos/Controller/PersistenceController.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Nos/Controller/PersistenceController.swift b/Nos/Controller/PersistenceController.swift index 4897bd1f8..b6db124aa 100644 --- a/Nos/Controller/PersistenceController.swift +++ b/Nos/Controller/PersistenceController.swift @@ -191,8 +191,8 @@ class PersistenceController { /// invalidate old items to keep it from growing indefinitely. /// /// This should only be called once right at app launch. - func cleanupEntities() async { - guard let authorKey = await currentUser.author?.hexadecimalPublicKey else { + @MainActor func cleanupEntities() async { + guard let authorKey = currentUser.author?.hexadecimalPublicKey else { return } From 64248e86f83f5cf7e1898146e2782007cd9827e0 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Fri, 21 Jun 2024 11:25:19 -0400 Subject: [PATCH 13/14] Combine similar Event.protectedFromCleanupPredicate functions --- Nos/Models/CoreData/Event+CoreDataClass.swift | 61 +++++-------------- .../EventReference+CoreDataClass.swift | 4 +- 2 files changed, 17 insertions(+), 48 deletions(-) diff --git a/Nos/Models/CoreData/Event+CoreDataClass.swift b/Nos/Models/CoreData/Event+CoreDataClass.swift index 641a14395..557642c84 100644 --- a/Nos/Models/CoreData/Event+CoreDataClass.swift +++ b/Nos/Models/CoreData/Event+CoreDataClass.swift @@ -377,68 +377,37 @@ public class Event: NosManagedObject, VerifiableEvent { } /// This constructs a predicate for events that should be protected from deletion when we are purging the database. - /// - Parameter before: The date before which events will be considered for cleanup. /// - Parameter user: The Author record for the currently logged in user. Special treatment is given to their data. - @nonobjc public class func protectedFromCleanupPredicate(for user: Author) -> NSPredicate { - // NOTE: This code is pretty much the same as that of `protectedFromCleanupSubqueryPredicate` but I can't - // figure out how to share it. If you change any of this you should probably make equivalent changes there. + /// - Parameter asSubquery: If true then each attribute in the predicate will prefixed with "$event." so the + /// predicate can be used in a SUBQUERY. + @nonobjc public class func protectedFromCleanupPredicate( + for user: Author, + asSubquery: Bool = false + ) -> NSPredicate { guard let userKey = user.hexadecimalPublicKey else { return NSPredicate.false } - // protect all events authored by the current user - let userEventsPredicate = NSPredicate(format: "author.hexadecimalPublicKey = '\(userKey)'") - - // protect stories that were read recently, so we don't redownload and show them as unread again - let oldStoryCutoffDate = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now - let recentlyReadStoriesPredicate = NSPredicate( - format: "(isRead = 1 AND receivedAt > %@)", // TODO: stubbed events? - oldStoryCutoffDate as CVarArg - ) - - // keep author reports from people we follow - let userReportPredicate = NSPredicate( - format: "(kind == \(EventKind.report.rawValue) AND " + - "SUBQUERY(authorReferences, $references, TRUEPREDICATE).@count > 0 AND " + - "SUBQUERY(eventReferences, $references, TRUEPREDICATE).@count == 0 AND " + - "ANY author.followers.source.hexadecimalPublicKey == %@)", - userKey - ) - - return NSCompoundPredicate( - orPredicateWithSubpredicates: [ - userEventsPredicate, - recentlyReadStoriesPredicate, - userReportPredicate - ] - ) - } - - /// Same as `protectedFromCleanupPredicate(for:)` but constructs a predicate designed to be used in the SUBQUERY - /// of another predicate. - @nonobjc public class func protectedFromCleanupSubqueryPredicate(for user: Author) -> NSPredicate { - // NOTE: This code is pretty much the same as that of `protectedFromCleanupPredicate` but I can't figure out - // how to share it. If you change any of this you should probably make equivalent changes there. - guard let userKey = user.hexadecimalPublicKey else { - return NSPredicate.false - } + // The string we use to reference the current event if we are constructing this predicate to be used in a + // subquery + let eventReference = asSubquery ? "$event." : "" // protect all events authored by the current user - let userEventsPredicate = NSPredicate(format: "$event.author.hexadecimalPublicKey = '\(userKey)'") + let userEventsPredicate = NSPredicate(format: "\(eventReference)author.hexadecimalPublicKey = '\(userKey)'") // protect stories that were read recently, so we don't redownload and show them as unread again let oldStoryCutoffDate = Calendar.current.date(byAdding: .day, value: -2, to: .now) ?? .now let recentlyReadStoriesPredicate = NSPredicate( - format: "($event.isRead = 1 AND $event.receivedAt > %@)", // TODO: stubbed events? + format: "(\(eventReference)isRead = 1 AND \(eventReference)receivedAt > %@)", oldStoryCutoffDate as CVarArg ) // keep author reports from people we follow let userReportPredicate = NSPredicate( - format: "($event.kind == \(EventKind.report.rawValue) AND " + - "SUBQUERY($event.authorReferences, $references, TRUEPREDICATE).@count > 0 AND " + - "SUBQUERY($event.eventReferences, $references, TRUEPREDICATE).@count == 0 AND " + - "ANY $event.author.followers.source.hexadecimalPublicKey == %@)", + format: "(\(eventReference)kind == \(EventKind.report.rawValue) AND " + + "SUBQUERY(\(eventReference)authorReferences, $references, TRUEPREDICATE).@count > 0 AND " + + "SUBQUERY(\(eventReference)eventReferences, $references, TRUEPREDICATE).@count == 0 AND " + + "ANY \(eventReference)author.followers.source.hexadecimalPublicKey == %@)", userKey ) diff --git a/Nos/Models/CoreData/EventReference+CoreDataClass.swift b/Nos/Models/CoreData/EventReference+CoreDataClass.swift index 33b644abc..75f818f95 100644 --- a/Nos/Models/CoreData/EventReference+CoreDataClass.swift +++ b/Nos/Models/CoreData/EventReference+CoreDataClass.swift @@ -35,12 +35,12 @@ public class EventReference: NosManagedObject { /// This fetches all the references that can be deleted during the `DatabaseCleaner` routine. It takes care /// to only select references before a given date that are not referenced by events we are keeping, and it also - /// accounts for "protected events" from `Event.protectedFromCleanupSubqueryPredicate()` to make sure we keep + /// accounts for "protected events" from `Event.protectedFromCleanupPredicate(...)` to make sure we keep /// events published by the current user etc. static func cleanupRequest(before date: Date, user: Author) -> NSFetchRequest { let fetchRequest = NSFetchRequest(entityName: "EventReference") fetchRequest.sortDescriptors = [NSSortDescriptor(keyPath: \EventReference.eventId, ascending: false)] - let protectedEventsPredicate = Event.protectedFromCleanupSubqueryPredicate(for: user) + let protectedEventsPredicate = Event.protectedFromCleanupPredicate(for: user, asSubquery: true) let referencedEventIsNotProtected = NSPredicate( format: "SUBQUERY(referencedEvent, $event, \(protectedEventsPredicate.predicateFormat)).@count == 0" ) From e50fd0770f5f71bde659351dacffff4e800c2ab8 Mon Sep 17 00:00:00 2001 From: mplorentz Date: Fri, 21 Jun 2024 11:40:27 -0400 Subject: [PATCH 14/14] Rename oldEventClause to oldUnreferencedEventsClause --- Nos/Models/CoreData/Event+CoreDataClass.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Nos/Models/CoreData/Event+CoreDataClass.swift b/Nos/Models/CoreData/Event+CoreDataClass.swift index 557642c84..844676562 100644 --- a/Nos/Models/CoreData/Event+CoreDataClass.swift +++ b/Nos/Models/CoreData/Event+CoreDataClass.swift @@ -357,12 +357,12 @@ public class Event: NosManagedObject, VerifiableEvent { let request = NSFetchRequest(entityName: "Event") request.sortDescriptors = [NSSortDescriptor(keyPath: \Event.receivedAt, ascending: true)] - let oldEventClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" + let oldUnreferencedEventsClause = "(receivedAt < %@ OR receivedAt == nil) AND referencingEvents.@count = 0" let notOwnEventClause = "(author != %@)" let readStoryClause = "(isRead = 1 AND receivedAt > %@)" let userReportClause = "(kind == \(EventKind.report.rawValue) AND " + "authorReferences.@count > 0 AND eventReferences.@count == 0)" - let clauses = "\(oldEventClause) AND" + + let clauses = "\(oldUnreferencedEventsClause) AND" + "\(notOwnEventClause) AND " + "NOT \(readStoryClause) AND " + "NOT \(userReportClause)"