Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "chore: waku_archive add protection against queries longer than 24h" #3278

Merged
merged 2 commits into from
Feb 6, 2025
Merged

Revert "chore: waku_archive add protection against queries longer than 24h" #3278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
118bc57
Revert "chore: waku_archive add protection against queries longer tha…
Ivansete-status Feb 5, 2025
6fe96b1
nph again
Ivansete-status Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 34 additions & 31 deletions tests/node/test_wakunode_legacy_store.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,16 @@ suite "Waku Store - End to End - Sorted Archive":
let timeOrigin = now()
archiveMessages =
@[
fakeWakuMessage(@[byte 00], ts = ts(-90, timeOrigin)),
fakeWakuMessage(@[byte 01], ts = ts(-80, timeOrigin)),
fakeWakuMessage(@[byte 02], ts = ts(-70, timeOrigin)),
fakeWakuMessage(@[byte 03], ts = ts(-60, timeOrigin)),
fakeWakuMessage(@[byte 04], ts = ts(-50, timeOrigin)),
fakeWakuMessage(@[byte 05], ts = ts(-40, timeOrigin)),
fakeWakuMessage(@[byte 06], ts = ts(-30, timeOrigin)),
fakeWakuMessage(@[byte 07], ts = ts(-20, timeOrigin)),
fakeWakuMessage(@[byte 08], ts = ts(-10, timeOrigin)),
fakeWakuMessage(@[byte 09], ts = ts(00, timeOrigin)),
fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)),
fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)),
fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)),
fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)),
fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)),
fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)),
fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)),
fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)),
fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)),
fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)),
]

historyQuery = HistoryQuery(
Expand Down Expand Up @@ -657,23 +657,23 @@ suite "Waku Store - End to End - Archive with Multiple Topics":
pageSize: 5,
)

let timeOrigin = now() - 90
let timeOrigin = now()
originTs = proc(offset = 0): Timestamp {.gcsafe, raises: [].} =
ts(offset, timeOrigin)

archiveMessages =
@[
fakeWakuMessage(@[byte 00], ts = originTs(-90), contentTopic = contentTopic),
fakeWakuMessage(@[byte 01], ts = originTs(-80), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 02], ts = originTs(-70), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 03], ts = originTs(-60), contentTopic = contentTopic),
fakeWakuMessage(@[byte 04], ts = originTs(-50), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 05], ts = originTs(-40), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 06], ts = originTs(-30), contentTopic = contentTopic),
fakeWakuMessage(@[byte 07], ts = originTs(-20), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 08], ts = originTs(-10), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic),
fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic),
fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic),
fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC),
fakeWakuMessage(
@[byte 09], ts = originTs(00), contentTopic = contentTopicSpecials
@[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials
),
]

Expand Down Expand Up @@ -827,31 +827,34 @@ suite "Waku Store - End to End - Archive with Multiple Topics":
suite "Validation of Time-based Filtering":
asyncTest "Basic Time Filtering":
# Given a history query with start and end time

historyQuery.startTime = some(originTs(-90))
historyQuery.endTime = some(originTs(-70))
historyQuery.startTime = some(originTs(20))
historyQuery.endTime = some(originTs(40))

# When making a history query
let queryResponse = await client.query(historyQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages ==
@[archiveMessages[0], archiveMessages[1], archiveMessages[2]]
@[archiveMessages[2], archiveMessages[3], archiveMessages[4]]

asyncTest "Only Start Time Specified":
# Given a history query with only start time
historyQuery.startTime = some(originTs(-20))
historyQuery.startTime = some(originTs(20))
historyQuery.endTime = none(Timestamp)
historyQuery.pubsubTopic = none(string)

# When making a history query
let queryResponse = await client.query(historyQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages ==
@[archiveMessages[7], archiveMessages[8], archiveMessages[9]]
@[
archiveMessages[2],
archiveMessages[3],
archiveMessages[4],
archiveMessages[5],
]

asyncTest "Only End Time Specified":
# Given a history query with only end time
Expand Down Expand Up @@ -886,16 +889,16 @@ suite "Waku Store - End to End - Archive with Multiple Topics":

asyncTest "Time Filtering with Content Filtering":
# Given a history query with time and content filtering
historyQuery.startTime = some(originTs(-90))
historyQuery.endTime = some(originTs(-60))
historyQuery.startTime = some(originTs(20))
historyQuery.endTime = some(originTs(60))
historyQuery.contentTopics = @[contentTopicC]

# When making a history query
let queryResponse = await client.query(historyQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages == @[archiveMessages[2]]
queryResponse.get().messages == @[archiveMessages[2], archiveMessages[5]]

asyncTest "Messages Outside of Time Range":
# Given a history query with a valid time range which does not contain any messages
Expand Down
79 changes: 43 additions & 36 deletions tests/node/test_wakunode_store.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ suite "Waku Store - End to End - Sorted Archive":
let timeOrigin = now()
let messages =
@[
fakeWakuMessage(@[byte 00], ts = ts(-90, timeOrigin)),
fakeWakuMessage(@[byte 01], ts = ts(-80, timeOrigin)),
fakeWakuMessage(@[byte 02], ts = ts(-70, timeOrigin)),
fakeWakuMessage(@[byte 03], ts = ts(-60, timeOrigin)),
fakeWakuMessage(@[byte 04], ts = ts(-50, timeOrigin)),
fakeWakuMessage(@[byte 05], ts = ts(-40, timeOrigin)),
fakeWakuMessage(@[byte 06], ts = ts(-30, timeOrigin)),
fakeWakuMessage(@[byte 07], ts = ts(-20, timeOrigin)),
fakeWakuMessage(@[byte 08], ts = ts(-10, timeOrigin)),
fakeWakuMessage(@[byte 09], ts = ts(00, timeOrigin)),
fakeWakuMessage(@[byte 00], ts = ts(00, timeOrigin)),
fakeWakuMessage(@[byte 01], ts = ts(10, timeOrigin)),
fakeWakuMessage(@[byte 02], ts = ts(20, timeOrigin)),
fakeWakuMessage(@[byte 03], ts = ts(30, timeOrigin)),
fakeWakuMessage(@[byte 04], ts = ts(40, timeOrigin)),
fakeWakuMessage(@[byte 05], ts = ts(50, timeOrigin)),
fakeWakuMessage(@[byte 06], ts = ts(60, timeOrigin)),
fakeWakuMessage(@[byte 07], ts = ts(70, timeOrigin)),
fakeWakuMessage(@[byte 08], ts = ts(80, timeOrigin)),
fakeWakuMessage(@[byte 09], ts = ts(90, timeOrigin)),
]
archiveMessages = messages.mapIt(
WakuMessageKeyValue(
Expand Down Expand Up @@ -909,17 +909,17 @@ suite "Waku Store - End to End - Archive with Multiple Topics":

let messages =
@[
fakeWakuMessage(@[byte 00], ts = originTs(-90), contentTopic = contentTopic),
fakeWakuMessage(@[byte 01], ts = originTs(-80), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 02], ts = originTs(-70), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 03], ts = originTs(-60), contentTopic = contentTopic),
fakeWakuMessage(@[byte 04], ts = originTs(-50), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 05], ts = originTs(-40), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 06], ts = originTs(-30), contentTopic = contentTopic),
fakeWakuMessage(@[byte 07], ts = originTs(-20), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 08], ts = originTs(-10), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 00], ts = originTs(00), contentTopic = contentTopic),
fakeWakuMessage(@[byte 01], ts = originTs(10), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 02], ts = originTs(20), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 03], ts = originTs(30), contentTopic = contentTopic),
fakeWakuMessage(@[byte 04], ts = originTs(40), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 05], ts = originTs(50), contentTopic = contentTopicC),
fakeWakuMessage(@[byte 06], ts = originTs(60), contentTopic = contentTopic),
fakeWakuMessage(@[byte 07], ts = originTs(70), contentTopic = contentTopicB),
fakeWakuMessage(@[byte 08], ts = originTs(80), contentTopic = contentTopicC),
fakeWakuMessage(
@[byte 09], ts = originTs(00), contentTopic = contentTopicSpecials
@[byte 09], ts = originTs(90), contentTopic = contentTopicSpecials
),
]

Expand Down Expand Up @@ -1089,46 +1089,53 @@ suite "Waku Store - End to End - Archive with Multiple Topics":
suite "Validation of Time-based Filtering":
asyncTest "Basic Time Filtering":
# Given a history query with start and end time
storeQuery.startTime = some(originTs(-90))
storeQuery.endTime = some(originTs(-60))
storeQuery.contentTopics = @[contentTopic, contentTopicB, contentTopicC]
storeQuery.startTime = some(originTs(20))
storeQuery.endTime = some(originTs(40))

# When making a history query
let queryResponse = await client.query(storeQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages ==
@[
archiveMessages[0],
archiveMessages[1],
archiveMessages[2],
archiveMessages[3],
]
@[archiveMessages[2], archiveMessages[3], archiveMessages[4]]

asyncTest "Only Start Time Specified":
# Given a history query with only start time
storeQuery.startTime = some(originTs(-40))
storeQuery.startTime = some(originTs(20))
storeQuery.endTime = none(Timestamp)

# When making a history query
let queryResponse = await client.query(storeQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages == @[archiveMessages[5]]
queryResponse.get().messages ==
@[
archiveMessages[2],
archiveMessages[3],
archiveMessages[4],
archiveMessages[5],
]

asyncTest "Only End Time Specified":
# Given a history query with only end time
storeQuery.startTime = none(Timestamp)
storeQuery.endTime = some(originTs(-80))
storeQuery.endTime = some(originTs(40))

# When making a history query
let queryResponse = await client.query(storeQuery, serverRemotePeerInfo)

# Then the response contains no messages
check:
queryResponse.get().messages == @[archiveMessages[0], archiveMessages[1]]
queryResponse.get().messages ==
@[
archiveMessages[0],
archiveMessages[1],
archiveMessages[2],
archiveMessages[3],
archiveMessages[4],
]

asyncTest "Invalid Time Range":
# Given a history query with invalid time range
Expand All @@ -1144,16 +1151,16 @@ suite "Waku Store - End to End - Archive with Multiple Topics":

asyncTest "Time Filtering with Content Filtering":
# Given a history query with time and content filtering
storeQuery.startTime = some(originTs(-60))
storeQuery.endTime = some(originTs(-20))
storeQuery.startTime = some(originTs(20))
storeQuery.endTime = some(originTs(60))
storeQuery.contentTopics = @[contentTopicC]

# When making a history query
let queryResponse = await client.query(storeQuery, serverRemotePeerInfo)

# Then the response contains the messages
check:
queryResponse.get().messages == @[archiveMessages[5]]
queryResponse.get().messages == @[archiveMessages[2], archiveMessages[5]]

asyncTest "Messages Outside of Time Range":
# Given a history query with a valid time range which does not contain any messages
Expand Down
43 changes: 12 additions & 31 deletions tests/waku_archive/test_waku_archive.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,7 @@ procSuite "Waku Archive - find messages":
response.messages.mapIt(it.timestamp) == @[ts(30, timeOrigin), ts(50, timeOrigin)]

test "handle temporal history query with a zero-size time window":
## A zero-size window results in an error to the store client. That kind of queries
## are pointless and we need to rapidly inform about that to the client.
## A zero-size window results in an empty list of history messages
## Given
let req = ArchiveQuery(
contentTopics: @[ContentTopic("1")],
Expand All @@ -504,45 +503,27 @@ procSuite "Waku Archive - find messages":
let res = waitFor archiveA.findMessages(req)

## Then
check not res.isOk()
check res.isOk()

let response = res.tryGet()
check:
response.messages.len == 0

test "handle temporal history query with an invalid time window":
## A query with an invalid time range should immediately return a query error to the client
## A history query with an invalid time range results in an empty list of history messages
## Given
let req = ArchiveQuery(
contentTopics: @[ContentTopic("1")],
startTime: some(Timestamp(5)),
endTime: some(Timestamp(4)),
endTime: some(Timestamp(2)),
)

## When
let res = waitFor archiveA.findMessages(req)

## Then
check not res.isOk()

test "time range should be smaller than 24h":
let oneDayRangeNanos = 86_400_000_000_000
let now = getNowInNanosecondTime()

var res = waitFor archiveA.findMessages(
ArchiveQuery(
contentTopics: @[ContentTopic("1")],
startTime: some(Timestamp(now - oneDayRangeNanos - 1)),
endTime: some(Timestamp(now)),
)
)

## It fails if range is a bit bigger than 24h
check not res.isOk()

res = waitFor archiveA.findMessages(
ArchiveQuery(
contentTopics: @[ContentTopic("1")],
startTime: some(Timestamp(now - oneDayRangeNanos)),
endTime: some(Timestamp(now)),
)
)

## Ok if range is 24h
check res.isOk()

let response = res.tryGet()
check:
response.messages.len == 0
1 change: 1 addition & 0 deletions tests/waku_rln_relay/test_rln_group_manager_onchain.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ suite "Onchain group manager":
(await manager.startGroupSync()).isOkOr:
raiseAssert $error


asyncTest "startGroupSync: should guard against uninitialized state":
(await manager.startGroupSync()).isErrOr:
raiseAssert "Expected error when not initialized"
Expand Down
3 changes: 2 additions & 1 deletion waku/incentivization/eligibility_manager.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ type EligibilityManager* = ref object # FIXME: make web3 private?
proc init*(
T: type EligibilityManager, ethClient: string
): Future[EligibilityManager] {.async.} =
return EligibilityManager(web3: await newWeb3(ethClient), seenTxIds: initHashSet[TxHash]())
return
EligibilityManager(web3: await newWeb3(ethClient), seenTxIds: initHashSet[TxHash]())
# TODO: handle error if web3 instance is not established

# Clean up the web3 instance
Expand Down
30 changes: 0 additions & 30 deletions waku/waku_archive/archive.nim
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,41 +163,11 @@ proc syncMessageIngress*(

return ok()

proc validateTimeRange(
startTime: Option[Timestamp], endTime: Option[Timestamp]
): Result[void, ArchiveError] =
## Returns ok if the given time range is shorter than one day, and error otherwise.
## We restrict the maximum allowed time of 24h to prevent excessive big queries.

let oneDayRangeNanos = 86_400_000_000_000
let now = getNowInNanosecondTime()

var startTimeToValidate = now - oneDayRangeNanos
if startTime.isSome():
startTimeToValidate = startTime.get()

var endTimeToValidate = now
if endTime.isSome():
endTimeToValidate = endTime.get()

if startTimeToValidate >= endTimeToValidate:
return err(ArchiveError.invalidQuery("startTime should be before endTime"))

if (endTimeToValidate - startTimeToValidate) > oneDayRangeNanos:
return err(
ArchiveError.invalidQuery("time range should be smaller than one day in nanos")
)

return ok()

proc findMessages*(
self: WakuArchive, query: ArchiveQuery
): Future[ArchiveResult] {.async, gcsafe.} =
## Search the archive to return a single page of messages matching the query criteria

validateTimeRange(query.startTime, query.endTime).isOkOr:
return err(error)

if query.cursor.isSome():
let cursor = query.cursor.get()

Expand Down
Loading
Loading