Skip to content

Commit

Permalink
make loadBuildRange() timezone-aware + align tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@KSDaemon
KSDaemon committed Feb 26, 2025
1 parent 90f6f27 commit 120d17d
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 47 deletions.
8 changes: 7 additions & 1 deletion packages/cubejs-backend-shared/src/time.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,13 @@ export const inDbTimeZone = (timezone: string, timestampFormat: string, timestam
return moment.tz(timestamp, timezone).utc().format(timestampFormat);
};

export const utcToLocalTimeZone = (timezone: string, timestampFormat: string, timestamp: string): string => {
/**
* Takes timestamp in UTC, treat it as local time in provided timezone and returns the corresponding timestamp in UTC
*/
export const utcToLocalTimeZoneInUtc = (timezone: string, timestampFormat: string, timestamp: string): string | null => {
if (!timestamp) {
return null;
}
if (timestamp.length === 23) {
const zone = moment.tz.zone(timezone);
if (!zone) {
Expand Down
29 changes: 20 additions & 9 deletions packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import {
TO_PARTITION_RANGE,
MAX_SOURCE_ROW_LIMIT,
reformatInIsoLocal,
utcToLocalTimeZone,
utcToLocalTimeZoneInUtc,
timeSeries,
inDbTimeZone,
extractDate
Expand Down Expand Up @@ -136,9 +136,9 @@ export class PreAggregationPartitionRangeLoader {
return queryValues?.map(
param => {
if (param === BUILD_RANGE_START_LOCAL) {
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeStart);
} else if (param === BUILD_RANGE_END_LOCAL) {
return utcToLocalTimeZone(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, this.preAggregation.timestampFormat, buildRangeEnd);
} else {
return param;
}
Expand Down Expand Up @@ -396,31 +396,42 @@ export class PreAggregationPartitionRangeLoader {
const { preAggregationStartEndQueries } = this.preAggregation;
const [startDate, endDate] = await Promise.all(
preAggregationStartEndQueries.map(
async rangeQuery => PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
async rangeQuery => utcToLocalTimeZoneInUtc(
this.preAggregation.timezone,
'YYYY-MM-DDTHH:mm:ss.SSS',
PreAggregationPartitionRangeLoader.extractDate(await this.loadRangeQuery(rangeQuery)),
)
),
);

if (!this.preAggregation.partitionGranularity) {
return this.orNowIfEmpty([startDate, endDate]);
}

// startDate & endDate are `localized` here
const wholeSeriesRanges = PreAggregationPartitionRangeLoader.timeSeries(
this.preAggregation.partitionGranularity,
this.orNowIfEmpty([startDate, endDate]),
this.preAggregation.timestampPrecision,
);
const [rangeStart, rangeEnd] = await Promise.all(
preAggregationStartEndQueries.map(
async (rangeQuery, i) => PreAggregationPartitionRangeLoader.extractDate(
await this.loadRangeQuery(
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
async (rangeQuery, i) => utcToLocalTimeZoneInUtc(
this.preAggregation.timezone,
'YYYY-MM-DDTHH:mm:ss.SSS',
PreAggregationPartitionRangeLoader.extractDate(
await this.loadRangeQuery(
rangeQuery, i === 0 ? wholeSeriesRanges[0] : wholeSeriesRanges[wholeSeriesRanges.length - 1],
),
),
),
)
),
);
return this.orNowIfEmpty([rangeStart, rangeEnd]);
}

private now() {
return utcToLocalTimeZone(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
return utcToLocalTimeZoneInUtc(this.preAggregation.timezone, 'YYYY-MM-DDTHH:mm:ss.SSS', new Date().toJSON().substring(0, 23));
}

private orNowIfEmpty(dateRange: QueryDateRange): QueryDateRange {
Expand Down
74 changes: 37 additions & 37 deletions packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,25 +8,25 @@ import { CompilerApi } from '../../src/core/CompilerApi';
const schemaContent = `
cube('Foo', {
sql: \`select * from foo_\${SECURITY_CONTEXT.tenantId.unsafeValue()}\`,
measures: {
count: {
type: 'count'
},
total: {
sql: 'amount',
type: 'sum'
},
},
dimensions: {
time: {
sql: 'timestamp',
type: 'time'
}
},
preAggregations: {
main: {
type: 'originalSql',
Expand Down Expand Up @@ -87,20 +87,20 @@ cube('Foo', {
cube('Bar', {
sql: 'select * from bar',
measures: {
count: {
type: 'count'
}
},
dimensions: {
time: {
sql: 'timestamp',
type: 'time'
}
},
preAggregations: {
first: {
type: 'rollup',
Expand Down Expand Up @@ -131,63 +131,63 @@ const repositoryWithRollupJoin: SchemaFileRepository = {
{ fileName: 'main.js', content: `
cube(\`Users\`, {
sql: \`SELECT * FROM public.users\`,
preAggregations: {
usersRollup: {
dimensions: [CUBE.id],
},
},
measures: {
count: {
type: \`count\`,
},
},
dimensions: {
id: {
sql: \`id\`,
type: \`string\`,
primaryKey: true,
},
name: {
sql: \`name\`,
type: \`string\`,
},
},
});
cube('Orders', {
sql: \`SELECT * FROM orders\`,
preAggregations: {
ordersRollup: {
measures: [CUBE.count],
dimensions: [CUBE.userId, CUBE.status],
},
ordersRollupJoin: {
type: \`rollupJoin\`,
measures: [CUBE.count],
dimensions: [Users.name],
rollups: [Users.usersRollup, CUBE.ordersRollup],
},
},
joins: {
Users: {
relationship: \`belongsTo\`,
sql: \`\${CUBE.userId} = \${Users.id}\`,
},
},
measures: {
count: {
type: \`count\`,
},
},
dimensions: {
id: {
sql: \`id\`,
Expand Down Expand Up @@ -215,13 +215,13 @@ const repositoryWithoutPreAggregations: SchemaFileRepository = {
fileName: 'main.js', content: `
cube('Bar', {
sql: 'select * from bar',
measures: {
count: {
type: 'count'
}
},
dimensions: {
time: {
sql: 'timestamp',
Expand Down Expand Up @@ -698,63 +698,63 @@ describe('Refresh Scheduler', () => {
{ tableName: 'stb_pre_aggregations.foo_second20201231', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'UTC', fromTable: 'bar' },
{
tableName: 'stb_pre_aggregations.foo_first20201231',
tableName: 'stb_pre_aggregations.foo_first20201230',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.foo_orphaned20201231', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_second20201231',
tableName: 'stb_pre_aggregations.foo_second20201230',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.bar_first20201231', timezone: 'America/Los_Angeles', fromTable: 'bar' },
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },

{ tableName: 'stb_pre_aggregations.foo_first20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_second20201230', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'UTC', fromTable: 'bar' },
{
tableName: 'stb_pre_aggregations.foo_first20201230',
tableName: 'stb_pre_aggregations.foo_first20201229',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.foo_orphaned20201230', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_second20201230',
tableName: 'stb_pre_aggregations.foo_second20201229',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.bar_first20201230', timezone: 'America/Los_Angeles', fromTable: 'bar' },
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },

{ tableName: 'stb_pre_aggregations.foo_first20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_second20201229', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'UTC', fromTable: 'bar' },
{
tableName: 'stb_pre_aggregations.foo_first20201229',
tableName: 'stb_pre_aggregations.foo_first20201228',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.foo_orphaned20201229', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_second20201229',
tableName: 'stb_pre_aggregations.foo_second20201228',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.bar_first20201229', timezone: 'America/Los_Angeles', fromTable: 'bar' },
{ tableName: 'stb_pre_aggregations.bar_first20201228', timezone: 'America/Los_Angeles', fromTable: 'bar' },

{ tableName: 'stb_pre_aggregations.foo_first20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_second20201228', timezone: 'UTC', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_first20201228',
tableName: 'stb_pre_aggregations.foo_first20201227',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.foo_orphaned20201228', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_second20201228',
tableName: 'stb_pre_aggregations.foo_second20201227',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
Expand All @@ -763,13 +763,13 @@ describe('Refresh Scheduler', () => {
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_second20201227', timezone: 'UTC', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_first20201227',
tableName: 'stb_pre_aggregations.foo_first20201226',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
{ tableName: 'stb_pre_aggregations.foo_orphaned20201227', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{ tableName: 'stb_pre_aggregations.foo_orphaned20201226', timezone: 'America/Los_Angeles', fromTable: 'foo_tenant1' },
{
tableName: 'stb_pre_aggregations.foo_second20201227',
tableName: 'stb_pre_aggregations.foo_second20201226',
timezone: 'America/Los_Angeles',
fromTable: 'foo_tenant1',
},
Expand Down

0 comments on commit 120d17d

Please sign in to comment.