Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Schema 3.0.0 #41

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Schema 3.0.0 #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3edb2a1
save
balazskreith Jan 10, 2025
3818926
save
balazskreith Jan 17, 2025
a97ed56
save
balazskreith Jan 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ProtobufConverter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ export class ProtobufConverter {
} else if (types.length === 2) {
type = types[1];
required = false;
} else if (typeof types === 'object') {
type = types;
}
let isObject = typeof type === "object";
let isArray = isObject && type.type === "array";
Expand Down
13 changes: 10 additions & 3 deletions ProtobufConverterV3.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,17 @@ export class ProtobufConverterV3 {
} else if (types.length === 2) {
type = types[1];
required = false;
} else if (typeof types === 'object') {
type = types;
}
let isObject = typeof type === "object";
let isArray = isObject && type.type === "array";
if (isArray) {
type = type.items;
}
isObject = typeof type === "object";

// field.name === 'kind' && console.log('kind', field, type, isArray, required, isObject);
if (isObject && type.type === "enum") {
this._symbols.set(field.name, type.symbols);
if (!this._classEnum) {
Expand Down Expand Up @@ -110,7 +114,7 @@ export class ProtobufConverterV3 {
return;
}
let protoType = undefined;
if (this._uuidFields.has(field.name)) {
if (this._uuidFields.has(field.name) || field.name === "appData") {
protoType = "bytes";
} else {
protoType = this._mapPrimitive(type);
Expand Down Expand Up @@ -145,13 +149,14 @@ export class ProtobufConverterV3 {
return undefined;
}
_addField({ doc, name, type, isArray, required }) {
this._fields.push({
const field = {
doc,
name,
type,
isArray,
required,
});
};
this._fields.push(field);
}

_getMessageField({ fieldNum, name, type, isArray, required }) {
Expand All @@ -162,6 +167,7 @@ export class ProtobufConverterV3 {
// else if (required) result.push(`required`);
// else result.push(`optional`);

// (!name || name === 'kind') && console.log('name', name, 'required', required, type, isArray);
result.push(type);
result.push(`${name} = ${fieldNum}`);

Expand All @@ -183,6 +189,7 @@ export class ProtobufConverterV3 {
} else {
optionalFields.push(field);
}
// field.name === 'kind' && console.log('kind', field);
}
[arrayFields, requiredFields, optionalFields].forEach(fields => {
fields.sort((a, b) => {
Expand Down
40 changes: 1 addition & 39 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,51 +7,13 @@ Schemas describe the communication between services in the ObserveRTC stack. The
* [CSV](csv-headers/) when exporting reports in CSV format from the observer, the order of the columns might be important.


# Install

## NPM
```javascript
npm i @observertc/schemas
```


# Versioning
Schemas use SemVer version numbers of `MAJOR`.`MINOR`.`PATCH`.

Increasing the number of `PATCH`, `MINOR`, or `MAJOR` implies the following:

* `PATCH` indicates a change in the related libraries (e.g., bugfix in encoder/decoder).
* `MINOR` the source schema has been altered.
* `MINOR` Represents the day of the WebRTC Stats draft published the schema is based on (e.g.: 20241107)
* `MAJOR` implies conceptual changes in the schema.

## Change the schema

### Schema change requests

Depending on which minor (or major) version is coming, you can write it in the [discussion](https://github.com/ObserveRTC/schemas/discussions).


## Create PR for schema changes

Discussion is good to request a schema, PR is better.
Step to create a PR (after you cloned the repo):
1. create your branch
2. change the [source of the schemas](/sources), `sources/version.txt`, trace the change in the `sources/CHANGELOG.md`.
3. Generate `npm-lib` typescripts (see below)
4. open the PR




# Develop

To run the schema generator for npm-lib:

```javascript
git clone https://github.com/observertc/schemas && \
cd schemas && \
npm i && \
node index.js
```


79 changes: 19 additions & 60 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ const CSV_OUTPUTS_PATH = `./${OUTPUTS_PATH}/csv`;
const SQL_OUTPUTS_PATH = `./${OUTPUTS_PATH}/sql`;

const NPM_SAMPLES_LIB_PATH = "./npm-samples-lib";
const NPM_REPORTS_LIB_PATH = "./npm-reports-lib";
const W3C_STATS_IDENTIFIERS = "./sources/w3c/W3cStatsIdentifiers.ts";

async function rmDir(folderPath) {
Expand All @@ -44,7 +43,7 @@ async function rmDir(folderPath) {
}

function fetchChunks() {
for (const schemaType of ["reports", "samples"]) {
for (const schemaType of ["samples"]) {
const schemaPath = path.join(SOURCE_PATH, schemaType);
for (const file of fs.readdirSync(schemaPath)) {
if (!file.endsWith(".chunk.avsc")) continue;
Expand All @@ -60,7 +59,7 @@ function fetchChunks() {

function fetchSources() {
const sources = new Map();
for (const schemaType of ["reports", "samples"]) {
for (const schemaType of ["samples"]) {
const schemaPath = path.join(SOURCE_PATH, schemaType);
for (const file of fs.readdirSync(schemaPath)) {
if (!file.endsWith("avsc")) continue;
Expand Down Expand Up @@ -94,7 +93,6 @@ const main = async () => {
const sources = fetchSources();
const w3cStatsIdentifiers = fs.readFileSync(W3C_STATS_IDENTIFIERS, 'utf-8');
const npmSamplesLib = new NpmLib(NPM_SAMPLES_LIB_PATH);
const npmReportsLib = new NpmLib(NPM_REPORTS_LIB_PATH);
const npmEncoderLib = new NpmSampleEncoderLib();
const npmDecoderLib = new NpmSamplesDecoderLib();
const version = fs.readFileSync(path.join(SOURCE_PATH, "version.txt"), 'utf-8');
Expand All @@ -103,13 +101,9 @@ const main = async () => {
const markdownLists = [];
const bigQueryTables = [];
const redshiftTables = [];
const reportTypes = [
`/**\n * Schema Version: ${version} \n */`
];
for (const [fileName, source] of sources) {
const avsc = source.getAvsc();
const schemaType = source.getSchemaType();
const isReport = fileName.includes("report");
let schema;
try {
schema = JSON.parse(avsc);
Expand All @@ -133,71 +127,40 @@ const main = async () => {

fs.writeFileSync(path.join(TYPESCRIPT_OUTPUTS_PATH, `${schemaName}.ts`), module);

if (isReport) {
const { csvColumnList, createTable } = makeRedshiftSql(schema);
const { createTable: createBigqueryTable } = makeBigQuerySql(schema);
if (csvColumnList) {
fs.writeFileSync(path.join(CSV_OUTPUTS_PATH, `${schemaName}-csv-header.txt`), csvColumnList);
} else {
console.warn(`No csv header txt is generated for ${schemaName}`);
}

if (createTable) {
redshiftTables.push(createTable);
// fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `${schemaName}-redshift.sql`), createTable);
} else {
console.warn(`No redshift sql is generated for ${schemaName}`);
}
if (createBigqueryTable) {
bigQueryTables.push(createBigqueryTable);
// fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `${schemaName}-bigquery.sql`), createBigqueryTable);
} else {
console.warn(`No bigquery sql is generated for ${schemaName}`);
}
reportTypes.push(module);
npmReportsLib.addEntry({
fileName,
schemaName,
schemaType,
exports,
typescript: module,
markdown: markdownDoc,
});
} else {
if (fileName === "ClientSample") {
npmDecoderLib.addSamplesTsCode(module);
npmEncoderLib.addSamplesTsCode(module);
npmSamplesLib.addEntry({
fileName,
schemaName,
schemaType,
exports,
typescript: module,
markdown: markdownDoc,
});
}
npmSamplesLib.addEntry({
fileName,
schemaName,
schemaType,
exports,
typescript: module,
markdown: markdownDoc,
});

fs.writeFileSync(path.join(AVSC_OUTPUTS_PATH, `${schemaName}.avsc`), avsc);
}
fs.writeFileSync(`schemaList.md`, markdownLists.join(`\n`))
fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `bigquery.sql`), bigQueryTables.join("\n\n"));
fs.writeFileSync(path.join(SQL_OUTPUTS_PATH, `redshift.sql`), redshiftTables.join("\n\n"));
fs.writeFileSync(path.join(TYPESCRIPT_OUTPUTS_PATH, `ReportTypes.ts`), reportTypes.join("\n"));

// generate protobuf schema if we can

const samplesSource = sources.get("samples");
if (samplesSource) {
const schema = JSON.parse(samplesSource.getAvsc());
const clientSampleSource = sources.get("ClientSample");
if (clientSampleSource) {
const schema = JSON.parse(clientSampleSource.getAvsc());
const protobufSchema = protobufUtils.convertToProtobufSchema(schema, version);
fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufSamples.proto"), protobufSchema);
fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSample.proto"), protobufSchema);

const protobufSchemaV3 = protobufUtils.convertToProtobufSchemaV3(schema, version);
fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufSamplesV3.proto"), protobufSchemaV3);
fs.writeFileSync(path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSampleV3.proto"), protobufSchemaV3);

const protobufSchemaV3Optional = protobufUtils.convertToProtobufSchemaV3(schema, version, true);
const v3schemaOptionalPath = path.join(PROTO_OUTPUTS_PATH, "ProtobufSamplesV3Optional.proto");
const v3schemaOptionalPath = path.join(PROTO_OUTPUTS_PATH, "ProtobufClientSampleV3Optional.proto");
fs.writeFileSync(v3schemaOptionalPath, protobufSchemaV3Optional);
await protobufUtils.createTypescriptModels(v3schemaOptionalPath, path.join(TEMP_PATH));
const protobufSchemaV3OptionalTs = fs.readFileSync(path.join(TEMP_PATH, "outputs", "proto", "ProtobufSamplesV3Optional_pb.ts"), 'utf-8');
const protobufSchemaV3OptionalTs = fs.readFileSync(path.join(TEMP_PATH, "outputs", "proto", "ProtobufClientSampleV3Optional_pb.ts"), 'utf-8');
npmEncoderLib.addSamplesProtobufTsCode(protobufSchemaV3OptionalTs);
npmDecoderLib.addSamplesProtobufTsCode(protobufSchemaV3OptionalTs);
// console.log(protobufSchemaV3OptionalTs);
Expand All @@ -207,20 +170,16 @@ const main = async () => {

const changelog = fs.readFileSync(path.join(SOURCE_PATH, "CHANGELOG.md"), 'utf-8');
npmSamplesLib.version = version;
npmReportsLib.version = version;
npmEncoderLib.version = version;
npmDecoderLib.version = version;

npmSamplesLib.changelog = changelog;
npmReportsLib.changelog = changelog;

npmSamplesLib.clear();
npmReportsLib.clear();
npmEncoderLib.clear();
npmDecoderLib.clear();

npmSamplesLib.make();
npmReportsLib.make();
npmEncoderLib.make();
npmDecoderLib.make();

Expand Down
11 changes: 10 additions & 1 deletion makeTsModule.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,15 @@ function getTsType(avroType, addDoc = true) {
tsObj,
tsDependencies,
}

}

function getAttachmentsTsType() {
return {
tsType: "Record<string, unknown>",
tsObj: undefined,
tsDependencies: undefined,
}
}

function makeTsObj(avroSchema, addDoc = true) {
Expand All @@ -103,7 +112,7 @@ function makeTsObj(avroSchema, addDoc = true) {
const tsTypes = [];
for (const avroType of avroTypes) {
if (avroType === "null") continue;
const { tsType, tsObj, tsDependencies } = getTsType(avroType, addDoc);
const { tsType, tsObj, tsDependencies } = field.name === 'attachments' ? getAttachmentsTsType() : getTsType(avroType, addDoc);
if (tsObj) {
dependencies.push(tsObj);
if (tsDependencies) {
Expand Down
3 changes: 0 additions & 3 deletions npm-reports-lib/.gitignore
View file
Open in desktop

This file was deleted.

1 change: 0 additions & 1 deletion npm-reports-lib/.npmrc
View file
Open in desktop

This file was deleted.

Loading
Loading