From 20da832c020bb1316752e19850ad4822ba447ab1 Mon Sep 17 00:00:00 2001 From: Thanos Kolovos Date: Thu, 17 Nov 2022 12:38:28 +0200 Subject: [PATCH] Add download pseudo translations flag This uses the `pseudo` attribute in API to download files with pseudo translations. The change is on the pull command, in which, if `pseudo` flag is true we use the source file download instead of the translations file download because for compatibility issues. The `pseudo` attribute will be removed from the API for translations in the future and will be used only in source. --- README.md | 2 + cmd/tx/main.go | 6 ++ internal/txlib/pull.go | 28 +++++--- internal/txlib/pull_test.go | 66 ++++++++++++++++--- pkg/txapi/resource_strings_async_downloads.go | 3 +- 5 files changed, 86 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 376a11e..c2fdb39 100644 --- a/README.md +++ b/README.md @@ -757,6 +757,8 @@ default to taking the filesystem timestamp into account. speed. The `--workers` flag sets the number of concurrent downloads possible at any time. +- `--pseudo`: Generate mock string translations with a ~20% default length increase in characters. + ### Removing resources from Transifex The tx delete command lets you delete a resource that's in your `config` file and on Transifex. diff --git a/cmd/tx/main.go b/cmd/tx/main.go index 254c397..cc20348 100644 --- a/cmd/tx/main.go +++ b/cmd/tx/main.go @@ -482,6 +482,11 @@ func Main() { Name: "silent", Usage: "Whether to reduce verbosity of the output", }, + &cli.BoolFlag{ + Name: "pseudo", + Usage: "Generate mock string translations", + Value: false, + }, }, Action: func(c *cli.Context) error { cfg, err := config.LoadFromPaths(c.String("root-config"), @@ -534,6 +539,7 @@ func Main() { MinimumPercentage: c.Int("minimum-perc"), Workers: c.Int("workers"), Silent: c.Bool("silent"), + Pseudo: c.Bool("pseudo"), } if c.Bool("xliff") && c.Bool("json") { diff --git a/internal/txlib/pull.go b/internal/txlib/pull.go index 3cf2968..b4f4c69 100644 --- a/internal/txlib/pull.go +++ b/internal/txlib/pull.go @@ -32,6 +32,7 @@ type PullCommandArguments struct { MinimumPercentage int Workers int Silent bool + Pseudo bool } func PullCommand( @@ -403,6 +404,7 @@ func (task *FilePullTask) Run(send func(string), abort func()) { resource, args.ContentEncoding, args.FileType, + args.Pseudo, ) return err }, @@ -497,14 +499,24 @@ func (task *FilePullTask) Run(send func(string), abort func()) { err = handleThrottling( func() error { var err error - download, err = txapi.CreateTranslationsAsyncDownload( - api, - resource, - languageCode, - args.ContentEncoding, - args.FileType, - args.Mode, - ) + if args.Pseudo { + download, err = txapi.CreateResourceStringsAsyncDownload( + api, + resource, + args.ContentEncoding, + args.FileType, + args.Pseudo, + ) + } else { + download, err = txapi.CreateTranslationsAsyncDownload( + api, + resource, + languageCode, + args.ContentEncoding, + args.FileType, + args.Mode, + ) + } return err }, "Creating download job", diff --git a/internal/txlib/pull_test.go b/internal/txlib/pull_test.go index b9a3a1e..f9df558 100644 --- a/internal/txlib/pull_test.go +++ b/internal/txlib/pull_test.go @@ -47,7 +47,7 @@ func TestPullCommandResourceExists(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlAllLanguages) - testSimpleTranslationDownload(t, mockData) + testSimpleTranslationDownload(t, mockData, "false") testSimpleGet(t, mockData, translationDownloadUrl) assertFileContent(t, "aaa-el.json", "This is the content") @@ -90,7 +90,7 @@ func TestPullCommandFileExists(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlAllLanguages) - testSimpleTranslationDownload(t, mockData) + testSimpleTranslationDownload(t, mockData, "false") testSimpleGet(t, mockData, translationDownloadUrl) assertFileContent(t, "aaa-el.json", "This is the content") @@ -134,7 +134,7 @@ func TestPullCommandDownloadSource(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlSourceLanguage) - testSimpleSourceDownload(t, mockData) + testSimpleSourceDownload(t, mockData, "false") testSimpleGet(t, mockData, sourceDownloadUrl) assertFileContent(t, "aaa.json", "New source") @@ -234,7 +234,7 @@ func TestPullCommandProceedOnEqualTranslatedMinPerc(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlAllLanguages) - testSimpleTranslationDownload(t, mockData) + testSimpleTranslationDownload(t, mockData, "false") testSimpleGet(t, mockData, translationDownloadUrl) assertFileContent(t, "aaa-el.json", "This is the content") @@ -278,7 +278,7 @@ func TestPullCommandOverrides(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlAllLanguages) - testSimpleTranslationDownload(t, mockData) + testSimpleTranslationDownload(t, mockData, "false") testSimpleGet(t, mockData, translationDownloadUrl) assertFileContent(t, "custom_path.json", "This is the content") @@ -322,7 +322,7 @@ func TestPullCommandMultipleLangParameters(t *testing.T) { testSimpleGet(t, mockData, resourceUrl) testSimpleGet(t, mockData, projectUrl) testSimpleGet(t, mockData, statsUrlAllLanguages) - testSimpleTranslationDownload(t, mockData) + testSimpleTranslationDownload(t, mockData, "false") testSimpleGet(t, mockData, translationDownloadUrl) assertFileContent(t, "locale/el/aaa-el.json", "This is the content") @@ -547,6 +547,50 @@ func TestShouldSkipDueToStringPercentage(t *testing.T) { assert.Equal(t, result, true) } +func TestDownloadPseudoTranslations(t *testing.T) { + afterTest := beforeTest(t, []string{"el"}, nil) + defer afterTest() + + cfg := getStandardConfig() + cfg.Local.Resources[0].FileFilter = "locale//aaa-.json" + + ts := getNewTestServer("This is the content") + defer ts.Close() + + mockData := jsonapi.MockData{ + resourceUrl: getResourceEndpoint(), + projectUrl: getProjectEndpoint(), + statsUrlAllLanguages: getStatsEndpointAllLanguages(), + sourceDownloadsUrl: getSourceDownloadsEndpoint(), + sourceDownloadUrl: getDownloadEndpoint(ts.URL), + } + + api := jsonapi.GetTestConnection(mockData) + + arguments := PullCommandArguments{ + FileType: "default", + Mode: "default", + Force: true, + All: true, + ResourceIds: nil, + MinimumPercentage: -1, + Workers: 1, + Pseudo: true, + } + + err := PullCommand(cfg, &api, &arguments) + if err != nil { + t.Errorf("%s", err) + } + + assertFileContent(t, "locale/el/aaa-el.json", "This is the content") + testSimpleGet(t, mockData, resourceUrl) + testSimpleGet(t, mockData, projectUrl) + testSimpleGet(t, mockData, statsUrlAllLanguages) + testSimpleSourceDownload(t, mockData, "true") + testSimpleGet(t, mockData, sourceDownloadUrl) +} + func assertFileContent(t *testing.T, expectedPath, expectedContent string) { data, err := os.ReadFile(expectedPath) if err != nil { @@ -586,6 +630,7 @@ func getDownloadEndpoint(url string) *jsonapi.MockEndpoint { func testSimpleTranslationDownload( t *testing.T, mockData jsonapi.MockData, + pseudo string, ) { testSimplePost( t, @@ -597,13 +642,13 @@ func testSimpleTranslationDownload( "attributes": {"content_encoding": "", "file_type": "default", "mode": "default", - "pseudo": false}, + "pseudo": %s}, "relationships": { "language": {"data": {"type": "languages", "id": "l:el"}}, "resource": {"data": {"type": "resources", "id": "%s"}} } }}`, - resourceId, + pseudo, resourceId, ), ) } @@ -611,6 +656,7 @@ func testSimpleTranslationDownload( func testSimpleSourceDownload( t *testing.T, mockData jsonapi.MockData, + pseudo string, ) { testSimplePost( t, @@ -621,12 +667,12 @@ func testSimpleSourceDownload( "type": "resource_strings_async_downloads", "attributes": {"content_encoding": "", "file_type": "default", - "pseudo": false}, + "pseudo": %s}, "relationships": { "resource": {"data": {"type": "resources", "id": "%s"}} } }}`, - resourceId, + pseudo, resourceId, ), ) } diff --git a/pkg/txapi/resource_strings_async_downloads.go b/pkg/txapi/resource_strings_async_downloads.go index ac9b4e2..bacb192 100644 --- a/pkg/txapi/resource_strings_async_downloads.go +++ b/pkg/txapi/resource_strings_async_downloads.go @@ -17,6 +17,7 @@ func CreateResourceStringsAsyncDownload( resource *jsonapi.Resource, contentEncoding string, fileType string, + pseudo bool, ) (*jsonapi.Resource, error) { download := &jsonapi.Resource{ API: api, @@ -24,7 +25,7 @@ func CreateResourceStringsAsyncDownload( Attributes: map[string]interface{}{ "content_encoding": contentEncoding, "file_type": fileType, - "pseudo": false, + "pseudo": pseudo, }, } download.SetRelated("resource", resource)