Fix shortlist... update to get cp311

jerinphilip · Aug 13, 2023 · 21b3352 · 21b3352
1 parent d560098
commit 21b3352
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 2 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -179,7 +179,7 @@ jobs:
               ${{github.workspace}}\ccache.exe -z # Print current cache stats
 
 
-            CIBW_BUILD: "cp{37,38,39,310,311}-*manylinux_x86_64 cp{37,38,39,310}-macosx_x86_64 cp3{8,9,10}-win_amd64 cp*macosx_arm64"
+            CIBW_BUILD: "cp{37,38,39,310,311}-*manylinux_x86_64 cp{37,38,39,310,311}-macosx_x86_64 cp3{8,9,10}-win_amd64 cp*macosx_arm64"
 
             CIBW_BEFORE_TEST_LINUX: |
               ccache -s # Print current ccache stats

diff --git a/src/translator/translation_model.cpp b/src/translator/translation_model.cpp
@@ -23,7 +23,29 @@ TranslationModel::TranslationModel(const Config &options, MemoryBundle &&memory
       vocabs_(options, std::move(memory_.vocabs)),
       textProcessor_(options, vocabs_, std::move(memory_.ssplitPrefixFile)),
       batchingPool_(options),
-      qualityEstimator_(createQualityEstimator(getQualityEstimatorModel(memory, options))) {}
+      qualityEstimator_(createQualityEstimator(getQualityEstimatorModel(memory, options))) {
+
+  // Try to load shortlist from memory-bundle. If not available, try to load from options_;
+
+  int srcIdx = 0, trgIdx = 1;
+  // vocabs_->sources().front() is invoked as we currently only support one source vocab
+  bool shared_vcb = (vocabs_.sources().front() == vocabs_.target());
+
+  if (memory_.shortlist.size() > 0 && memory_.shortlist.begin() != nullptr) {
+    bool check = options_->get<bool>("check-bytearray", false);
+    shortlistGenerator_ = New<data::BinaryShortlistGenerator>(memory_.shortlist.begin(), memory_.shortlist.size(),
+                                                              vocabs_.sources().front(), vocabs_.target(), srcIdx,
+                                                              trgIdx, shared_vcb, check);
+  } else if (options_->hasAndNotEmpty("shortlist")) {
+    // Changed to BinaryShortlistGenerator to enable loading binary shortlist file
+    // This class also supports text shortlist file
+    shortlistGenerator_ = New<data::BinaryShortlistGenerator>(options_, vocabs_.sources().front(), vocabs_.target(),
+                                                              srcIdx, trgIdx, shared_vcb);
+  } else {
+    // In this case, the loadpath does not load shortlist.
+    shortlistGenerator_ = nullptr;
+  }
+}
 
 void TranslationModel::loadBackend(MarianBackend &backend, Workspace &workspace) {
   auto &graph = backend.graph;