Migrate CTC decoder code (pytorch#2580)

Summary:
This commit gets rid of our copy of CTC decoder code and
replace it with upstream Flashlight-Text repo.

Pull Request resolved: pytorch#2580

Reviewed By: carolineechen

Differential Revision: D38244906

Pulled By: mthrok

fbshipit-source-id: d274240fc67675552d19ff35e9a363b9b9048721

.gitmodules

@@ -3,5 +3,8 @@
 	url = https://github.com/kaldi-asr/kaldi
 	ignore = dirty
 [submodule "third_party/kenlm/submodule"]
-	path = third_party/kenlm/submodule
+	path = third_party/kenlm/kenlm
 	url = https://github.com/kpu/kenlm
+[submodule "flashlight-text"]
+	path = third_party/flashlight-text/submodule
+	url = https://github.com/flashlight/text

third_party/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
 
 ################################################################################
 # sox
@@ -22,4 +22,5 @@ if (BUILD_CTC_DECODER)
   add_subdirectory(bzip2)
   add_subdirectory(lzma)
   add_subdirectory(kenlm)
+  add_subdirectory(flashlight-text)
 endif()

third_party/flashlight-text/CMakeLists.txt

@@ -0,0 +1,59 @@
+# Custom CMakeLists for building flashlight-text decoder
+#
+# The main difference from upstream native CMakeLists from flashlight-text.
+#
+# 1. Build compression libraries statically and make KenLM self-contained
+# 2. Build KenLM without Boost by compiling only what is used by flashlight-text
+# 3. Build KenLM and flashlight-text in one go (not required, but nice-to-have feature)
+# 4. Tweak the location of bindings so that its easier for TorchAudio build process to pick up.
+#    (the upstream CMakeLists.txt does not install them in the same location as libflashlight-text)
+# 5. Tweak the name of bindings. (remove suffix like cpython-37m-darwin)
+
+set(CMAKE_CXX_VISIBILITY_PRESET default)
+
+set(
+  libflashlight_src
+  submodule/flashlight/lib/text/decoder/Utils.cpp
+  submodule/flashlight/lib/text/decoder/lm/KenLM.cpp
+  submodule/flashlight/lib/text/decoder/lm/ZeroLM.cpp
+  submodule/flashlight/lib/text/decoder/lm/ConvLM.cpp
+  submodule/flashlight/lib/text/decoder/LexiconDecoder.cpp
+  submodule/flashlight/lib/text/decoder/LexiconFreeDecoder.cpp
+  submodule/flashlight/lib/text/decoder/LexiconFreeSeq2SeqDecoder.cpp
+  submodule/flashlight/lib/text/decoder/LexiconSeq2SeqDecoder.cpp
+  submodule/flashlight/lib/text/decoder/Trie.cpp
+  submodule/flashlight/lib/text/String.cpp
+  submodule/flashlight/lib/text/dictionary/Utils.cpp
+  submodule/flashlight/lib/text/dictionary/Dictionary.cpp
+  )
+
+torchaudio_library(
+  libflashlight-text
+  "${libflashlight_src}"
+  submodule
+  ""
+  FL_TEXT_USE_KENLM
+  )
+
+# TODO: update torchaudio_library to handle private links
+target_link_libraries(
+  libflashlight-text
+  PRIVATE
+  kenlm)
+
+if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
+  torchaudio_extension(
+    flashlight_lib_text_dictionary
+    submodule/bindings/python/flashlight/lib/text/_dictionary.cpp
+    submodule
+    libflashlight-text
+    ""
+    )
+  torchaudio_extension(
+    flashlight_lib_text_decoder
+    submodule/bindings/python/flashlight/lib/text/_decoder.cpp
+    submodule
+    libflashlight-text
+    FL_TEXT_USE_KENLM
+    )
+endif()

third_party/kenlm/CMakeLists.txt

@@ -1,45 +1,45 @@
 set(
   KENLM_UTIL_SOURCES
-  submodule/util/bit_packing.cc
-  submodule/util/double-conversion/bignum.cc
-  submodule/util/double-conversion/bignum-dtoa.cc
-  submodule/util/double-conversion/cached-powers.cc
-  submodule/util/double-conversion/diy-fp.cc
-  submodule/util/double-conversion/double-conversion.cc
-  submodule/util/double-conversion/fast-dtoa.cc
-  submodule/util/double-conversion/fixed-dtoa.cc
-  submodule/util/double-conversion/strtod.cc
-  submodule/util/ersatz_progress.cc
-  submodule/util/exception.cc
-  submodule/util/file.cc
-  submodule/util/file_piece.cc
-  submodule/util/float_to_string.cc
-  submodule/util/integer_to_string.cc
-  submodule/util/mmap.cc
-  submodule/util/murmur_hash.cc
-  submodule/util/pool.cc
-  submodule/util/read_compressed.cc
-  submodule/util/scoped.cc
-  submodule/util/spaces.cc
-  submodule/util/string_piece.cc
+  kenlm/util/bit_packing.cc
+  kenlm/util/double-conversion/bignum.cc
+  kenlm/util/double-conversion/bignum-dtoa.cc
+  kenlm/util/double-conversion/cached-powers.cc
+  kenlm/util/double-conversion/diy-fp.cc
+  kenlm/util/double-conversion/double-conversion.cc
+  kenlm/util/double-conversion/fast-dtoa.cc
+  kenlm/util/double-conversion/fixed-dtoa.cc
+  kenlm/util/double-conversion/strtod.cc
+  kenlm/util/ersatz_progress.cc
+  kenlm/util/exception.cc
+  kenlm/util/file.cc
+  kenlm/util/file_piece.cc
+  kenlm/util/float_to_string.cc
+  kenlm/util/integer_to_string.cc
+  kenlm/util/mmap.cc
+  kenlm/util/murmur_hash.cc
+  kenlm/util/pool.cc
+  kenlm/util/read_compressed.cc
+  kenlm/util/scoped.cc
+  kenlm/util/spaces.cc
+  kenlm/util/string_piece.cc
   )
 
 set(
   KENLM_SOURCES
-  submodule/lm/bhiksha.cc
-  submodule/lm/binary_format.cc
-  submodule/lm/config.cc
-  submodule/lm/lm_exception.cc
-  submodule/lm/model.cc
-  submodule/lm/quantize.cc
-  submodule/lm/read_arpa.cc
-  submodule/lm/search_hashed.cc
-  submodule/lm/search_trie.cc
-  submodule/lm/trie.cc
-  submodule/lm/trie_sort.cc
-  submodule/lm/value_build.cc
-  submodule/lm/virtual_interface.cc
-  submodule/lm/vocab.cc
+  kenlm/lm/bhiksha.cc
+  kenlm/lm/binary_format.cc
+  kenlm/lm/config.cc
+  kenlm/lm/lm_exception.cc
+  kenlm/lm/model.cc
+  kenlm/lm/quantize.cc
+  kenlm/lm/read_arpa.cc
+  kenlm/lm/search_hashed.cc
+  kenlm/lm/search_trie.cc
+  kenlm/lm/trie.cc
+  kenlm/lm/trie_sort.cc
+  kenlm/lm/value_build.cc
+  kenlm/lm/virtual_interface.cc
+  kenlm/lm/vocab.cc
   )
 
 add_library(
@@ -51,8 +51,7 @@ add_library(
 target_include_directories(
   kenlm
   BEFORE
-  PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../install/include"
-  PUBLIC submodule
+  PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}"
   )
 
 target_compile_definitions(
@@ -65,6 +64,7 @@ target_compile_definitions(
 
 target_link_libraries(
   kenlm
+  PRIVATE
   zlib
   bzip2
   lzma

tools/setup_helpers/extension.py

@@ -51,8 +51,9 @@ def get_ext_modules():
     if _BUILD_CTC_DECODER:
         modules.extend(
             [
-                Extension(name="torchaudio.lib.libtorchaudio_decoder", sources=[]),
-                Extension(name="torchaudio._torchaudio_decoder", sources=[]),
+                Extension(name="torchaudio.lib.libflashlight-text", sources=[]),
+                Extension(name="torchaudio.flashlight_lib_text_decoder", sources=[]),
+                Extension(name="torchaudio.flashlight_lib_text_dictionary", sources=[]),
             ]
         )
     if _USE_FFMPEG:

torchaudio/csrc/CMakeLists.txt

@@ -119,6 +119,7 @@ endif()
 #------------------------------------------------------------------------------#
 # END OF CUSTOMIZATION LOGICS
 #------------------------------------------------------------------------------#
+
 torchaudio_library(
   libtorchaudio
   "${LIBTORCHAUDIO_SOURCES}"
@@ -127,38 +128,6 @@ torchaudio_library(
   "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
   )
 
-
-################################################################################
-# libtorchaudio_decoder.so
-################################################################################
-if (BUILD_CTC_DECODER)
-  set(
-    LIBTORCHAUDIO_DECODER_SOURCES
-    decoder/src/decoder/LexiconDecoder.cpp
-    decoder/src/decoder/LexiconFreeDecoder.cpp
-    decoder/src/decoder/Trie.cpp
-    decoder/src/decoder/Utils.cpp
-    decoder/src/decoder/lm/KenLM.cpp
-    decoder/src/decoder/lm/ZeroLM.cpp
-    decoder/src/dictionary/Dictionary.cpp
-    decoder/src/dictionary/String.cpp
-    decoder/src/dictionary/System.cpp
-    decoder/src/dictionary/Utils.cpp
-    )
-  set(
-    LIBTORCHAUDIO_DECODER_DEFINITIONS
-    BUILD_CTC_DECODER
-    )
-  torchaudio_library(
-    libtorchaudio_decoder
-    "${LIBTORCHAUDIO_DECODER_SOURCES}"
-    "${PROJECT_SOURCE_DIR}"
-    "torch;kenlm"
-    "${LIBTORCHAUDIO_COMPILE_DEFINITIONS};${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
-    )
-endif()
-
-# TODO: Add libtorchaudio_decoder
 if (APPLE)
   set(TORCHAUDIO_LIBRARY libtorchaudio CACHE INTERNAL "")
 else()
@@ -224,15 +193,6 @@ if (BUILD_TORCHAUDIO_PYTHON_EXTENSION)
     libtorchaudio
     "${LIBTORCHAUDIO_COMPILE_DEFINITIONS}"
     )
-  if(BUILD_CTC_DECODER)
-    torchaudio_extension(
-      _torchaudio_decoder
-      decoder/bindings/pybind.cpp
-      ""
-      "libtorchaudio_decoder"
-      "${LIBTORCHAUDIO_DECODER_DEFINITIONS}"
-      )
-  endif()
   if(USE_FFMPEG)
     set(
       FFMPEG_EXTENSION_SOURCES