Merge branch 'master' into alessio/BFD-3810__update-pipeline-observab…

…ility
CMSgov · Jan 30, 2025 · 9eeffe1 · 9eeffe1
2 parents f7620c7 + ded24f0
commit 9eeffe1
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 23 deletions.
diff --git a/apps/bfd-data-fda/pom.xml b/apps/bfd-data-fda/pom.xml
@@ -114,6 +114,7 @@
                             <mainClass>gov.cms.bfd.data.fda.utility.App</mainClass>
                             <arguments>
                                 <argument>${basedir}/target/fda-drug-csv-output</argument>
+                                <argument>${parent.version}</argument>
                             </arguments>
                         </configuration>
                     </execution>

diff --git a/apps/bfd-data-fda/src/main/java/gov/cms/bfd/data/fda/utility/App.java b/apps/bfd-data-fda/src/main/java/gov/cms/bfd/data/fda/utility/App.java
@@ -23,13 +23,14 @@ public class App {
    *     files
    */
   public static void main(String[] args) {
-    if (args.length < 1) {
-      throw new IllegalArgumentException("OUTPUT_DIR argument not specified for FDA NDC download.");
+    if (args.length < 2) {
+      throw new IllegalArgumentException(
+          "Invalid number of arguments supplied for FDA NDC download.");
     }
-    if (args.length > 1) {
+    if (args.length > 2) {
       throw new IllegalArgumentException("Invalid arguments supplied for FDA NDC download.");
     }
 
-    DataUtilityCommons.getFDADrugCodes(args[0], FDA_PRODUCTS_RESOURCE);
+    DataUtilityCommons.getFDADrugCodes(args[0], args[1], FDA_PRODUCTS_RESOURCE);
   }
 }
diff --git a/apps/bfd-data-fda/src/main/java/gov/cms/bfd/data/fda/utility/DataUtilityCommons.java b/apps/bfd-data-fda/src/main/java/gov/cms/bfd/data/fda/utility/DataUtilityCommons.java
@@ -3,12 +3,14 @@
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
-import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
+import java.net.HttpURLConnection;
+import java.net.URL;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CharsetEncoder;
@@ -17,12 +19,11 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.Comparator;
-import java.util.Objects;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
-import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,10 +38,11 @@ public class DataUtilityCommons {
    * Gets the fda drug codes from the fda file.
    *
    * @param outputDir the output directory.
+   * @param version The BFD version.
    * @param fdaFile the fda file.
    */
   @SuppressWarnings("java:S5443")
-  public static void getFDADrugCodes(String outputDir, String fdaFile)
+  public static void getFDADrugCodes(String outputDir, String version, String fdaFile)
       throws IllegalStateException {
     Path outputPath = Paths.get(outputDir);
     if (!Files.isDirectory(outputPath)) {
@@ -55,7 +57,7 @@ public static void getFDADrugCodes(String outputDir, String fdaFile)
       Path convertedNdcDataFile = outputPath.resolve(fdaFile);
 
       try {
-        buildProductsResource(convertedNdcDataFile, workingDir);
+        buildProductsResource(convertedNdcDataFile, workingDir, version);
       } finally {
         // Recursively delete the working dir.
         recursivelyDelete(workingDir);
@@ -71,27 +73,31 @@ public static void getFDADrugCodes(String outputDir, String fdaFile)
    *
    * @param convertedNdcDataFile the output file/resource to produce.
    * @param workingDir a directory that temporary/working files can be written to.
+   * @param version The BFD version.
    * @throws IOException (any errors encountered will be bubbled up).
    */
-  public static void buildProductsResource(Path convertedNdcDataFile, Path workingDir)
+  public static void buildProductsResource(
+      Path convertedNdcDataFile, Path workingDir, String version)
       throws IOException, IllegalStateException {
     // download FDA NDC file
     Path downloadedNdcZipFile =
         Paths.get(workingDir.resolve("ndctext.zip").toFile().getAbsolutePath());
+    URL ndctextZipUrl = new URL("https://www.accessdata.fda.gov/cder/ndctext.zip");
     if (!Files.isReadable(downloadedNdcZipFile)) {
-      ClassLoader classLoader = DataUtilityCommons.class.getClassLoader();
-      File ndcFile =
-          new File(Objects.requireNonNull(classLoader.getResource("ndctext.zip")).getFile());
-      FileUtils.copyFile(ndcFile, new File(downloadedNdcZipFile.toFile().getAbsolutePath()));
+      HttpURLConnection connection = (HttpURLConnection) ndctextZipUrl.openConnection();
+      connection.setRequestProperty(
+          "User-Agent", String.format("BFD/%s (Beneficiary FHIR Data Server)", version));
+      try (InputStream in = connection.getInputStream()) {
+        Files.copy(in, downloadedNdcZipFile, StandardCopyOption.REPLACE_EXISTING);
+      }
     }
-
     // unzip FDA NDC file
     unzip(downloadedNdcZipFile, workingDir);
     Path originalNdcDataFile = workingDir.resolve("product.txt");
     if (!Files.isReadable(originalNdcDataFile))
       originalNdcDataFile = workingDir.resolve("Product.txt");
     if (!Files.isReadable(originalNdcDataFile))
-      throw new IllegalStateException("Unable to locate product.txt in ndctext.zip");
+      throw new IllegalStateException("Unable to locate product.txt in " + ndctextZipUrl);
 
     // convert file format from cp1252 to utf8
     CharsetDecoder inDec =

diff --git a/apps/bfd-data-fda/src/main/resources/ndctext.zip b/apps/bfd-data-fda/src/main/resources/ndctext.zip
diff --git a/apps/bfd-data-fda/src/test/java/gov/cms/bfd/data/fda/utility/AppTest.java b/apps/bfd-data-fda/src/test/java/gov/cms/bfd/data/fda/utility/AppTest.java
@@ -19,7 +19,8 @@ public void fdappThrowsIllegalArgumentExceptionWhenNoArgumentsPassed() {
             () -> {
               App.main(new String[] {});
             });
-    assertEquals("OUTPUT_DIR argument not specified for FDA NDC download.", exception.getMessage());
+    assertEquals(
+        "Invalid number of arguments supplied for FDA NDC download.", exception.getMessage());
   }
 
   /** Return illegal argument exception when more than one arguments are passed. */
@@ -29,7 +30,7 @@ public void fdaAppThrowsIllegalArgumentExceptionWhenMoreThanOneArgumentsPassed()
         assertThrows(
             IllegalArgumentException.class,
             () -> {
-              App.main(new String[] {"Argument 1", "Argument 2"});
+              App.main(new String[] {"Argument 1", "Argument 2", "Argument 3"});
             });
     assertEquals("Invalid arguments supplied for FDA NDC download.", exception.getMessage());
   }
@@ -41,7 +42,7 @@ public void fdaAppThrowsIllegalStateExceptionWhenOneArgumentsPassedThatIsNotADir
         assertThrows(
             IllegalStateException.class,
             () -> {
-              App.main(new String[] {"Argument 1"});
+              App.main(new String[] {"Argument 1", "1.0.0"});
             });
     assertEquals("OUTPUT_DIR does not exist for FDA NDC download.", exception.getMessage());
   }
@@ -55,13 +56,13 @@ public void fdaAppPassesWithValidParameters() {
       String expectedFdaFile = App.FDA_PRODUCTS_RESOURCE;
 
       dataUtilityCommons
-          .when(() -> DataUtilityCommons.getFDADrugCodes(any(), any()))
+          .when(() -> DataUtilityCommons.getFDADrugCodes(any(), any(), any()))
           .thenAnswer((Answer<Void>) invocation -> null);
-      App.main(new String[] {outputDir});
+      App.main(new String[] {outputDir, "1.0.0"});
 
       // Verify that DataUtilityCommons.getFDADrugCodes was called with specific arguments
       dataUtilityCommons.verify(
-          () -> DataUtilityCommons.getFDADrugCodes(outputDir, expectedFdaFile));
+          () -> DataUtilityCommons.getFDADrugCodes(outputDir, "1.0.0", expectedFdaFile));
     }
   }
 }
diff --git a/apps/bfd-data-fda/src/test/java/gov/cms/bfd/data/fda/utility/DataUtilityCommonsTest.java b/apps/bfd-data-fda/src/test/java/gov/cms/bfd/data/fda/utility/DataUtilityCommonsTest.java
@@ -31,7 +31,7 @@ public void getFDADrugCodesThrowsExceptionWhenFileIsNotADirectory() {
               assertThrows(
                   IllegalStateException.class,
                   () -> {
-                    DataUtilityCommons.getFDADrugCodes(outputDir, any());
+                    DataUtilityCommons.getFDADrugCodes(outputDir, "1.0.0", any());
                   });
           assertEquals("OUTPUT_DIR does not exist for FDA NDC download.", exception.getMessage());
         }