Add github repo to markdown command to CLI tool

bin/cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
+const fs = require('fs')
 const gpt4 = require('gpt-tokenizer/cjs/model/gpt-4')
-const { CompletionService } = require('langxlang')
+const { CompletionService, tools } = require('langxlang')
 
 function countTokens (text) {
   return gpt4.encode(text).length
@@ -10,8 +11,11 @@ function raise (msg) {
   if (msg) console.error(msg)
   console.error('Usage: langxlang <command> ...args')
   console.error('Usage: langxlang count <tokenizer> <file>')
+  console.error('Usage: langxlang githubRepoToMarkdown <repo> <branch or ref> [output file]')
+  console.error('Usage (alias): langxlang repo2md <repo> <branch or ref> [output file]')
   console.error('Example: langxlang count gpt4 myfile.js')
   console.error('Example: langxlang count gemini1.5pro myfile.txt')
+  console.error('Example: langxlang githubRepoToMarkdown PrismarineJS/vec3 master vec3.md')
 }
 
 if (process.argv.length < 3) {
@@ -38,9 +42,19 @@ const commands = {
       console.error('Unknown tokenizer', tokenizer)
       process.exit(1)
     }
+  },
+  githubRepoToMarkdown (repo, branch, outFile = 'repo.md') {
+    const files = tools.collectGithubRepoFiles(repo, {
+      branch,
+      truncateLargeFiles: 16_000 // 16k
+    })
+    const md = tools.concatFilesToMarkdown(files)
+    fs.writeFileSync(outFile, md)
   }
 }
 
+commands.repo2md = commands.githubRepoToMarkdown
+
 const [, , command, ...args] = process.argv
 console.error(`command: ${command}`, args)
 commands[command](...args)