Merge branch 'master' of https://github.com/vgteam/vg

.gitlab-ci.yml

@@ -11,7 +11,7 @@ before_script:
   - sudo apt-get -q -y update
   # Make sure we have some curl stuff for pycurl which we need for some Python stuff
   # And the CI report upload needs uuidgen from uuid-runtime
-  - sudo apt-get -q -y install --no-upgrade docker.io python3-pip python3-virtualenv libcurl4-gnutls-dev python-dev npm nodejs node-gyp uuid-runtime libgnutls28-dev doxygen libzstd-dev
+  - sudo apt-get -q -y install --no-upgrade docker.io python3-pip python3-virtualenv libcurl4-gnutls-dev python-dev npm nodejs node-gyp uuid-runtime libgnutls28-dev doxygen libzstd-dev bcftools
   - which junit-merge || sudo npm install -g junit-merge
   # Configure Docker to use a mirror for Docker Hub and restart the daemon
   - |

Makefile

@@ -215,6 +215,9 @@ ifeq ($(shell uname -s),Darwin)
     # We don't actually do any static linking on Mac, so we leave this empty.
     START_STATIC =
     END_STATIC =
+
+    # We need to use special flags to let us rename libraries
+    LD_RENAMEABLE_FLAGS = -Wl,-headerpad -Wl,-headerpad_max_install_names
 else
     # We are not running on OS X
     $(info OS is Linux)
@@ -252,7 +255,8 @@ else
     # Note that END_STATIC is only safe to use in a mostly-dynamic build, and has to appear or we will try to statically link secret trailing libraries.
     END_STATIC = -Wl,-Bdynamic
 
-
+    # We don't need any flags because we don't need to rename libraries with install_name_tool
+    LD_RENAMEABLE_FLAGS =
 endif
 
 # Set the C++ standard we are using
@@ -656,7 +660,7 @@ ifeq ($(shell uname -s),Darwin)
 endif
 
 $(LIB_DIR)/libdeflate.a: $(LIBDEFLATE_DIR)/*.h $(LIBDEFLATE_DIR)/lib/*.h $(LIBDEFLATE_DIR)/lib/*/*.h $(LIBDEFLATE_DIR)/lib/*.c $(LIBDEFLATE_DIR)/lib/*/*.c
-	+. ./source_me.sh && cd $(LIBDEFLATE_DIR) && V=1 $(MAKE) $(FILTER) && cp libdeflate.a $(CWD)/$(LIB_DIR) && cp libdeflate.h $(CWD)/$(INC_DIR)
+	+. ./source_me.sh && cd $(LIBDEFLATE_DIR) && V=1 LDFLAGS="$(LDFLAGS) $(LD_RENAMEABLE_FLAGS)" $(MAKE) $(FILTER) && cp libdeflate.a $(CWD)/$(LIB_DIR) && cp libdeflate.h $(CWD)/$(INC_DIR)
 
 # We build htslib after libdeflate so it can use libdeflate.
 # We need to do some wizardry to get it to pick up the right build and target system types on modern autotools.

README.md

@@ -55,27 +55,41 @@ The easiest way to get vg is to download one of our release builds for Linux. We
 
 If you don't want to or can't use a pre-built release of vg, or if you want to become a vg developer, you can build it from source instead.
 
+#### Linux: Clone VG
+
 First, obtain the repo and its submodules:
 
     git clone --recursive https://github.com/vgteam/vg.git
     cd vg
+
+#### Linux: Install Dependencies
 
 Then, install VG's dependencies. You'll need the protobuf and jansson development libraries installed, and to run the tests you will need:
-    * `jq`, `bc`, `rs`, and `parallel`
-    * `hexdump` and `column` from `bsdmainutils`
-    * [`npm` for testing documentation examples](https://github.com/anko/txm)).
+* `jq`, `bc`, `rs`, and `parallel`
+* `hexdump` and `column` from `bsdmainutils`
+* [`npm` for testing documentation examples](https://github.com/anko/txm)).
+
 On Ubuntu, you should be able to do:
 
     make get-deps
+
+If you get complaints that `sudo` is not found, install it:
+
+    apt update
+    apt install sudo
+
+If you get a bunch of errors like `E: Unable to locate package build-essential`, make sure your package index files are up to date by running:
+
+    sudo apt update
 
-On other distros, you will need to perform the equivalent of:
+On other distros, or if you do not have root access, you will need to perform the equivalent of:
 
     sudo apt-get install build-essential git cmake pkg-config libncurses-dev libbz2-dev  \
                          protobuf-compiler libprotoc-dev libprotobuf-dev libjansson-dev \
                          automake gettext autopoint libtool jq bsdmainutils bc rs parallel \
                          npm curl unzip redland-utils librdf-dev bison flex gawk lzma-dev \
                          liblzma-dev liblz4-dev libffi-dev libcairo-dev libboost-all-dev \
-                         libzstd-devel pybind11-dev python3-pybind11
+                         libzstd-dev pybind11-dev python3-pybind11
                          
 Note that **Ubuntu 16.04** does not ship a sufficiently new Protobuf; vg requires **Protobuf 3** which will have to be manually installed.
 
@@ -85,22 +99,47 @@ Other libraries may be required. Please report any build difficulties.
 
 Note that a 64-bit OS is required. Ubuntu 20.04 should work.
 
-When you are ready, build with `. ./source_me.sh && make`, and run with `./bin/vg`.
+#### Linux: Build
+
+When you are ready, build with `. ./source_me.sh && make`. You can use `make -j16` to run 16 build threads at a time, which greatly accelerates the process. If you have more CPU cores, you can use higher numbers.
 
 Note that vg can take anywhere from 10 minutes to more than an hour to compile depending on your machine and the number of threads used. 
 
 You can also produce a static binary with `make static`, assuming you have static versions of all the dependencies installed on your system.
 
+#### Linux: Run
+
+Once vg is built, the binary will be at `bin/vg` inside the vg repository directory. You can run it with:
+
+```
+./bin/vg
+```
+
+You can also add its directory to your `PATH` enviornment variable, so that you can invoke `vg` from any directory. To do that on Bash, use this command from the vg repository directory:
+
+```
+echo 'export PATH="${PATH}:'"$(pwd)"'/bin"' >>~/.bashrc
+```
+
+Then close your terminal and open a new one. Run `vg` to make sure it worked.
+
+If it did not work, make sure that you have a `.bash_profile` file in your home directory that will run your `.bashrc`:
+```
+if [ -f ~/.bashrc ]; then
+   source ~/.bashrc
+fi
+```
+
 ### Building on MacOS
 
-#### Clone VG
+#### Mac: Clone VG
 
 The first step is to clone the vg repository:
 
     git clone --recursive https://github.com/vgteam/vg.git
     cd vg
 
-#### Install Dependencies
+#### Mac: Install Dependencies
 
 VG depends on a number of packages being installed on the system where it is being built. Dependencies can be installed using either [MacPorts](https://www.macports.org/install.php) or [Homebrew](http://brew.sh/).
 
@@ -118,17 +157,35 @@ Homebrew provides another package management solution for OSX, and may be prefer
     # Install all the dependencies in the Brewfile
     brew bundle
 
-#### Build
+#### Mac: Build
 
 With dependencies installed, VG can now be built:
 
     . ./source_me.sh && make
+
+As with Linux, you can add `-j16` or other numbers at the end to run multiple build tasks at once, if your computer can handle them.
 
 **Note that static binaries cannot yet be built for Mac.**
 
 Our team has successfully built vg on Mac with GCC versions 4.9, 5.3, 6, 7, and 7.3, as well as Clang 9.0.
 
-#### Migrating to ARM Macs
+#### Mac: Run
+
+Once vg is built, the binary will be at `bin/vg` inside the vg repository directory. You can run it with:
+
+```
+./bin/vg
+```
+
+You can also add its directory to your `PATH` enviornment variable, so that you can invoke `vg` from any directory. To do that on the default `zsh` Mac shell, use this command from the vg repository directory:
+
+```
+echo 'export PATH="${PATH}:'"$(pwd)"'/bin"' >>~/.zshrc
+```
+
+Then close your terminal and open a new one. Run `vg` to make sure it worked.
+
+##### Migrate a VG installation from x86 to ARM
 
 The Mac platform is moving to ARM, with Apple's M1, M1 Pro, M1 Max, and subsequent chip designs. The vg codebase supports ARM on Mac as well as on Linux. **The normal installation instructions work on a factory-fresh ARM Mac**.
 
@@ -396,14 +453,6 @@ vg index hla.vg -x hla.xg
 vg deconstruct hla.xg -e -p "gi|568815592:29791752-29792749" > hla_variants.vcf
 ```
 
-Variants can also be inferred strictly from topology by not using `-e`, though unlike the above example, cycles are not supported.  "Deconstruct" the VCF variants that were used to construct the graph. The output will be similar but identical to `small/x.vcf.gz` as `vg construct` can add edges between adjacent alts and/or do some normalization:
-
-<!-- !test check Deconstruct from construct -->
-```sh
-# using the same graph from the `map` example
-vg deconstruct x.xg -p x > x.vcf
-```
-
 Haplotype paths from `.gbz` or `.gbwt` indexes input can be considered using `-z` and `-g', respectively.
 
 As with `vg call`, it is best to compute snarls separately and pass them in with `-r` when working with large graphs.

ontology/vg.html

@@ -688,7 +688,7 @@ <h2>
 <td><a href="http://www.w3.org/2000/01/rdf-schema#comment">
 					rdfs:comment</a></td>
 <td>
-						"A step along a path in the variant graph. A series of steps along a path represent an assembled sequence that was originally inserted into the the variant graph. A step points to a :Node or the reverse complement of a node and has a rank (step number)."
+						"A step along a path in the variant graph. A series of steps along a path represent an assembled sequence that was originally inserted into the variant graph. A step points to a :Node or the reverse complement of a node and has a rank (step number)."
 						<sup><a href="http://www.w3.org/1999/02/22-rdf-syntax-ns#http://www.w3.org/2001/XMLSchema#string">
 				xsd:string</a></sup>
 </td>

ontology/vg.ttl

@@ -31,7 +31,7 @@
 .
 :Step
   rdf:type owl:Class ;
-  rdfs:comment "A step along a path in the variant graph. A series of steps along a path represent an assembled sequence that was originally inserted into the the variant graph. A step points to a :Node or the reverse complement of a node and has a rank (step number)."^^xsd:string ;
+  rdfs:comment "A step along a path in the variant graph. A series of steps along a path represent an assembled sequence that was originally inserted into the variant graph. A step points to a :Node or the reverse complement of a node and has a rank (step number)."^^xsd:string ;
   rdfs:label "Step"^^xsd:string ;
   rdfs:subClassOf owl:Thing ;
 .