ARROW-4245: [Rust] Add Rustdoc header to source files

This commit adds a short description of each module

Author: Paul Kernfeld <[email protected]>

Closes apache#3448 from paulkernfeld/arrow-4245-add-rustdoc-headers and squashes the following commits:

9b8cc7f <Paul Kernfeld> Fix flaky test, improve rustdocs
3225839 <Paul Kernfeld> Summarize each module
32d73df <Paul Kernfeld> Remove example from arrow lib.rs
1d95b03 <Paul Kernfeld> ARROW-4245:  Add Rustdoc header to source files

rust/arrow/src/array.rs

@@ -15,7 +15,37 @@
 // specific language governing permissions and limitations
 // under the License.
 
-///! Array types
+//! Defines public types representing Apache Arrow arrays. Arrow's specification defines an array as
+//! "a sequence of values with known length all having the same type." For example, the type
+//! `Int16Array` represents an Apache Arrow array of 16-bit integers.
+//!
+//! ```
+//! extern crate arrow;
+//!
+//! use arrow::array::Int16Array;
+//!
+//! // Create a new builder with a capacity of 100
+//! let mut builder = Int16Array::builder(100);
+//!
+//! // Append a single primitive value
+//! builder.append_value(1).unwrap();
+//!
+//! // Append a null value
+//! builder.append_null().unwrap();
+//!
+//! // Append a slice of primitive values
+//! builder.append_slice(&[2, 3, 4]).unwrap();
+//!
+//! // Build the array
+//! let array = builder.finish();
+//!
+//! assert_eq!(5, array.len(), "The array has 5 values, counting the null value");
+//!
+//! assert_eq!(2, array.value(2), "Get the value with index 2");
+//!
+//! assert_eq!(array.value_slice(3, 2), &[3, 4], "Get slice of len 2 starting at idx 3")
+//! ```
+
 use std::any::Any;
 use std::convert::From;
 use std::io::Write;

rust/arrow/src/array_data.rs

@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Contains `ArrayData`, a generic representation of Arrow array data which encapsulates common
+//! attributes and operations for Arrow array.
+
 use std::sync::Arc;
 
 use crate::bitmap::Bitmap;

rust/arrow/src/array_ops.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Defines primitive computations on arrays
+//! Defines primitive computations on arrays, e.g. addition, equality, boolean logic.
 
 use std::ops::{Add, Div, Mul, Sub};
 

rust/arrow/src/bitmap.rs

@@ -15,6 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Defines a bitmap, which is used to track which values in an Arrow array are null. This is called
+//! a "validity bitmap" in the Arrow documentation.
+
 use super::buffer::Buffer;
 use crate::util::bit_util;
 

rust/arrow/src/buffer.rs

@@ -15,6 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! The main type in the module is `Buffer`, a contiguous immutable memory region of fixed size
+//! aligned at a 64-byte boundary. `MutableBuffer` is like `Buffer`, but it can be mutated and
+//! grown.
+
 use std::cmp;
 use std::io::{Error as IoError, ErrorKind, Result as IoResult, Write};
 use std::mem;

rust/arrow/src/csv/mod.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Transfer data between the Arrow memory format and CSV (comma-separated values).
+
 pub mod reader;
 
 pub use self::reader::Reader;

rust/arrow/src/error.rs

@@ -15,10 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Defines `ArrowError` for representing failures in various Arrow operations
 use std::error::Error;
 
 use csv as csv_crate;
 
+/// Many different operations in the `arrow` crate return this error type
 #[derive(Debug, Clone, PartialEq)]
 pub enum ArrowError {
     MemoryError(String),

rust/arrow/src/lib.rs

@@ -15,6 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! A native Rust implementation of [Apache Arrow](https://arrow.apache.org), a cross-language
+//! development platform for in-memory data.
+//!
+//! Currently the project is developed and tested against nightly Rust. To learn more about the
+//! status of Arrow in Rust, see `README.md`.
+
 #![feature(type_ascription)]
 #![feature(rustc_private)]
 #![feature(specialization)]

rust/arrow/src/memory.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Defines memory-related functions, currently mostly to make this library play nicely with C.
+
 use libc;
 use std::cmp;
 use std::mem;

rust/arrow/src/record_batch.rs

@@ -15,6 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! According to the [Arrow Metadata Specification](https://arrow.apache.org/docs/metadata.html):
+//!
+//! > A record batch is a collection of top-level named, equal length Arrow arrays (or vectors). If
+//! > one of the arrays contains nested data, its child arrays are not required to be the same
+//! > length as the top-level arrays.
+
 use std::sync::Arc;
 
 use crate::array::*;

rust/arrow/src/tensor.rs

@@ -15,7 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Arrow Tensor Type
+//! Arrow Tensor Type, defined in
+//! [`format/Tensor.fbs`](https://github.com/apache/arrow/blob/master/format/Tensor.fbs).
+
 use std::marker::PhantomData;
 use std::mem;
 

rust/arrow/src/util/bit_util.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Utils for working with bits
+
 static BIT_MASK: [u8; 8] = [1, 2, 4, 8, 16, 32, 64, 128];
 
 static POPCOUNT_TABLE: [u8; 256] = [

rust/arrow/src/util/test_util.rs

@@ -15,6 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Utils to make testing easier
+
 use rand::{thread_rng, Rng};
 
 /// Returns a vector of size `n`, filled with randomly generated bytes.