Skip to content

Commit

Permalink
Merge branch 'bc/hash-algo'
Browse files Browse the repository at this point in the history 
More abstraction of hash function from the codepath.

* bc/hash-algo:
  hash: update obsolete reference to SHA1_HEADER
  bulk-checkin: abstract SHA-1 usage
  csum-file: abstract uses of SHA-1
  csum-file: rename sha1file to hashfile
  read-cache: abstract away uses of SHA-1
  pack-write: switch various SHA-1 values to abstract forms
  pack-check: convert various uses of SHA-1 to abstract forms
  fast-import: switch various uses of SHA-1 to the_hash_algo
  sha1_file: switch uses of SHA-1 to the_hash_algo
  builtin/unpack-objects: switch uses of SHA-1 to the_hash_algo
  builtin/index-pack: improve hash function abstraction
  hash: create union for hash context allocation
  hash: move SHA-1 macros to hash.h
  • Loading branch information
@gitster
gitster committed Feb 15, 2018
2 parents 0dbd562 + b212c0c commit 0fd90da
Show file tree
Hide file tree
Showing 15 changed files with 348 additions and 344 deletions.
114 changes: 57 additions & 57 deletions builtin/index-pack.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ static unsigned int input_offset, input_len;
static off_t consumed_bytes;
static off_t max_input_size;
static unsigned deepest_delta;
static git_SHA_CTX input_ctx;
static git_hash_ctx input_ctx;
static uint32_t input_crc32;
static int input_fd, output_fd;
static const char *curr_pack;
Expand Down Expand Up @@ -253,7 +253,7 @@ static void flush(void)
if (input_offset) {
if (output_fd >= 0)
write_or_die(output_fd, input_buffer, input_offset);
git_SHA1_Update(&input_ctx, input_buffer, input_offset);
the_hash_algo->update_fn(&input_ctx, input_buffer, input_offset);
memmove(input_buffer, input_buffer + input_offset, input_len);
input_offset = 0;
}
Expand Down Expand Up @@ -326,7 +326,7 @@ static const char *open_pack_file(const char *pack_name)
output_fd = -1;
nothread_data.pack_fd = input_fd;
}
git_SHA1_Init(&input_ctx);
the_hash_algo->init_fn(&input_ctx);
return pack_name;
}

Expand Down Expand Up @@ -437,22 +437,22 @@ static int is_delta_type(enum object_type type)
}

static void *unpack_entry_data(off_t offset, unsigned long size,
enum object_type type, unsigned char *sha1)
enum object_type type, struct object_id *oid)
{
static char fixed_buf[8192];
int status;
git_zstream stream;
void *buf;
git_SHA_CTX c;
git_hash_ctx c;
char hdr[32];
int hdrlen;

if (!is_delta_type(type)) {
hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %lu", typename(type), size) + 1;
git_SHA1_Init(&c);
git_SHA1_Update(&c, hdr, hdrlen);
the_hash_algo->init_fn(&c);
the_hash_algo->update_fn(&c, hdr, hdrlen);
} else
sha1 = NULL;
oid = NULL;
if (type == OBJ_BLOB && size > big_file_threshold)
buf = fixed_buf;
else
Expand All @@ -469,8 +469,8 @@ static void *unpack_entry_data(off_t offset, unsigned long size,
stream.avail_in = input_len;
status = git_inflate(&stream, 0);
use(input_len - stream.avail_in);
if (sha1)
git_SHA1_Update(&c, last_out, stream.next_out - last_out);
if (oid)
the_hash_algo->update_fn(&c, last_out, stream.next_out - last_out);
if (buf == fixed_buf) {
stream.next_out = buf;
stream.avail_out = sizeof(fixed_buf);
Expand All @@ -479,15 +479,15 @@ static void *unpack_entry_data(off_t offset, unsigned long size,
if (stream.total_out != size || status != Z_STREAM_END)
bad_object(offset, _("inflate returned %d"), status);
git_inflate_end(&stream);
if (sha1)
git_SHA1_Final(sha1, &c);
if (oid)
the_hash_algo->final_fn(oid->hash, &c);
return buf == fixed_buf ? NULL : buf;
}

static void *unpack_raw_entry(struct object_entry *obj,
off_t *ofs_offset,
unsigned char *ref_sha1,
unsigned char *sha1)
struct object_id *ref_oid,
struct object_id *oid)
{
unsigned char *p;
unsigned long size, c;
Expand Down Expand Up @@ -515,8 +515,8 @@ static void *unpack_raw_entry(struct object_entry *obj,

switch (obj->type) {
case OBJ_REF_DELTA:
hashcpy(ref_sha1, fill(20));
use(20);
hashcpy(ref_oid->hash, fill(the_hash_algo->rawsz));
use(the_hash_algo->rawsz);
break;
case OBJ_OFS_DELTA:
p = fill(1);
Expand Down Expand Up @@ -546,7 +546,7 @@ static void *unpack_raw_entry(struct object_entry *obj,
}
obj->hdr_size = consumed_bytes - obj->idx.offset;

data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, sha1);
data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, oid);
obj->idx.crc32 = input_crc32;
return data;
}
Expand Down Expand Up @@ -1118,11 +1118,11 @@ static void *threaded_second_pass(void *data)
* - calculate SHA1 of all non-delta objects;
* - remember base (SHA1 or offset) for all deltas.
*/
static void parse_pack_objects(unsigned char *sha1)
static void parse_pack_objects(unsigned char *hash)
{
int i, nr_delays = 0;
struct ofs_delta_entry *ofs_delta = ofs_deltas;
unsigned char ref_delta_sha1[20];
struct object_id ref_delta_oid;
struct stat st;

if (verbose)
Expand All @@ -1132,16 +1132,16 @@ static void parse_pack_objects(unsigned char *sha1)
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
void *data = unpack_raw_entry(obj, &ofs_delta->offset,
ref_delta_sha1,
obj->idx.oid.hash);
&ref_delta_oid,
&obj->idx.oid);
obj->real_type = obj->type;
if (obj->type == OBJ_OFS_DELTA) {
nr_ofs_deltas++;
ofs_delta->obj_no = i;
ofs_delta++;
} else if (obj->type == OBJ_REF_DELTA) {
ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc);
hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_sha1);
hashcpy(ref_deltas[nr_ref_deltas].sha1, ref_delta_oid.hash);
ref_deltas[nr_ref_deltas].obj_no = i;
nr_ref_deltas++;
} else if (!data) {
Expand All @@ -1159,10 +1159,10 @@ static void parse_pack_objects(unsigned char *sha1)

/* Check pack integrity */
flush();
git_SHA1_Final(sha1, &input_ctx);
if (hashcmp(fill(20), sha1))
the_hash_algo->final_fn(hash, &input_ctx);
if (hashcmp(fill(the_hash_algo->rawsz), hash))
die(_("pack is corrupted (SHA1 mismatch)"));
use(20);
use(the_hash_algo->rawsz);

/* If input_fd is a file, we should have reached its end now. */
if (fstat(input_fd, &st))
Expand Down Expand Up @@ -1238,21 +1238,21 @@ static void resolve_deltas(void)
/*
* Third pass:
* - append objects to convert thin pack to full pack if required
* - write the final 20-byte SHA-1
* - write the final pack hash
*/
static void fix_unresolved_deltas(struct sha1file *f);
static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_sha1)
static void fix_unresolved_deltas(struct hashfile *f);
static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_hash)
{
if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) {
stop_progress(&progress);
/* Flush remaining pack final 20-byte SHA1. */
/* Flush remaining pack final hash. */
flush();
return;
}

if (fix_thin_pack) {
struct sha1file *f;
unsigned char read_sha1[20], tail_sha1[20];
struct hashfile *f;
unsigned char read_hash[GIT_MAX_RAWSZ], tail_hash[GIT_MAX_RAWSZ];
struct strbuf msg = STRBUF_INIT;
int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas;
int nr_objects_initial = nr_objects;
Expand All @@ -1261,20 +1261,20 @@ static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned cha
REALLOC_ARRAY(objects, nr_objects + nr_unresolved + 1);
memset(objects + nr_objects + 1, 0,
nr_unresolved * sizeof(*objects));
f = sha1fd(output_fd, curr_pack);
f = hashfd(output_fd, curr_pack);
fix_unresolved_deltas(f);
strbuf_addf(&msg, Q_("completed with %d local object",
"completed with %d local objects",
nr_objects - nr_objects_initial),
nr_objects - nr_objects_initial);
stop_progress_msg(&progress, msg.buf);
strbuf_release(&msg);
sha1close(f, tail_sha1, 0);
hashcpy(read_sha1, pack_sha1);
fixup_pack_header_footer(output_fd, pack_sha1,
hashclose(f, tail_hash, 0);
hashcpy(read_hash, pack_hash);
fixup_pack_header_footer(output_fd, pack_hash,
curr_pack, nr_objects,
read_sha1, consumed_bytes-20);
if (hashcmp(read_sha1, tail_sha1) != 0)
read_hash, consumed_bytes-the_hash_algo->rawsz);
if (hashcmp(read_hash, tail_hash) != 0)
die(_("Unexpected tail checksum for %s "
"(disk corruption?)"), curr_pack);
}
Expand All @@ -1285,7 +1285,7 @@ static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned cha
nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas);
}

static int write_compressed(struct sha1file *f, void *in, unsigned int size)
static int write_compressed(struct hashfile *f, void *in, unsigned int size)
{
git_zstream stream;
int status;
Expand All @@ -1299,7 +1299,7 @@ static int write_compressed(struct sha1file *f, void *in, unsigned int size)
stream.next_out = outbuf;
stream.avail_out = sizeof(outbuf);
status = git_deflate(&stream, Z_FINISH);
sha1write(f, outbuf, sizeof(outbuf) - stream.avail_out);
hashwrite(f, outbuf, sizeof(outbuf) - stream.avail_out);
} while (status == Z_OK);

if (status != Z_STREAM_END)
Expand All @@ -1309,7 +1309,7 @@ static int write_compressed(struct sha1file *f, void *in, unsigned int size)
return size;
}

static struct object_entry *append_obj_to_pack(struct sha1file *f,
static struct object_entry *append_obj_to_pack(struct hashfile *f,
const unsigned char *sha1, void *buf,
unsigned long size, enum object_type type)
{
Expand All @@ -1326,15 +1326,15 @@ static struct object_entry *append_obj_to_pack(struct sha1file *f,
}
header[n++] = c;
crc32_begin(f);
sha1write(f, header, n);
hashwrite(f, header, n);
obj[0].size = size;
obj[0].hdr_size = n;
obj[0].type = type;
obj[0].real_type = type;
obj[1].idx.offset = obj[0].idx.offset + n;
obj[1].idx.offset += write_compressed(f, buf, size);
obj[0].idx.crc32 = crc32_end(f);
sha1flush(f);
hashflush(f);
hashcpy(obj->idx.oid.hash, sha1);
return obj;
}
Expand All @@ -1346,7 +1346,7 @@ static int delta_pos_compare(const void *_a, const void *_b)
return a->obj_no - b->obj_no;
}

static void fix_unresolved_deltas(struct sha1file *f)
static void fix_unresolved_deltas(struct hashfile *f)
{
struct ref_delta_entry **sorted_by_pos;
int i;
Expand Down Expand Up @@ -1402,7 +1402,7 @@ static const char *derive_filename(const char *pack_name, const char *suffix,
}

static void write_special_file(const char *suffix, const char *msg,
const char *pack_name, const unsigned char *sha1,
const char *pack_name, const unsigned char *hash,
const char **report)
{
struct strbuf name_buf = STRBUF_INIT;
Expand All @@ -1413,7 +1413,7 @@ static void write_special_file(const char *suffix, const char *msg,
if (pack_name)
filename = derive_filename(pack_name, suffix, &name_buf);
else
filename = odb_pack_name(&name_buf, sha1, suffix);
filename = odb_pack_name(&name_buf, hash, suffix);

fd = odb_pack_keep(filename);
if (fd < 0) {
Expand All @@ -1437,7 +1437,7 @@ static void write_special_file(const char *suffix, const char *msg,
static void final(const char *final_pack_name, const char *curr_pack_name,
const char *final_index_name, const char *curr_index_name,
const char *keep_msg, const char *promisor_msg,
unsigned char *sha1)
unsigned char *hash)
{
const char *report = "pack";
struct strbuf pack_name = STRBUF_INIT;
Expand All @@ -1454,34 +1454,34 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
}

if (keep_msg)
write_special_file("keep", keep_msg, final_pack_name, sha1,
write_special_file("keep", keep_msg, final_pack_name, hash,
&report);
if (promisor_msg)
write_special_file("promisor", promisor_msg, final_pack_name,
sha1, NULL);
hash, NULL);

if (final_pack_name != curr_pack_name) {
if (!final_pack_name)
final_pack_name = odb_pack_name(&pack_name, sha1, "pack");
final_pack_name = odb_pack_name(&pack_name, hash, "pack");
if (finalize_object_file(curr_pack_name, final_pack_name))
die(_("cannot store pack file"));
} else if (from_stdin)
chmod(final_pack_name, 0444);

if (final_index_name != curr_index_name) {
if (!final_index_name)
final_index_name = odb_pack_name(&index_name, sha1, "idx");
final_index_name = odb_pack_name(&index_name, hash, "idx");
if (finalize_object_file(curr_index_name, final_index_name))
die(_("cannot store index file"));
} else
chmod(final_index_name, 0444);

if (!from_stdin) {
printf("%s\n", sha1_to_hex(sha1));
printf("%s\n", sha1_to_hex(hash));
} else {
struct strbuf buf = STRBUF_INIT;

strbuf_addf(&buf, "%s\t%s\n", report, sha1_to_hex(sha1));
strbuf_addf(&buf, "%s\t%s\n", report, sha1_to_hex(hash));
write_or_die(1, buf.buf, buf.len);
strbuf_release(&buf);

Expand Down Expand Up @@ -1652,7 +1652,7 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
struct strbuf index_name_buf = STRBUF_INIT;
struct pack_idx_entry **idx_objects;
struct pack_idx_option opts;
unsigned char pack_sha1[20];
unsigned char pack_hash[GIT_MAX_RAWSZ];
unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */
int report_end_of_input = 0;

Expand Down Expand Up @@ -1789,11 +1789,11 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
if (show_stat)
obj_stat = xcalloc(st_add(nr_objects, 1), sizeof(struct object_stat));
ofs_deltas = xcalloc(nr_objects, sizeof(struct ofs_delta_entry));
parse_pack_objects(pack_sha1);
parse_pack_objects(pack_hash);
if (report_end_of_input)
write_in_full(2, "\0", 1);
resolve_deltas();
conclude_pack(fix_thin_pack, curr_pack, pack_sha1);
conclude_pack(fix_thin_pack, curr_pack, pack_hash);
free(ofs_deltas);
free(ref_deltas);
if (strict)
Expand All @@ -1805,14 +1805,14 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
ALLOC_ARRAY(idx_objects, nr_objects);
for (i = 0; i < nr_objects; i++)
idx_objects[i] = &objects[i].idx;
curr_index = write_idx_file(index_name, idx_objects, nr_objects, &opts, pack_sha1);
curr_index = write_idx_file(index_name, idx_objects, nr_objects, &opts, pack_hash);
free(idx_objects);

if (!verify)
final(pack_name, curr_pack,
index_name, curr_index,
keep_msg, promisor_msg,
pack_sha1);
pack_hash);
else
close(input_fd);
free(objects);
Expand Down
Loading

0 comments on commit 0fd90da

Please sign in to comment.