Skip to content

Commit

Permalink
skip duplicate lstats
Browse files Browse the repository at this point in the history 
this only helps on filesystems that do not support struct dirent d_type
move lstat_called and statuso into struct work
  • Loading branch information
@calccrypto
calccrypto committed Feb 6, 2025
1 parent b55ab75 commit df98b64
Show file tree
Hide file tree
Showing 21 changed files with 384 additions and 365 deletions.
50 changes: 25 additions & 25 deletions contrib/make_testindex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,17 +488,17 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
SNPRINTF(ed.osstext1, MAXXATTR, "osstext1 %zu", i);
SNPRINTF(ed.osstext2, MAXXATTR, "osstext2 %zu", i);

ed.statuso.st_ino = rng(gen);
ed.statuso.st_mode = (arg->settings->permissions > 0777)?(rng(gen) & 0777):arg->settings->permissions;
ed.statuso.st_nlink = 1;
ed.statuso.st_uid = arg->uid;
ed.statuso.st_gid = arg->gid;
ed.statuso.st_size = static_cast <off_t> (size_rngs[bucket](gen));
ed.statuso.st_blksize = arg->settings->blocksize;
ed.statuso.st_blocks = ed.statuso.st_size / ed.statuso.st_blksize;
ed.statuso.st_ctime = time_rng(gen);
ed.statuso.st_atime = std::max(ed.statuso.st_ctime, (time_t) time_rng(gen));
ed.statuso.st_mtime = std::max(ed.statuso.st_ctime, (time_t) time_rng(gen));
work->statuso.st_ino = rng(gen);
work->statuso.st_mode = (arg->settings->permissions > 0777)?(rng(gen) & 0777):arg->settings->permissions;
work->statuso.st_nlink = 1;
work->statuso.st_uid = arg->uid;
work->statuso.st_gid = arg->gid;
work->statuso.st_size = static_cast <off_t> (size_rngs[bucket](gen));
work->statuso.st_blksize = arg->settings->blocksize;
work->statuso.st_blocks = work->statuso.st_size / work->statuso.st_blksize;
work->statuso.st_ctime = time_rng(gen);
work->statuso.st_atime = std::max(work->statuso.st_ctime, (time_t) time_rng(gen));
work->statuso.st_mtime = std::max(work->statuso.st_ctime, (time_t) time_rng(gen));

work->pinode = rng(gen);
ed.crtime = rng(gen);
Expand All @@ -508,7 +508,7 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
ed.ossint4 = rng(gen);

// add this data to the summary
sumit(&summary, &ed);
sumit(&summary, work, &ed);

// insert the row
insertdbgo(work, &ed, res);
Expand All @@ -517,8 +517,8 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
// to prevent lock contention and
// because nothing is inserted until
// the transaction ends
curr_size += ed.statuso.st_size;
curr_sizes.push_back(ed.statuso.st_size);
curr_size += work->statuso.st_size;
curr_sizes.push_back(work->statuso.st_size);

free(work);
}
Expand Down Expand Up @@ -566,17 +566,17 @@ void generatecurr(ThreadArgs *arg, const std::size_t files, std::list <off_t> &s
SNPRINTF(ed.osstext1, MAXXATTR, "osstext1");
SNPRINTF(ed.osstext2, MAXXATTR, "osstext2");

ed.statuso.st_ino = rng(gen);
ed.statuso.st_mode = 0775;
ed.statuso.st_nlink = 1;
ed.statuso.st_uid = arg->uid;
ed.statuso.st_gid = arg->gid;
ed.statuso.st_size = curr_size;
ed.statuso.st_blksize = rng(gen);
ed.statuso.st_blocks = rng(gen);
ed.statuso.st_ctime = time_rng(gen);
ed.statuso.st_atime = ed.statuso.st_ctime + time_rng(gen);
ed.statuso.st_mtime = ed.statuso.st_ctime + time_rng(gen);
work->statuso.st_ino = rng(gen);
work->statuso.st_mode = 0775;
work->statuso.st_nlink = 1;
work->statuso.st_uid = arg->uid;
work->statuso.st_gid = arg->gid;
work->statuso.st_size = curr_size;
work->statuso.st_blksize = rng(gen);
work->statuso.st_blocks = rng(gen);
work->statuso.st_ctime = time_rng(gen);
work->statuso.st_atime = work->statuso.st_ctime + time_rng(gen);
work->statuso.st_mtime = work->statuso.st_ctime + time_rng(gen);

ed.crtime = rng(gen);
ed.ossint1 = rng(gen);
Expand Down
12 changes: 6 additions & 6 deletions contrib/verifytraceintree.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,23 +193,23 @@ static int check_stanza(QPTPool_t *, const size_t id, void *data, void *args) {
xattrs_cleanup(&ed.xattrs);

// make sure the permissions are correct
if (st.st_mode != ed.statuso.st_mode) {
if (st.st_mode != work->statuso.st_mode) {
std::cerr << "Permission mismatch on directory: " << parent << std::endl;
csa->correct = false;
delete sa;
return 1;
}

// make sure the uid is correct
if (st.st_uid != ed.statuso.st_uid) {
if (st.st_uid != work->statuso.st_uid) {
std::cerr << "UID mismatch on directory: " << parent << std::endl;
csa->correct = false;
delete sa;
return 1;
}

// make sure the gid is correct
if (st.st_gid != ed.statuso.st_gid) {
if (st.st_gid != work->statuso.st_gid) {
std::cerr << "GID mismatch on directory: " << parent << std::endl;
csa->correct = false;
delete sa;
Expand Down Expand Up @@ -293,19 +293,19 @@ static int check_stanza(QPTPool_t *, const size_t id, void *data, void *args) {
continue;
}

if (ca.st.st_mode != ed.statuso.st_mode) {
if (ca.st.st_mode != work->statuso.st_mode) {
std::cerr << "Permission mismatch on entry \"" << bufbase << "\""
<< " in " << db_name << std::endl;
continue;
}

if (ca.st.st_uid != ed.statuso.st_uid) {
if (ca.st.st_uid != work->statuso.st_uid) {
std::cerr << "UID mismatch on entry \"" << bufbase << "\""
<< " in " << db_name << std::endl;
continue;
}

if (ca.st.st_gid != ed.statuso.st_gid) {
if (ca.st.st_gid != work->statuso.st_gid) {
std::cerr << "GID mismatch on entry \"" << bufbase << "\""
<< " in " << db_name << std::endl;
continue;
Expand Down
4 changes: 2 additions & 2 deletions include/bf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,8 @@ struct work {
size_t name_len; /* == strlen(name) - meaning excludes NUL! */
size_t basename_len; /* can usually get through readdir */
long long int pinode;
uint8_t lstat_called;
struct stat statuso;
size_t recursion_level;

/* probably shouldn't be here */
Expand All @@ -362,8 +364,6 @@ struct entry_data {
int parent_fd; /* holds an FD that can be used for fstatat(2), etc. */
char type;
char linkname[MAXPATH];
uint8_t lstat_called;
struct stat statuso;
long long int offset;
struct xattrs xattrs;
int crtime;
Expand Down
2 changes: 1 addition & 1 deletion include/dbutils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ sqlite3_stmt *insertdbprep(sqlite3 *db, const char *sqli);
/* insert entries and xattr names */
int insertdbgo(struct work *pwork, struct entry_data *data, sqlite3_stmt *res);
/* insert directly into xattrs_avail in the associated db */
int insertdbgo_xattrs_avail(struct entry_data *data, sqlite3_stmt *res);
int insertdbgo_xattrs_avail(const ino_t inode, struct entry_data *data, sqlite3_stmt *res);
/* figure out where the xattr should go and insert it there */
int insertdbgo_xattrs(struct input *in, struct stat *dir,
struct work *work, struct entry_data *data,
Expand Down
2 changes: 1 addition & 1 deletion include/utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ uint64_t get_queue_limit(const uint64_t target_memory_footprint, const uint64_t

int zeroit(struct sum *summary);

int sumit(struct sum *summary, struct entry_data *data);
int sumit(struct sum *summary, struct work *work, struct entry_data *data);

int tsumit (struct sum *sumin, struct sum *smout);

Expand Down
68 changes: 34 additions & 34 deletions src/bfwreaddirplus2db.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ static int insertdbgor(struct work *pwork, struct entry_data *ed, sqlite3_stmt *
char *ztype = sqlite3_mprintf("%c", ed->type);
sqlite3_bind_text (res, 1, zname, -1, SQLITE_TRANSIENT);
sqlite3_bind_text (res, 2, ztype, -1, SQLITE_TRANSIENT);
sqlite3_bind_int64(res, 3, ed->statuso.st_ino);
sqlite3_bind_int64(res, 3, pwork->statuso.st_ino);
sqlite3_bind_int64(res, 4, pwork->pinode);
sqlite3_bind_int64(res, 5, ed->suspect);

Expand All @@ -160,7 +160,7 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
struct entry_data ed;
memset(&ed, 0, sizeof(ed));

if (lstat(passmywork->name, &ed.statuso) != 0) {
if (lstat(passmywork->name, &passmywork->statuso) != 0) {
return 1;
}

Expand All @@ -175,7 +175,7 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
if (in->buildindex == 1) {
SNPRINTF(dbpath, MAXPATH, "%s/%s", passmywork->name, DBNAME);
} else {
SNPRINTF(dbpath, MAXPATH, "%s/%" STAT_ino, in->nameto.data, ed.statuso.st_ino);
SNPRINTF(dbpath, MAXPATH, "%s/%" STAT_ino, in->nameto.data, passmywork->statuso.st_ino);
}

/*
Expand Down Expand Up @@ -224,9 +224,9 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {

struct entry_data qwork_ed;
memset(&qwork_ed, 0, sizeof(qwork_ed));
qwork->pinode = ed.statuso.st_ino;
qwork->pinode = passmywork->statuso.st_ino;

lstat(qwork->name, &qwork_ed.statuso);
lstat(qwork->name, &qwork->statuso);
xattrs_setup(&qwork_ed.xattrs);
if (in->process_xattrs) {
xattrs_get(qwork->name, &qwork_ed.xattrs);
Expand All @@ -236,15 +236,15 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
* there is no work to do for a directory here - we are
* processing files and links of this dir into a gufi db
*/
if (!S_ISDIR(qwork_ed.statuso.st_mode)) {
if (S_ISLNK(qwork_ed.statuso.st_mode)) {
if (!S_ISDIR(qwork->statuso.st_mode)) {
if (S_ISLNK(qwork->statuso.st_mode)) {
qwork_ed.type = 'l';
readlink(qwork->name, qwork_ed.linkname, MAXPATH);
} else if (S_ISREG(qwork_ed.statuso.st_mode)) {
} else if (S_ISREG(qwork->statuso.st_mode)) {
qwork_ed.type = 'f';
}

sumit(&summary, &qwork_ed);
sumit(&summary, qwork, &qwork_ed);
insertdbgo(qwork, &qwork_ed, res);
}

Expand All @@ -260,8 +260,8 @@ static int reprocessdir(struct input *in, void *passv, DIR *dir) {
xattrs_cleanup(&ed.xattrs);
closedb(db);

chown(dbpath, ed.statuso.st_uid, ed.statuso.st_gid);
chmod(dbpath, (ed.statuso.st_mode & ~(S_IXUSR | S_IXGRP | S_IXOTH)) | S_IRUSR);
chown(dbpath, passmywork->statuso.st_uid, passmywork->statuso.st_gid);
chmod(dbpath, (passmywork->statuso.st_mode & ~(S_IXUSR | S_IXGRP | S_IXOTH)) | S_IRUSR);

return 0;
}
Expand All @@ -288,7 +288,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {

struct entry_data ed;
memset(&ed, 0, sizeof(ed));
if (lstat(passmywork->name, &ed.statuso) != 0) {
if (lstat(passmywork->name, &passmywork->statuso) != 0) {
const int err = errno;
fprintf(stderr, "couldn't stat dir '%s': %s\n",
passmywork->name, strerror(err));
Expand Down Expand Up @@ -325,7 +325,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
if (in->output == OUTDB) {
if (in->insertdir > 0) {
if (in->suspectmethod == 0) {
const int todb = stripe_index(in, ed.statuso.st_ino, id);
const int todb = stripe_index(in, passmywork->statuso.st_ino, id);
struct ThreadArgs *ta = &pa->ta[todb];

if (in->stride > 0) {
Expand All @@ -341,18 +341,18 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
}
}
else if (in->suspectmethod == 1) { /* look up inode in trie to see if this is a suspect dir */
ed.suspect = searchmyll(pa, ed.statuso.st_ino, DFL_DIR); /* set the directory suspect flag on so we will mark it in output */
ed.suspect = searchmyll(pa, passmywork->statuso.st_ino, DFL_DIR); /* set the directory suspect flag on so we will mark it in output */
}
else if (in->suspectmethod > 1) {
/* mark the dir suspect if mtime or ctime are >= provided last run time */
lstat(passmywork->name, &ed.statuso);
if (ed.statuso.st_ctime >= locsuspecttime) ed.suspect = 1;
if (ed.statuso.st_mtime >= locsuspecttime) ed.suspect = 1;
lstat(passmywork->name, &passmywork->statuso);
if (passmywork->statuso.st_ctime >= locsuspecttime) ed.suspect = 1;
if (passmywork->statuso.st_mtime >= locsuspecttime) ed.suspect = 1;
}
}
}
else if (in->output == OUTFILE) {
const int tooutfile = stripe_index(in, ed.statuso.st_ino, id);
const int tooutfile = stripe_index(in, passmywork->statuso.st_ino, id);
struct ThreadArgs *ta = &pa->ta[tooutfile];

if (in->stride > 0) {
Expand All @@ -361,11 +361,11 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {

/* only directories are here so sortf is set to the directory full pathname */
fprintf(ta->file, "%s%c%" STAT_ino "%c%lld%c%c%c%s%c\n",
passmywork->name, in->delim,
ed.statuso.st_ino, in->delim,
passmywork->pinode, in->delim,
ed.type, in->delim,
passmywork->name, in->delim);
passmywork->name, in->delim,
passmywork->statuso.st_ino, in->delim,
passmywork->pinode, in->delim,
ed.type, in->delim,
passmywork->name, in->delim);

if (in->stride > 0) {
pthread_mutex_unlock(&ta->mutex);
Expand Down Expand Up @@ -393,8 +393,8 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
struct entry_data qwork_ed;
memset(&qwork_ed, 0, sizeof(qwork_ed));

qwork->pinode = ed.statuso.st_ino;
qwork_ed.statuso.st_ino = entry->d_ino;
qwork->pinode = passmywork->statuso.st_ino;
qwork->statuso.st_ino = entry->d_ino;

if (entry->d_type == DT_DIR) {
QPTPool_enqueue(ctx, id, processdir, qwork);
Expand All @@ -414,7 +414,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
if (in->suspectmethod) {
if (in->output == OUTDB) {
if (in->insertfl > 0) {
const int todb = stripe_index(in, ed.statuso.st_ino, id);
const int todb = stripe_index(in, qwork->statuso.st_ino, id);
struct ThreadArgs *ta = &pa->ta[todb];

if (in->stride > 0) {
Expand All @@ -434,7 +434,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
if (ed.suspect == 0) { /* if suspect dir just skip looking any further */
if ((in->suspectmethod == 1) ||
(in->suspectmethod == 2)) { /* if method 1 or 2 we look up the inode in trie and mark dir suspect or not */
ed.suspect = searchmyll(pa, qwork_ed.statuso.st_ino, DFL_FL); /* set the directory suspect flag on so we will mark it in output */
ed.suspect = searchmyll(pa, qwork->statuso.st_ino, DFL_FL); /* set the directory suspect flag on so we will mark it in output */
}
if (in->suspectmethod > 2) {
/* stat the file/link and if ctime or mtime is >= provided last run time mark dir suspect */
Expand All @@ -446,7 +446,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
}

if (in->output == OUTFILE) {
const int tooutfile = stripe_index(in, ed.statuso.st_ino, id);
const int tooutfile = stripe_index(in, qwork->statuso.st_ino, id);
struct ThreadArgs *ta = &pa->ta[tooutfile];

if (in->stride > 0) {
Expand All @@ -458,11 +458,11 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
* the file or link without the name as the sortf
*/
fprintf(ta->file, "%s%c%" STAT_ino "%c%lld%c%c%c%s%c\n",
qwork->name, in->delim,
qwork_ed.statuso.st_ino, in->delim,
qwork->pinode, in->delim,
qwork_ed.type, in->delim,
passmywork->name, in->delim);
qwork->name, in->delim,
qwork->statuso.st_ino, in->delim,
qwork->pinode, in->delim,
qwork_ed.type, in->delim,
passmywork->name, in->delim);

if (in->stride > 0) {
pthread_mutex_unlock(&ta->mutex);
Expand All @@ -480,7 +480,7 @@ static int processdir(QPTPool_t *ctx, const size_t id, void *data, void *args) {
if (in->suspectmethod > 0) {
if (in->output == OUTDB) {
if (in->insertdir > 0) {
const int todb = stripe_index(in, ed.statuso.st_ino, id);
const int todb = stripe_index(in, passmywork->statuso.st_ino, id);
struct ThreadArgs *ta = &pa->ta[todb];

if (in->stride > 0) {
Expand Down
Loading

0 comments on commit df98b64

Please sign in to comment.