Skip to content

Commit

Permalink
Merge pull request #564 from Netflix/dev
Browse files Browse the repository at this point in the history
Merging dev to v0.6
  • Loading branch information
ipapapa authored Jun 9, 2018
2 parents 8499586 + df884c0 commit f59986b
Show file tree
Hide file tree
Showing 20 changed files with 67 additions and 39 deletions.
4 changes: 2 additions & 2 deletions src/dyn_cbuf.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@
*
* It requires that the circular buffer size be a power of two, and the
* size of the buffer needs to smaller than the index. So an 8 bit index
* supports a circular buffer upto ( 1 << 7 ) = 128 entries, and a 16 bit index
* supports a circular buffer upto ( 1 << 15 ) = 32768 entries.
* supports a circular buffer up to ( 1 << 7 ) = 128 entries, and a 16 bit
* index supports a circular buffer up to ( 1 << 15 ) = 32768 entries.
*
* The basis for these routines came from an article in Jack Ganssle's
* Embedded Muse: http://www.ganssle.com/tem/tem110.pdf
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_client.c
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ client_handle_response(struct conn *conn, msgid_t reqid, struct msg *rsp)
// we have to submit the response irrespective of the unref status.
rstatus_t status = msg_handle_response(req, rsp);
if (conn->waiting_to_unref) {
// dont care about the status.
// don't care about the status.
if (req->awaiting_rsps)
return DN_OK;
// all responses received
Expand Down
12 changes: 9 additions & 3 deletions src/dyn_connection_internal.c
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@ _conn_get(void)
{
struct conn *conn;

// Generate a new key for each connection
unsigned char *aes_key = generate_aes_key();
if (aes_key == NULL) {
return NULL;
}

if (!TAILQ_EMPTY(&free_connq)) {
ASSERT(nfree_connq > 0);

Expand All @@ -97,6 +103,9 @@ _conn_get(void)
conn->owner = NULL;
conn->conn_pool = NULL;

// Save a key generated earlier within the connection
memcpy(conn->aes_key, aes_key, AES_KEYLEN);

conn->sd = -1;
string_init(&conn->pname);
/* {family, addrlen, addr} are initialized in enqueue handler */
Expand Down Expand Up @@ -143,9 +152,6 @@ _conn_get(void)
conn_set_write_consistency(conn, g_write_consistency);
conn->type = CONN_UNSPECIFIED;

unsigned char *aes_key = generate_aes_key();
strncpy((char *)conn->aes_key, (char *)aes_key, strlen((char *)aes_key)); //generate a new key for each connection

return conn;
}

Expand Down
2 changes: 1 addition & 1 deletion src/dyn_dict.c
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ int dictReplace(dict *d, void *key, void *val)
dictEntry *entry, auxentry;

/* Try to add the element. If the key
* does not exists dictAdd will suceed. */
* does not exists dictAdd will succeed. */
if (dictAdd(d, key, val) == DICT_OK)
return 1;
/* It already exists, get the entry */
Expand Down
4 changes: 2 additions & 2 deletions src/dyn_dnode_client.c
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,7 @@ dnode_rsp_send_next(struct context *ctx, struct conn *conn)
//
// Every time a message is sent to a peer within dynomite, a DNODE header is
// prepended which is created using dmsg_write. A message remembers this case
// in dnode_header_prepended, so that if the messsage is sent in parts, the
// in dnode_header_prepended, so that if the message is sent in parts, the
// header is not prepended again for the subsequent parts.
//
// Like I said earlier there is a pos pointer in mbuf. If a message is sent
Expand Down Expand Up @@ -458,7 +458,7 @@ dnode_rsp_send_next(struct context *ctx, struct conn *conn)
if (status == DN_ERROR) {
loga("OOM to obtain an mbuf for encryption!");
mbuf_put(header_buf);
req_put(rsp);
rsp_put(rsp);
return NULL;
}

Expand Down
7 changes: 3 additions & 4 deletions src/dyn_dnode_msg.c
Original file line number Diff line number Diff line change
Expand Up @@ -447,7 +447,7 @@ dyn_parse_req(struct msg *r, const struct string *hash_tag)
if (dmsg->mlen > 1) {
//Decrypt AES key
dyn_rsa_decrypt(dmsg->data, aes_decrypted_buf);
strncpy((char*)r->owner->aes_key, (char*)aes_decrypted_buf, strlen((char*)aes_decrypted_buf));
memcpy(r->owner->aes_key, aes_decrypted_buf, AES_KEYLEN);
SCOPED_CHARPTR(encoded_aes_key) = base64_encode(r->owner->aes_key, AES_KEYLEN);
if (encoded_aes_key)
loga("AES decryption key: %s\n", (char*)encoded_aes_key);
Expand Down Expand Up @@ -479,7 +479,7 @@ dyn_parse_req(struct msg *r, const struct string *hash_tag)

}

//substract alraedy received bytes
//substract already received bytes
dmsg->plen -= (uint32_t)(b->last - b->pos);

return;
Expand Down Expand Up @@ -563,8 +563,7 @@ void dyn_parse_rsp(struct msg *r, const struct string *UNUSED)
if (dmsg->mlen > 1) {
//Decrypt AES key
dyn_rsa_decrypt(dmsg->data, aes_decrypted_buf);
strncpy((char *)r->owner->aes_key, (char *)aes_decrypted_buf,
strlen((char *)aes_decrypted_buf));
memcpy(r->owner->aes_key, aes_decrypted_buf, AES_KEYLEN);
}

// we have received all the remaining ecrypted data
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_dnode_peer.c
Original file line number Diff line number Diff line change
Expand Up @@ -1208,7 +1208,7 @@ dnode_rsp_gos_syn(struct context *ctx, struct conn *p_conn, struct msg *msg)
//ASSERT(p_conn->type == CONN_DNODE_PEER_CLIENT);
//add messsage
//add message
struct mbuf *nbuf = mbuf_get();
if (nbuf == NULL) {
log_debug(LOG_ERR, "Error happened in calling mbuf_get");
Expand Down
4 changes: 2 additions & 2 deletions src/dyn_gossip.c
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ gossip_forward_state(struct server_pool *sp)
*pos = ',';
pos += 1;

//write addresss
//write address
for(k=0; k<gnode->name.len; k++, pos++) {
*pos = *(gnode->name.data + k);
}
Expand Down Expand Up @@ -990,7 +990,7 @@ gossip_pool_init(struct context *ctx)
} else {

unsigned char *local_ip4 = hostname_to_private_ip4( (char *) gnode->name.data);
// Use the local_ipv4 instead of the hostname, thats what we use for
// Use the local_ipv4 instead of the hostname, that's what we use for
// comparison eventually anyways
if (local_ip4 != NULL) {
string_deinit(&gnode->name);
Expand Down
4 changes: 2 additions & 2 deletions src/dyn_message.c
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
* + + .
* | | .
* / \ .
* Request Response .../ dyn_mbuf.[ch] (mesage buffers)
* Request Response .../ dyn_mbuf.[ch] (message buffers)
* dyn_request.c dyn_response.c .../ dyn_memcache.c; dyn_redis.c (message parser)
*
* Messages in dynomite are manipulated by a chain of processing handlers,
Expand Down Expand Up @@ -757,7 +757,7 @@ uint32_t
msg_payload_crc32(struct msg *rsp)
{
ASSERT(rsp != NULL);
// take a continous buffer crc
// take a continuous buffer crc
uint32_t crc = 0;
struct mbuf *mbuf;
/* Since we want to checksum only the payload, we have to start from the
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_message.h
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ dn_strerror(dyn_error_t err)
case DYNOMITE_OK:
return "Success";
case DYNOMITE_UNKNOWN_ERROR:
return "Unknow Error";
return "Unknown Error";
case DYNOMITE_INVALID_STATE:
return "Dynomite's current state does not allow this request";
case DYNOMITE_INVALID_ADMIN_REQ:
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_response_mgr.c
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ rspmgr_check_is_done(struct response_mgr *rspmgr)
log_info("req %lu quorum achieved", rspmgr->msg->id);
rspmgr->done = true;
} else if (pending_responses) {
// Theres a mismatch in checksum. Wait for any pending responses
// There's a mismatch in checksum. Wait for any pending responses
rspmgr->done = false;
} else {
// no pending responses, and the checksum do not match.
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_stats.c
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ stats_create_bufs(struct stats *st)

/* footer */
size += 2;
// Accomodate for new fields that are directly added using stats_add_num_str
// Accommodate for new fields that are directly added using stats_add_num_str
size += 1024;

size = DN_ALIGN(size, DN_ALIGNMENT);
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_string.c
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* by the null character '\0'. The length of the string is pre-computed and
* made available explicitly as an additional field. This means that we don't
* have to walk the entire character sequence until the null terminating
* character everytime that the length of the String is requested
* character every time that the length of the String is requested
*
* The only way to create a String is to initialize it using, string_init()
* and duplicate an existing String - string_duplicate() or copy an existing
Expand Down
2 changes: 1 addition & 1 deletion src/dyn_task.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ msec_t time_to_next_task(void);

/* Execute expired tasks one after other calling individual handlers */
/* limit = 0 (execute all expired tasks)
* > 0 (upto limit tasks)
* > 0 (up to limit tasks)
*/
void execute_expired_tasks(uint32_t limit);

Expand Down
2 changes: 1 addition & 1 deletion src/dyn_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ aes_test(void)
unsigned char msg[MAX_MSG_LEN+1];
print_banner("AES");
unsigned char* aes_key = generate_aes_key();
SCOPED_CHARPTR(aes_key_print) = base64_encode(aes_key, strlen((char*)aes_key));
SCOPED_CHARPTR(aes_key_print) = base64_encode(aes_key, AES_KEYLEN);
loga("aesKey is '%s'", aes_key_print);

size_t i=0;
Expand Down
2 changes: 1 addition & 1 deletion src/dynomite.c
Original file line number Diff line number Diff line change
Expand Up @@ -561,7 +561,7 @@ dn_post_run(struct instance *nci)

/**
* Call method to initialize buffers, messages and connections. Then start the
* core dynomite loop to process messsages. When dynomite is shutting down, call
* core dynomite loop to process messages. When dynomite is shutting down, call
* method to deinitialize buffers, messages and connections.
* @param[in] nci Dynomite instance.
* @return rstatus_t Return status code.
Expand Down
2 changes: 1 addition & 1 deletion src/hashkit/dyn_jenkins.c
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ hash_jenkins(const unsigned char *key, size_t length, struct dyn_token *token)
* rest of the string. Every machine with memory protection I've seen
* does it on word boundaries, so is OK with this. But VALGRIND will
* still catch it and complain. The masking trick does make the hash
* noticably faster for short strings (like English words).
* noticeably faster for short strings (like English words).
*/
switch(length)
{
Expand Down
2 changes: 1 addition & 1 deletion src/proto/dyn_redis.c
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ redis_arg0(struct msg *r)

case MSG_REQ_REDIS_SCARD:
case MSG_REQ_REDIS_SMEMBERS:
case MSG_REQ_REDIS_SPOP:
case MSG_REQ_REDIS_SRANDMEMBER:

case MSG_REQ_REDIS_ZCARD:
Expand Down Expand Up @@ -251,6 +250,7 @@ redis_argn(struct msg *r)
case MSG_REQ_REDIS_SUNION:
case MSG_REQ_REDIS_SUNIONSTORE:
case MSG_REQ_REDIS_SSCAN:
case MSG_REQ_REDIS_SPOP:

case MSG_REQ_REDIS_ZADD:
case MSG_REQ_REDIS_ZINTERSTORE:
Expand Down
Binary file removed src/proto/stOHIQrl
Binary file not shown.
47 changes: 35 additions & 12 deletions src/seedsprovider/dyn_florida.c
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ florida_get_seeds(struct context * ctx, struct mbuf *seeds_buf) {

struct sockaddr_in *remote;
uint32_t sock;
uint32_t tmpres;
int32_t tmpres;
uint8_t buf[BUFSIZ + 1];

log_debug(LOG_VVERB, "Running florida_get_seeds!");
Expand Down Expand Up @@ -137,25 +137,27 @@ florida_get_seeds(struct context * ctx, struct mbuf *seeds_buf) {
uint8_t * htmlcontent;
uint8_t *ok = NULL;

//assume that the respsone payload is under BUF_SIZE
while ((tmpres = recv(sock, buf, BUFSIZ, 0)) > 0) {
bool socket_has_data = true;
uint32_t rx_total = 0;

// Look for a OK response in the first buffer output.
while (socket_has_data) {
// Read socket data until we get them all or RX buffer becomes full
while ((rx_total < BUFSIZ) && (tmpres = recv(sock, buf + rx_total, BUFSIZ - rx_total, 0)) > 0) {
rx_total += tmpres;
}

// Look for a OK response in the first buffer output.
if (!ok)
ok = (uint8_t *) strstr((char *)buf, "200 OK\r\n");
if (ok == NULL) {
log_error("Received Error from Florida while getting seeds");
loga_hexdump(buf, tmpres, "Florida Response with %ld bytes of data", tmpres);
loga_hexdump(buf, rx_total, "Florida Response with %ld bytes of data", rx_total);
close(sock);
dn_free(remote);
return DN_ERROR;
}

if (htmlstart == 0) {
/* Under certain conditions this will not work.
* If the \r\n\r\n part is splitted into two messages
* it will fail to detect the beginning of HTML content
*/
htmlcontent = (uint8_t *) strstr((char *)buf, "\r\n\r\n");
if(htmlcontent != NULL) {
htmlstart = 1;
Expand All @@ -165,11 +167,32 @@ florida_get_seeds(struct context * ctx, struct mbuf *seeds_buf) {
htmlcontent = buf;
}

if(htmlstart) {
mbuf_copy(seeds_buf, htmlcontent, tmpres - (htmlcontent - buf));
if (htmlstart) {
mbuf_copy(seeds_buf, htmlcontent, rx_total - (htmlcontent - buf));
}

memset(buf, 0, tmpres);
// If socket still has data for reading
if (tmpres > 0) {
if ((htmlstart == 0) && (rx_total >= 3)) {
/* In some corner cases (eg. when the read buffer size is near to the
* response header size) we can get into a situations when 4-bytes html
* content start sequence '\r\n\r\n' splits between two read iterations.
* To deal with this case the easiest way to restore splitted sequence
* before the next read iteration by move 3 last bytes (3 is enough to
* cover all split variants) from the current read iteration to the buffer
* head.
* Please notice, we repeat this step until html content is found.
*/
memcpy(buf, buf + (rx_total - 3) , 3);
memset(buf + 3, 0, rx_total - 3);
rx_total = 3;
} else {
memset(buf, 0, rx_total);
rx_total = 0;
}
} else {
socket_has_data = false;
}
}

if(tmpres < 0) {
Expand Down

0 comments on commit f59986b

Please sign in to comment.