Fixes to SQUASH

fw/cache.c

@@ -965,7 +965,7 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce)
 		goto err_setup;
 
 	tfw_h2_stream_unlink_from_req(req);
-	tfw_h2_resp_fwd(resp, true);
+	tfw_h2_resp_fwd(resp);
 
 	return;
 err_setup:

fw/http.c

@@ -970,13 +970,23 @@ tfw_http_conn_req_clean(TfwHttpReq *req)
 /*
  * Free the request and the paired response.
  */
-void
+static inline void
 tfw_http_resp_pair_free(TfwHttpReq *req)
 {
 	tfw_http_conn_msg_free(req->pair);
 	tfw_http_conn_msg_free((TfwHttpMsg *)req);
 }
 
+void
+tfw_http_resp_pair_free_and_put_conn(TfwHttpResp *resp)
+{
+	TfwHttpReq *req = resp->req;
+
+	BUG_ON(!req || !req->conn);
+	tfw_connection_put(req->conn);
+	tfw_http_resp_pair_free(req);
+}
+
 /*
  * Close the client connection and free unpaired request. This function
  * is needed for cases when we cannot prepare response for this request.
@@ -1073,27 +1083,27 @@ tfw_h2_resp_status_write(TfwHttpResp *resp, unsigned short status,
 }
 
 void
-tfw_h2_resp_fwd(TfwHttpResp *resp, bool should_free)
+tfw_h2_resp_fwd(TfwHttpResp *resp)
 {
 	TfwHttpReq *req = resp->req;
+	TfwConn *conn = req->conn;
+	bool resp_in_xmit = test_bit(TFW_HTTP_B_RESP_XMIT, resp->flags);
 
-	tfw_connection_get(req->conn);
+	tfw_connection_get(conn);
 	do_access_log(resp);
 
-	if (tfw_cli_conn_send((TfwCliConn *)req->conn, (TfwMsg *)resp)) {
+	if (tfw_cli_conn_send((TfwCliConn *)conn, (TfwMsg *)resp)) {
 		T_DBG("%s: cannot send data to client via HTTP/2\n", __func__);
 		TFW_INC_STAT_BH(serv.msgs_otherr);
-		tfw_connection_close(req->conn, true);
-		should_free = true;
-	}
-	else {
+		tfw_connection_close(conn, true);
+		/* We can't send response, so we can should free it here. */
+		resp_in_xmit = false;
+	} else {
 		TFW_INC_STAT_BH(serv.msgs_forwarded);
 	}
 
-	if (should_free) {
-		tfw_connection_put(req->conn);
-		tfw_http_resp_pair_free(req);
-	}
+	if (!resp_in_xmit)
+		tfw_http_resp_pair_free_and_put_conn(resp);
 }
 
 /*
@@ -1119,7 +1129,7 @@ tfw_h2_send_resp(TfwHttpReq *req, TfwStr *msg, int status,
 		goto err_setup;
 
 	/* Send resulting HTTP/2 response and release HPACK encoder index. */
-	tfw_h2_resp_fwd(resp, true);
+	tfw_h2_resp_fwd(resp);
 
 	return;
 
@@ -4821,6 +4831,17 @@ tfw_h2_append_predefined_body(TfwHttpResp *resp, const TfwStr *body)
 	return 0;
 }
 
+static inline void
+tfw_h2_resp_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state,
+			  unsigned long h_len, unsigned long b_len)
+{
+	resp->stream_for_xmit = resp->req->stream;
+	if (state == HTTP2_ENCODE_HEADERS)
+		set_bit(TFW_HTTP_B_RESP_XMIT, resp->flags);
+	skb_set_tfw_cb_ptr(((TfwMsg *)resp)->skb_head, resp);
+	tfw_h2_stream_init_for_xmit(resp->req->stream, state, h_len, b_len);
+}
+
 /**
  * Frame forwarded response.
  */
@@ -4832,9 +4853,7 @@ tfw_h2_frame_fwd_resp(TfwHttpResp *resp, unsigned int stream_id)
 	if(!resp->req->stream)
 		return -EPIPE;
 
-	skb_set_tfw_cb(resp->msg.skb_head, stream_id);
-	tfw_h2_stream_init_for_xmit(resp->req->stream, resp,
-				    HTTP2_ENCODE_HEADERS, 0, 0);
+	tfw_h2_resp_init_for_xmit(resp, HTTP2_ENCODE_HEADERS, 0, 0);
 	return 0;
 }
 
@@ -4857,9 +4876,8 @@ tfw_h2_frame_local_resp(TfwHttpResp *resp, unsigned int stream_id,
 	if (unlikely(r))
 		return r;
 
-	skb_set_tfw_cb(resp->msg.skb_head, stream_id);
-	tfw_h2_stream_init_for_xmit(resp->req->stream, NULL,
-				    HTTP2_MAKE_HEADERS_FRAMES, h_len, b_len);
+	tfw_h2_resp_init_for_xmit(resp, HTTP2_MAKE_HEADERS_FRAMES,
+				  h_len, b_len);
 	return r;
 }
 
@@ -5298,7 +5316,7 @@ tfw_h2_resp_adjust_fwd(TfwHttpResp *resp)
 		goto clean;
 
 	tfw_h2_stream_unlink_from_req(req);
-	tfw_h2_resp_fwd(resp, false);
+	tfw_h2_resp_fwd(resp);
 
 	return;
 clean:
@@ -5328,7 +5346,7 @@ tfw_http_req_cache_service(TfwHttpResp *resp)
 	WARN_ON_ONCE(!list_empty(&req->nip_list));
 
 	if (TFW_MSG_H2(req))
-		tfw_h2_resp_fwd(resp, true);
+		tfw_h2_resp_fwd(resp);
 	else
 		tfw_http_resp_fwd(resp);
 
@@ -6065,7 +6083,7 @@ tfw_http_req_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb,
 	 */
 	if (unlikely(req->resp)) {
 		if (TFW_MSG_H2(req))
-			tfw_h2_resp_fwd(req->resp, true);
+			tfw_h2_resp_fwd(req->resp);
 		else
 			tfw_http_resp_fwd(req->resp);
 	}

fw/http.h

@@ -313,6 +313,8 @@ enum {
 	TFW_HTTP_B_HDR_LMODIFIED,
 	/* Response is fully processed and ready to be forwarded to the client. */
 	TFW_HTTP_B_RESP_READY,
+	/* Response should be alive, since it is used in xmit callback. */
+	TFW_HTTP_B_RESP_XMIT,
 	/*
 	 * Response has header 'Etag: ' and this header is
 	 * not enclosed in double quotes.
@@ -565,6 +567,7 @@ typedef struct {
  * @cut 	    - descriptors of http chunked body to be cut during
  *		      HTTP1 to HTTP2 transformation and ignored during
  *		      caching;
+ * @stream	    - stream in which the response is sent;
  */
 struct tfw_http_resp_t {
 	TFW_HTTP_MSG_COMMON;
@@ -578,6 +581,7 @@ struct tfw_http_resp_t {
 	char			*body_start_data;
 	struct sk_buff		*body_start_skb;
 	TfwStr			cut;
+	TfwStream		*stream_for_xmit;
 };
 
 /**
@@ -729,7 +733,7 @@ int tfw_http_expand_stale_warn(TfwHttpResp *resp);
 int tfw_http_expand_hdr_date(TfwHttpResp *resp);
 int tfw_http_expand_hbh(TfwHttpResp *resp, unsigned short status);
 int tfw_http_expand_hdr_via(TfwHttpResp *resp);
-void tfw_h2_resp_fwd(TfwHttpResp *resp, bool should_free);
+void tfw_h2_resp_fwd(TfwHttpResp *resp);
 int tfw_h2_hdr_map(TfwHttpResp *resp, const TfwStr *hdr, unsigned int id);
 int tfw_h2_add_hdr_date(TfwHttpResp *resp, bool cache);
 int tfw_h2_set_stale_warn(TfwHttpResp *resp);
@@ -746,7 +750,7 @@ int tfw_http_prep_redir(TfwHttpResp *resp, unsigned short status, TfwStr *rmark,
 int tfw_http_prep_304(TfwHttpReq *req, struct sk_buff **skb_head,
 		      TfwMsgIter *it);
 void tfw_http_conn_msg_free(TfwHttpMsg *hm);
-void tfw_http_resp_pair_free(TfwHttpReq *req);
+void tfw_http_resp_pair_free_and_put_conn(TfwHttpResp *resp);
 void tfw_http_send_err_resp(TfwHttpReq *req, int status, const char *reason);
 
 /* Helper functions */

fw/http_frame.c

@@ -761,9 +761,7 @@ __tfw_h2_destroy_stream_send_queue(struct rb_node *node)
 		TfwHttpResp*resp = stream->xmit.resp;
 
 		if (resp) {
-			BUG_ON(!resp->req || !resp->req->conn);
-			tfw_connection_put(resp->req->conn);
-			tfw_http_resp_pair_free(resp->req);
+			tfw_http_resp_pair_free_and_put_conn(resp);
 			stream->xmit.resp = NULL;
 		}
 		if (stream->xmit.skb_head) {
@@ -2489,8 +2487,7 @@ do {									\
 		stream->xmit.skb_head = ((TfwMsg *)resp)->skb_head;
 		((TfwMsg *)resp)->skb_head = NULL;
 
-		tfw_connection_put(resp->req->conn);
-		tfw_http_resp_pair_free(resp->req);
+		tfw_http_resp_pair_free_and_put_conn(resp);
 		stream->xmit.resp = NULL;
 
 		T_FSM_NEXT();

fw/http_stream.c

@@ -469,6 +469,7 @@ tfw_h2_add_stream(TfwStreamSched *sched, TfwStreamState state, unsigned int id,
 void
 tfw_h2_delete_stream(TfwStream *stream)
 {
+	BUG_ON(stream->xmit.resp || stream->xmit.skb_head);
 	kmem_cache_free(stream_cache, stream);
 }
 
@@ -483,9 +484,7 @@ tfw_h2_stop_stream(TfwStreamSched *sched, TfwStream *stream)
 	assert_spin_locked(&((TfwConn *)conn)->sk->sk_lock.slock);
 
 	if (resp) {
-		BUG_ON(!resp->req || !resp->req->conn);
-		tfw_connection_put(resp->req->conn);
-		tfw_http_resp_pair_free(resp->req);
+		tfw_http_resp_pair_free_and_put_conn(resp);
 		stream->xmit.resp = NULL;
 	}
 	if (stream->xmit.skb_head)

fw/http_stream.h

@@ -24,6 +24,7 @@
 #include "msg.h"
 #include "http_parser.h"
 #include "lib/str.h"
+#include "ss_skb.h"
 
 /**
  * States for HTTP/2 streams processing.
@@ -218,11 +219,10 @@ tfw_h2_stream_try_unblock(TfwStream *stream)
 }
 
 static inline void
-tfw_h2_stream_init_for_xmit(TfwStream *stream, TfwHttpResp*resp,
-			    TfwStreamXmitState state, unsigned long h_len,
-			    unsigned long b_len)
+tfw_h2_stream_init_for_xmit(TfwStream *stream, TfwStreamXmitState state,
+			    unsigned long h_len, unsigned long b_len)
 {
-	stream->xmit.resp = resp;
+	stream->xmit.resp = NULL;
 	stream->xmit.skb_head = NULL;
 	stream->xmit.h_len = h_len;
 	stream->xmit.b_len = b_len;

fw/msg.h

@@ -25,7 +25,7 @@
 
 #include <linux/skbuff.h>
 
-#include "sync_socket.h"
+#include "str.h"
 
 /**
  * @seq_list	- member in the ordered queue of messages;

fw/sock.c

@@ -38,6 +38,7 @@
 #include "tempesta_fw.h"
 #include "work_queue.h"
 #include "connection.h"
+#include "http.h"
 
 typedef enum {
 	SS_SEND,
@@ -397,7 +398,7 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags)
 	TfwConn *conn = (TfwConn *)sk->sk_user_data;
 	TfwH2Ctx *h2 = NULL;
 	TfwStream *stream = NULL;
-	unsigned int stream_id;
+	TfwHttpResp *resp = skb_get_tfw_cb_ptr(*skb_head);
 
 	T_DBG3("[%d]: %s: sk=%pK queue_empty=%d send_head=%pK"
 	       " sk_state=%d mss=%d size=%d\n",
@@ -407,6 +408,8 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags)
 
 	/* If the socket is inactive, there's no recourse. Drop the data. */
 	if (unlikely(!conn || !ss_sock_active(sk))) {
+		if (resp)
+			tfw_http_resp_pair_free_and_put_conn(resp);
 		ss_skb_queue_purge(skb_head);
 		return;
 	}
@@ -416,13 +419,12 @@ ss_do_send(struct sock *sk, struct sk_buff **skb_head, int flags)
 
 	h2 = tfw_h2_context(conn);
 
-	if ((stream_id = skb_tfw_cb(*skb_head))) {
-		stream = tfw_h2_find_not_closed_stream(h2, stream_id, false);
-		if (!stream) {
-			ss_skb_queue_purge(skb_head);
-			return;
-		}
-		BUG_ON(stream->xmit.skb_head);
+	if (resp) {
+		stream = resp->stream_for_xmit;
+
+		BUG_ON(!stream || stream->xmit.skb_head);
+		if (test_bit(TFW_HTTP_B_RESP_XMIT, resp->flags))
+			stream->xmit.resp = resp;
 		stream->xmit.mark = mark;
 		stream->xmit.tls_type = tls_type;
 	}
@@ -1043,6 +1045,7 @@ ss_tcp_state_change(struct sock *sk)
 {
 	T_DBG3("[%d]: %s: sk=%p state=%s\n",
 	       smp_processor_id(), __func__, sk, ss_statename[sk->sk_state]);
+
 	ss_sk_incoming_cpu_update(sk);
 	assert_spin_locked(&sk->sk_lock.slock);
 	TFW_VALIDATE_SK_LOCK_OWNER(sk);
@@ -1498,6 +1501,7 @@ ss_tx_action(void)
 	budget = max(10UL, ss_wq_local_size(wq));
 	while ((!ss_active() || budget--) && !ss_wq_pop(wq, &sw, &ticket)) {
 		struct sock *sk = sw.sk;
+		TfwHttpResp *resp;
 
 		bh_lock_sock(sk);
 
@@ -1563,6 +1567,11 @@ ss_tx_action(void)
 		}
 dead_sock:
 		sock_put(sk); /* paired with push() calls */
+		if (sw.skb_head && (resp = skb_get_tfw_cb_ptr(sw.skb_head))) {
+			if (resp)
+				tfw_http_resp_pair_free_and_put_conn(resp);
+		}
+
 		while ((skb = ss_skb_dequeue(&sw.skb_head)))
 			kfree_skb(skb);
 	}

fw/ss_skb.h

@@ -23,6 +23,7 @@
 #define __TFW_SS_SKB_H__
 
 #include <linux/skbuff.h>
+#include <net/tcp.h>
 
 #include "str.h"
 #include "lib/log.h"