From 0b1f490fc318fd201b460f0590f5ec624834cfba Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Thu, 28 Feb 2019 12:51:46 -0500 Subject: [PATCH 01/10] Start taking notes on "new" API (to match http_open) --- notes.org | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 notes.org diff --git a/notes.org b/notes.org new file mode 100644 index 0000000..a459c28 --- /dev/null +++ b/notes.org @@ -0,0 +1,14 @@ +* New Architecture + Based around how ~http_open/3~ works with keep-alive. + + Keep a pool of connections for the given host & port; each call to put/post/whatever checks if there's already a connection & re-uses that state. + + I guess still allow setting headers to control caching? + + Do we still have a worker thread? + + Maybe have an option to either synchronously wait for the given request to finish or give a hook to run async? + + Have the request thing return a fake stream & the client worker thread keeps a map of HTTP2 stream id to Prolog stream & can just write the decoded data as it goes? + + I think compatibility with the ~http_open~ API would require it to be blocking per request though, to fill out the headers at least. From 3b8401254ea4d1467d9fd9be6548bd768ce0f78c Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Tue, 17 Dec 2019 15:38:49 -0500 Subject: [PATCH 02/10] Formatting --- prolog/http2_client.pl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prolog/http2_client.pl b/prolog/http2_client.pl index d3a95f3..f2fbcfe 100644 --- a/prolog/http2_client.pl +++ b/prolog/http2_client.pl @@ -71,7 +71,7 @@ http2_open(URL, Http2Ctx, Options) :- % Open TLS connection parse_url(URL, [protocol(https),host(Host)|Attrs]), - (memberchk(port(Port), Attrs) ; Port = 443), !, + ( memberchk(port(Port), Attrs) ; Port = 443 ), !, debug(http2_client(open), "URL ~w -> Host ~w:~w", [URL, Host, Port]), ssl_context(client, Ctx, [host(Host), close_parent(true), @@ -95,7 +95,7 @@ send_frame(Stream, settings_frame([enable_push-0])), flush_output(Stream), % XXX: ...then we read a SETTINGS from from server & ACK it - (memberchk(close_cb(CloseCb), Options), ! ; CloseCb = default_close_cb), + ( memberchk(close_cb(CloseCb), Options), ! ; CloseCb = default_close_cb ), make_http2_state([authority(Host), stream(Stream), close_cb(CloseCb)], From 40df72f2b8c259881df50cbec0fbc1b6e2c774b8 Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Tue, 17 Dec 2019 15:38:58 -0500 Subject: [PATCH 03/10] WIP: making http_open/3-eseque API --- prolog/simple_client.pl | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 prolog/simple_client.pl diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl new file mode 100644 index 0000000..d1b563b --- /dev/null +++ b/prolog/simple_client.pl @@ -0,0 +1,57 @@ +:- module(simple_client, [http2_simple_open/4]). + +:- use_module(http2_client, [http2_close/1, + http2_open/3, + http2_request/4]). +:- use_module(library(unix), [pipe/2]). +:- use_module(library(url), [parse_url/2]). + +simple_complete_cb(OutStream, RespHeaders, RespHeaders, Body) :- + debug(xxx, "complete ~w ~w", [RespHeaders, Body]), + format(OutStream, "~s", [Body]), + close(OutStream). + +close_cb(Ctx, _Data) :- + debug(xxx, "closed", []). + +url_base_path(URL, Base, Path) :- + parse_url(URL, [protocol(Proto), host(Host)|URLAttrs]), + ( memberchk(port(PortN), URLAttrs) + -> format(string(Port), ":~w", [PortN]) + ; Port = "" + ), + format(string(Base), "~w://~w~w", [Proto, Host, Port]), + memberchk(path(Path_), URLAttrs), + ( memberchk(search(Search), URLAttrs) + -> ( parse_url_search(Qcs, Search), + format(string(Query), "?~s", [Qcs]) ) + ; Query = "" ), + ( memberchk(fragment(Frag), URLAttrs) + -> format(string(Fragment), "#~w", [Frag]) + ; Fragment = "" ), + atomic_list_concat([Path_, Query, Fragment], '', Path). + +http2_simple_open(Ctx, URL, Read, Options) :- + url_base_path(URL, BaseURL, Path), + debug(xxx, "opening ~w", [BaseURL]), + http2_open(BaseURL, Ctx, [close_cb(simple_client:close_cb(Ctx))]), + pipe(Read, Write), + + ( memberchk(method(Meth), Options) ; Meth = get ), + ( memberchk(headers(RespHeaders), Options) ; RespHeaders = _ ), + string_upper(Meth, Method), + debug(xxx, "Requesting ~w ~w", [Method, Path]), + http2_request(Ctx, [':method'-Method, ':path'-Path|Opts], + [], + simple_complete_cb(Write, RespHeaders)). + +test :- + debug(xxx), + http2_simple_open(Ctx, + 'https://nghttp2.org/httpbin/ip', + Stream, + []), + read_string(Stream, _, Body), + debug(xxx, "body ~w", [Body]), + close(Stream), + http2_close(Ctx). From 28511b9fa46134661f10ae37f04735c91be61626 Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Tue, 17 Dec 2019 19:43:04 -0500 Subject: [PATCH 04/10] Block until request is done, so we can set headers --- prolog/simple_client.pl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index d1b563b..b6a7d25 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -6,13 +6,12 @@ :- use_module(library(unix), [pipe/2]). :- use_module(library(url), [parse_url/2]). -simple_complete_cb(OutStream, RespHeaders, RespHeaders, Body) :- - debug(xxx, "complete ~w ~w", [RespHeaders, Body]), +simple_complete_cb(ThreadId, OutStream, Headers, Body) :- format(OutStream, "~s", [Body]), - close(OutStream). + close(OutStream), + thread_send_message(ThreadId, finished(Headers)). -close_cb(Ctx, _Data) :- - debug(xxx, "closed", []). +close_cb(_Ctx, _Data) :- true. url_base_path(URL, Base, Path) :- parse_url(URL, [protocol(Proto), host(Host)|URLAttrs]), @@ -33,25 +32,26 @@ http2_simple_open(Ctx, URL, Read, Options) :- url_base_path(URL, BaseURL, Path), - debug(xxx, "opening ~w", [BaseURL]), http2_open(BaseURL, Ctx, [close_cb(simple_client:close_cb(Ctx))]), pipe(Read, Write), ( memberchk(method(Meth), Options) ; Meth = get ), ( memberchk(headers(RespHeaders), Options) ; RespHeaders = _ ), string_upper(Meth, Method), - debug(xxx, "Requesting ~w ~w", [Method, Path]), + thread_self(ThisId), http2_request(Ctx, [':method'-Method, ':path'-Path|Opts], [], - simple_complete_cb(Write, RespHeaders)). + simple_complete_cb(ThisId, Write)), + thread_get_message(finished(RespHeaders)). test :- debug(xxx), http2_simple_open(Ctx, 'https://nghttp2.org/httpbin/ip', Stream, - []), + [headers(Headers)]), read_string(Stream, _, Body), debug(xxx, "body ~w", [Body]), + debug(xxx, "response headers ~w", [Headers]), close(Stream), http2_close(Ctx). From 65001b02d59c87f6329810e35e9cdafb2ade5268 Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Tue, 17 Dec 2019 20:43:26 -0500 Subject: [PATCH 05/10] Make loading specific headers work --- prolog/simple_client.pl | 53 ++++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index b6a7d25..02f5de0 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -1,14 +1,19 @@ -:- module(simple_client, [http2_simple_open/4]). +:- module(simple_client, [http2_simple_open/3]). :- use_module(http2_client, [http2_close/1, http2_open/3, http2_request/4]). :- use_module(library(unix), [pipe/2]). :- use_module(library(url), [parse_url/2]). +:- use_module(library(pcre), [re_replace/4]). -simple_complete_cb(ThreadId, OutStream, Headers, Body) :- +unwrap_header(Wrapped, Unwrapped) :- + Wrapped =.. [_, Unwrapped]. + +simple_complete_cb(ThreadId, OutStream, WrappedHeaders, Body) :- format(OutStream, "~s", [Body]), close(OutStream), + maplist(unwrap_header, WrappedHeaders, Headers), thread_send_message(ThreadId, finished(Headers)). close_cb(_Ctx, _Data) :- true. @@ -30,8 +35,28 @@ ; Fragment = "" ), atomic_list_concat([Path_, Query, Fragment], '', Path). -http2_simple_open(Ctx, URL, Read, Options) :- +canonical_header(Header, CanonicalHeader) :- + string_lower(Header, HeaderLower), + re_replace("-"/g, "_", HeaderLower, CanonicalHeader). + +extract_headers(Options, Headers) :- + bagof(CKey-Value, + Key^( member(header(Key, Value), Options), + canonical_header(Key, CKey) ), + WantHeaders), + maplist({Headers}/[CKey-Value]>>( + ( member(Header-V, Headers), + canonical_header(Header, CKey), + Value = V + ) ; Value = '' + ), + WantHeaders). + +build_options(_OpenOptions, []). + +http2_simple_open(URL, Read, Options) :- url_base_path(URL, BaseURL, Path), + % [TODO] keep conn open, cache? http2_open(BaseURL, Ctx, [close_cb(simple_client:close_cb(Ctx))]), pipe(Read, Write), @@ -39,19 +64,31 @@ ( memberchk(headers(RespHeaders), Options) ; RespHeaders = _ ), string_upper(Meth, Method), thread_self(ThisId), + + build_options(Options, Opts), + http2_request(Ctx, [':method'-Method, ':path'-Path|Opts], [], simple_complete_cb(ThisId, Write)), - thread_get_message(finished(RespHeaders)). + thread_get_message(finished(RespHeaders)), + http2_close(Ctx), % [TODO] + extract_headers(Options, RespHeaders). test :- debug(xxx), - http2_simple_open(Ctx, - 'https://nghttp2.org/httpbin/ip', + http2_simple_open('https://nghttp2.org/httpbin/ip', Stream, - [headers(Headers)]), + [headers(Headers), + header('Content-Length', ContentLen), + header(x_frame_options, FrameOpts), + header(some_other_thing, Nope) + ]), read_string(Stream, _, Body), debug(xxx, "body ~w", [Body]), debug(xxx, "response headers ~w", [Headers]), + debug(xxx, "Frame opts ~w", [FrameOpts]), + debug(xxx, "content len ~w", [ContentLen]), + debug(xxx, "nonexistant header ~k", [Nope]), close(Stream), - http2_close(Ctx). + %% http2_close(Ctx). + true. From 445dde12071cd4b5675ea47e1e4307463dbe723b Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Wed, 18 Dec 2019 16:02:31 -0500 Subject: [PATCH 06/10] Cache http2 connections, give ability to close them --- prolog/simple_client.pl | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index 02f5de0..877e025 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -7,6 +7,8 @@ :- use_module(library(url), [parse_url/2]). :- use_module(library(pcre), [re_replace/4]). +:- dynamic existing_url_context/2. + unwrap_header(Wrapped, Unwrapped) :- Wrapped =.. [_, Unwrapped]. @@ -16,7 +18,9 @@ maplist(unwrap_header, WrappedHeaders, Headers), thread_send_message(ThreadId, finished(Headers)). -close_cb(_Ctx, _Data) :- true. +close_cb(BaseURL, _Ctx, _Data) :- + debug(xxx, "Closing ~w", [BaseURL]), + retractall(existing_url_context(BaseURL, _)). url_base_path(URL, Base, Path) :- parse_url(URL, [protocol(Proto), host(Host)|URLAttrs]), @@ -54,10 +58,18 @@ build_options(_OpenOptions, []). +url_context(BaseURL, Ctx) :- + existing_url_context(BaseURL, Ctx), !. +url_context(BaseURL, Ctx) :- + debug(xxx, "Opening new connection ~w", [BaseURL]), + http2_open(BaseURL, Ctx, [close_cb(simple_client:close_cb(BaseURL, Ctx))]), + assertz(existing_url_context(BaseURL, Ctx)). + http2_simple_open(URL, Read, Options) :- url_base_path(URL, BaseURL, Path), - % [TODO] keep conn open, cache? - http2_open(BaseURL, Ctx, [close_cb(simple_client:close_cb(Ctx))]), + + url_context(BaseURL, Ctx), + pipe(Read, Write), ( memberchk(method(Meth), Options) ; Meth = get ), @@ -71,9 +83,14 @@ [], simple_complete_cb(ThisId, Write)), thread_get_message(finished(RespHeaders)), - http2_close(Ctx), % [TODO] extract_headers(Options, RespHeaders). +http2_simple_close(URL) :- + url_base_path(URL, Base, _), + existing_url_context(Base, Ctx), !, + http2_close(Ctx). +http2_simple_close(_). + test :- debug(xxx), http2_simple_open('https://nghttp2.org/httpbin/ip', From be8c9ce1d55d6145ed8e0ae2d12bb98cf59a5c9c Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Wed, 18 Dec 2019 16:07:09 -0500 Subject: [PATCH 07/10] Use convlist/3 to refactor extracting headers --- prolog/simple_client.pl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index 877e025..e1725e5 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -1,5 +1,8 @@ :- module(simple_client, [http2_simple_open/3]). +:- use_module(library(apply_macros)). +:- use_module(library(apply), [convlist/3, + maplist/3]). :- use_module(http2_client, [http2_close/1, http2_open/3, http2_request/4]). @@ -44,17 +47,14 @@ re_replace("-"/g, "_", HeaderLower, CanonicalHeader). extract_headers(Options, Headers) :- - bagof(CKey-Value, - Key^( member(header(Key, Value), Options), - canonical_header(Key, CKey) ), - WantHeaders), - maplist({Headers}/[CKey-Value]>>( - ( member(Header-V, Headers), - canonical_header(Header, CKey), - Value = V - ) ; Value = '' - ), - WantHeaders). + convlist({Headers}/[header(Key, Value), _]>>( + ( canonical_header(Key, CKey), + member(Header-V, Headers), + canonical_header(Header, CKey), + Value = V + ) ; Value = '' + ), + Options, _). build_options(_OpenOptions, []). From c9de726d381809c91edd3aec3230b3ed26ae808c Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Wed, 18 Dec 2019 16:13:51 -0500 Subject: [PATCH 08/10] Add more to test --- prolog/simple_client.pl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index e1725e5..8ce34c1 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -91,6 +91,7 @@ http2_close(Ctx). http2_simple_close(_). +% Sample usage test :- debug(xxx), http2_simple_open('https://nghttp2.org/httpbin/ip', @@ -107,5 +108,17 @@ debug(xxx, "content len ~w", [ContentLen]), debug(xxx, "nonexistant header ~k", [Nope]), close(Stream), - %% http2_close(Ctx). - true. + + % reusing the same connection + http2_simple_open('https://nghttp2.org/httpbin/headers', Stream2, []), + read_string(Stream2, _, Body2), + close(Body2), + debug(xxx, "headers body ~w", [Body2]), + + % reusing the same connection + http2_simple_open('https://nghttp2.org/httpbin/get', Stream3, []), + read_string(Stream3, _, Body3), + close(Body3), + debug(xxx, "get body ~w", [Body3]), + + http2_simple_close('https://nghttp2.org'). From a6153e4a1bc2f9aa772874737955e543c42e0595 Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Wed, 18 Dec 2019 16:26:34 -0500 Subject: [PATCH 09/10] Handle some more options from http_open/3 --- prolog/simple_client.pl | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index 8ce34c1..c16a0dd 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -47,16 +47,28 @@ re_replace("-"/g, "_", HeaderLower, CanonicalHeader). extract_headers(Options, Headers) :- - convlist({Headers}/[header(Key, Value), _]>>( - ( canonical_header(Key, CKey), - member(Header-V, Headers), - canonical_header(Header, CKey), - Value = V - ) ; Value = '' - ), - Options, _). - -build_options(_OpenOptions, []). + convlist(extract_header(Headers), Options, _). + +extract_header(Headers, header(Key, Value), _) :- + canonical_header(Key, CKey), + member(Header-V, Headers), + canonical_header(Header, CKey), !, + Value = V. +extract_header(_Headers, header(_, Value), _) :- + Value = ''. +extract_header(Headers, status_code(Code), _) :- + memberchk(':status'-Code, Headers). +extract_header(Headers, size(Size), _) :- + member(Header-V, Headers), + canonical_header(Header, "content_length"), + V = Size. +extract_header(_, version(2), _). + +options_headers(OpenOptions, Headers) :- + convlist(option_header, OpenOptions, Headers). + +option_header(user_agent(Agent), 'user-agent'-Agent). +option_header(request_header(Name=Value), Name-Value). url_context(BaseURL, Ctx) :- existing_url_context(BaseURL, Ctx), !. @@ -77,10 +89,12 @@ string_upper(Meth, Method), thread_self(ThisId), - build_options(Options, Opts), + options_headers(Options, Headers), + + ( memberchk(post(Data), Options) ; Data = [] ), - http2_request(Ctx, [':method'-Method, ':path'-Path|Opts], - [], + http2_request(Ctx, [':method'-Method, ':path'-Path|Headers], + Data, simple_complete_cb(ThisId, Write)), thread_get_message(finished(RespHeaders)), extract_headers(Options, RespHeaders). From 7bd81596eee9d1e9f3629f925edcf1a0a5f7eb62 Mon Sep 17 00:00:00 2001 From: "James N. V. Cash" Date: Wed, 18 Dec 2019 16:29:37 -0500 Subject: [PATCH 10/10] Use occasionallycogent for testing, since nghttp2 seems flaky --- prolog/simple_client.pl | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/prolog/simple_client.pl b/prolog/simple_client.pl index c16a0dd..06c3e9e 100644 --- a/prolog/simple_client.pl +++ b/prolog/simple_client.pl @@ -108,31 +108,34 @@ % Sample usage test :- debug(xxx), - http2_simple_open('https://nghttp2.org/httpbin/ip', + http2_simple_open('https://occasionallycogent.com/entries.html', Stream, [headers(Headers), header('Content-Length', ContentLen), header(x_frame_options, FrameOpts), - header(some_other_thing, Nope) + header(some_other_thing, Nope), + status_code(Status), + size(Size) ]), - read_string(Stream, _, Body), + read_string(Stream, 50, Body), debug(xxx, "body ~w", [Body]), + debug(xxx, "Status code ~w", [Status]), debug(xxx, "response headers ~w", [Headers]), debug(xxx, "Frame opts ~w", [FrameOpts]), - debug(xxx, "content len ~w", [ContentLen]), + debug(xxx, "content len ~w (= ~w)", [ContentLen, Size]), debug(xxx, "nonexistant header ~k", [Nope]), close(Stream), % reusing the same connection - http2_simple_open('https://nghttp2.org/httpbin/headers', Stream2, []), - read_string(Stream2, _, Body2), - close(Body2), - debug(xxx, "headers body ~w", [Body2]), + http2_simple_open('https://occasionallycogent.com/index.html', Stream2, []), + read_string(Stream2, 50, Body2), + close(Stream2), + debug(xxx, "index body ~w", [Body2]), % reusing the same connection - http2_simple_open('https://nghttp2.org/httpbin/get', Stream3, []), - read_string(Stream3, _, Body3), - close(Body3), - debug(xxx, "get body ~w", [Body3]), + http2_simple_open('https://occasionallycogent.com/about.html', Stream3, []), + read_string(Stream3, 50, Body3), + close(Stream3), + debug(xxx, "about body ~w", [Body3]), http2_simple_close('https://nghttp2.org').