XHTTP client: Move x_padding into Referer header (#4298)

""Breaking"": Update the server side first, then client

transport/internet/browser_dialer/dialer.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/base64"
+	"encoding/json"
 	"net/http"
 	"time"
 
@@ -17,6 +18,12 @@ import (
 //go:embed dialer.html
 var webpage []byte
 
+type task struct {
+	Method string `json:"method"`
+	URL    string `json:"url"`
+	Extra  any    `json:"extra,omitempty"`
+}
+
 var conns chan *websocket.Conn
 
 var upgrader = &websocket.Upgrader{
@@ -55,23 +62,69 @@ func HasBrowserDialer() bool {
 	return conns != nil
 }
 
+type webSocketExtra struct {
+	Protocol string `json:"protocol,omitempty"`
+}
+
 func DialWS(uri string, ed []byte) (*websocket.Conn, error) {
-	data := []byte("WS " + uri)
-	if ed != nil {
-		data = append(data, " "+base64.RawURLEncoding.EncodeToString(ed)...)
+	task := task{
+		Method: "WS",
+		URL:    uri,
 	}
 
-	return dialRaw(data)
+	if ed != nil {
+		task.Extra = webSocketExtra{
+			Protocol: base64.RawURLEncoding.EncodeToString(ed),
+		}
+	}
+
+	return dialTask(task)
 }
 
-func DialGet(uri string) (*websocket.Conn, error) {
-	data := []byte("GET " + uri)
-	return dialRaw(data)
+type httpExtra struct {
+	Referrer string            `json:"referrer,omitempty"`
+	Headers  map[string]string `json:"headers,omitempty"`
 }
 
-func DialPost(uri string, payload []byte) error {
-	data := []byte("POST " + uri)
-	conn, err := dialRaw(data)
+func httpExtraFromHeaders(headers http.Header) *httpExtra {
+	if len(headers) == 0 {
+		return nil
+	}
+
+	extra := httpExtra{}
+	if referrer := headers.Get("Referer"); referrer != "" {
+		extra.Referrer = referrer
+		headers.Del("Referer")
+	}
+
+	if len(headers) > 0 {
+		extra.Headers = make(map[string]string)
+		for header := range headers {
+			extra.Headers[header] = headers.Get(header)
+		}
+	}
+
+	return &extra
+}
+
+func DialGet(uri string, headers http.Header) (*websocket.Conn, error) {
+	task := task{
+		Method: "GET",
+		URL:    uri,
+		Extra:  httpExtraFromHeaders(headers),
+	}
+
+	return dialTask(task)
+}
+
+func DialPost(uri string, headers http.Header, payload []byte) error {
+	task := task{
+		Method: "POST",
+		URL:    uri,
+		Extra:  httpExtraFromHeaders(headers),
+	}
+
+	conn, err := dialTask(task)
 	if err != nil {
 		return err
 	}
@@ -90,7 +143,12 @@ func DialPost(uri string, payload []byte) error {
 	return nil
 }
 
-func dialRaw(data []byte) (*websocket.Conn, error) {
+func dialTask(task task) (*websocket.Conn, error) {
+	data, err := json.Marshal(task)
+	if err != nil {
+		return nil, err
+	}
+
 	var conn *websocket.Conn
 	for {
 		conn = <-conns
@@ -100,7 +158,7 @@ func dialRaw(data []byte) (*websocket.Conn, error) {
 			break
 		}
 	}
-	err := CheckOK(conn)
+	err = CheckOK(conn)
 	if err != nil {
 		return nil, err
 	}

transport/internet/browser_dialer/dialer.html

@@ -14,10 +14,28 @@
 		let upstreamGetCount = 0;
 		let upstreamWsCount = 0;
 		let upstreamPostCount = 0;
+
+		function prepareRequestInit(extra) {
+			const requestInit = {};
+			if (extra.referrer) {
+				// note: we have to strip the protocol and host part.
+				// Browsers disallow that, and will reset the value to current page if attempted.
+				const referrer = URL.parse(extra.referrer);
+				requestInit.referrer = referrer.pathname + referrer.search + referrer.hash;
+				requestInit.referrerPolicy = "unsafe-url";
+			}
+
+			if (extra.headers) {
+				requestInit.headers = extra.headers;
+			}
+
+			return requestInit;
+		}
+
 		let check = function () {
 			if (clientIdleCount > 0) {
 				return;
-			};
+			}
 			clientIdleCount += 1;
 			console.log("Prepare", url);
 			let ws = new WebSocket(url);
@@ -29,12 +47,12 @@
 			// double-checking that this continues to work
 			ws.onmessage = function (event) {
 				clientIdleCount -= 1;
-				let [method, url, protocol] = event.data.split(" ");
-				switch (method) {
+				let task = JSON.parse(event.data);
+				switch (task.method) {
 					case "WS": {
 						upstreamWsCount += 1;
-						console.log("Dial WS", url, protocol);
-						const wss = new WebSocket(url, protocol);
+						console.log("Dial WS", task.url, task.extra.protocol);
+						const wss = new WebSocket(task.url, task.extra.protocol);
 						wss.binaryType = "arraybuffer";
 						let opened = false;
 						ws.onmessage = function (event) {
@@ -60,10 +78,12 @@
 							wss.close()
 						};
 						break;
-					};
+					}
 					case "GET": {
 						(async () => {
-							console.log("Dial GET", url);
+							const requestInit = prepareRequestInit(task.extra);
+
+							console.log("Dial GET", task.url);
 							ws.send("ok");
 							const controller = new AbortController();
 
@@ -83,58 +103,62 @@
 							ws.onclose = (event) => {
 								try {
 									reader && reader.cancel();
-								} catch(e) {};
+								} catch(e) {}
 
 								try {
 									controller.abort();
-								} catch(e) {};
+								} catch(e) {}
 							};
 
 							try {
 								upstreamGetCount += 1;
-								const response = await fetch(url, {signal: controller.signal});
+
+								requestInit.signal = controller.signal;
+								const response = await fetch(task.url, requestInit);
 
 								const body = await response.body;
 								reader = body.getReader();
 
 								while (true) {
 									const { done, value } = await reader.read();
-									ws.send(value);
+									if (value) ws.send(value);  // don't send back "undefined" string when received nothing
 									if (done) break;
-								};
+								}
 							} finally {
 								upstreamGetCount -= 1;
 								console.log("Dial GET DONE, remaining: ", upstreamGetCount);
 								ws.close();
-							};
+							}
 						})();
 						break;
-					};
+					}
 					case "POST": {
 						upstreamPostCount += 1;
-						console.log("Dial POST", url);
+
+						const requestInit = prepareRequestInit(task.extra);
+						requestInit.method = "POST";
+
+						console.log("Dial POST", task.url);
 						ws.send("ok");
 						ws.onmessage = async (event) => {
 							try {
-								const response = await fetch(
-									url,
-									{method: "POST", body: event.data}
-								);
+								requestInit.body = event.data;
+								const response = await fetch(task.url, requestInit);
 								if (response.ok) {
 									ws.send("ok");
 								} else {
 									console.error("bad status code");
 									ws.send("fail");
-								};
+								}
 							} finally {
 								upstreamPostCount -= 1;
 								console.log("Dial POST DONE, remaining: ", upstreamPostCount);
 								ws.close();
-							};
+							}
 						};
 						break;
-					};
-				};
+					}
+				}
 
 				check();
 			};