Add support for escaped URLs

.gitmodules

@@ -93,3 +93,6 @@
 	path = Sming/third-party/mqtt-codec
 	url = https://github.com/slaff/mqtt-codec.git
 	ignore = dirty
+[submodule "Sming/third-party/libyuarel"]
+	path = Sming/third-party/libyuarel
+	url = https://github.com/jacketizer/libyuarel.git

Sming/Makefile

@@ -202,6 +202,11 @@ THIRD_PARTY_DATA += third-party/mqtt-codec/Makefile
 EXTRA_INCDIR     += third-party/mqtt-codec/src
 CUSTOM_TARGETS   += $(USER_LIBDIR)/libmqttc.a
 
+# => yuarel
+THIRD_PARTY_DATA += third-party/libyuarel/Makefile
+MODULES          += third-party/libyuarel/
+EXTRA_INCDIR     += third-party/libyuarel/
+
 # => LOCALE
 ifdef LOCALE
 	CFLAGS += -DLOCALE=$(LOCALE)

Sming/SmingCore/Data/Stream/ReadWriteStream.h

@@ -41,9 +41,6 @@ class ReadWriteStream : public IDataSourceStream
      *  to implement it
      */
 	virtual size_t write(const uint8_t* buffer, size_t size) = 0;
-
-	//Use base class documentation
-	uint16_t readMemoryBlock(char* data, int bufSize) override;
 };
 
 /** @} */

Sming/SmingCore/Data/Stream/UrlencodedOutputStream.cpp

@@ -11,7 +11,6 @@
  ****/
 
 #include "UrlencodedOutputStream.h"
-#include "../Services/WebHelpers/escape.h"
 
 /*
  * @todo Revise this so stream produces encoded output line-by-line, rather than all at once.

Sming/SmingCore/Network/Ftp/FtpDataStream.h

@@ -49,7 +49,7 @@ class FtpDataStream : public TcpConnection
 		parent->response(code, text);
 	}
 
-	int write(const char* data, int len, uint8_t apiflags = 0)
+	int write(const char* data, int len, uint8_t apiflags = 0) override
 	{
 		written += len;
 		return TcpConnection::write(data, len, apiflags);

Sming/SmingCore/Network/Http/HttpBodyParser.cpp

@@ -7,12 +7,29 @@
  * HttpBodyParser.cpp
  *
  * @author: 2017 - Slavey Karadzhov <slav@attachix.com>
+ * 	Original author
+ *
+ * @author: 2019 - mikee47 <mike@sillyhouse.net>
+ * 	Implemented un-escaping of incoming parameters
  *
  ****/
 
 #include "HttpBodyParser.h"
 #include "../WebHelpers/escape.h"
 
+/*
+ * Content is received in chunks which we need to reassemble into name=value pairs.
+ * This structure stores the temporary values during parsing.
+ */
+typedef struct {
+	char searchChar = '=';
+	String postName;
+	String postValue;
+} FormUrlParserState;
+
+/*
+ * The incoming URL is parsed
+ */
 void formUrlParser(HttpRequest& request, const char* at, int length)
 {
 	auto state = static_cast<FormUrlParserState*>(request.args);
@@ -23,14 +40,15 @@ void formUrlParser(HttpRequest& request, const char* at, int length)
 		return;
 	}
 
+	assert(state != nullptr);
+
 	auto& params = request.postParams;
 
 	if(length == PARSE_DATAEND) {
-		// Unescape post parameters
-		// @todo this should be done within the HttpParams class
-		for(unsigned i = 0; i < params.count(); i++) {
-			uri_unescape_inplace(params.keyAt(i));
-			uri_unescape_inplace(params.valueAt(i));
+		// Store last parameter, if there is one
+		if(state->postName.length() != 0) {
+			uri_unescape_inplace(state->postValue);
+			params[state->postName] = state->postValue;
 		}
 
 		delete state;
@@ -44,31 +62,37 @@ void formUrlParser(HttpRequest& request, const char* at, int length)
 		return;
 	}
 
-	String data = String(at, length);
+	while(length > 0) {
+		// Look for search character ('=' or '&') in received text
+		auto found = static_cast<const char*>(memchr(at, state->searchChar, length));
+		unsigned foundLength = (found == nullptr) ? length : (found - at);
 
-	while(data.length()) {
-		int pos = data.indexOf(state->searchChar);
-		if(pos < 0) {
+		if(foundLength != 0) {
 			if(state->searchChar == '=') {
-				state->postName += data;
+				state->postName.concat(at, foundLength);
 			} else {
-				params[state->postName] += data;
+				state->postValue.concat(at, foundLength);
 			}
+		}
 
-			return;
+		if(found == nullptr) {
+			break;
 		}
 
-		String buf = data.substring(0, pos);
 		if(state->searchChar == '=') {
-			state->postName += buf;
+			uri_unescape_inplace(state->postName);
 			state->searchChar = '&';
 		} else {
-			params[state->postName] += buf;
+			uri_unescape_inplace(state->postValue);
+			params[state->postName] = state->postValue;
 			state->searchChar = '=';
-			state->postName = nullptr;
+			// Keep String memory allocated, but clear content
+			state->postName.setLength(0);
+			state->postValue.setLength(0);
 		}
-
-		data.remove(0, pos + 1);
+		++foundLength; // Skip the '=' or '&'
+		at += foundLength;
+		length -= foundLength;
 	}
 }
 
@@ -79,16 +103,16 @@ void bodyToStringParser(HttpRequest& request, const char* at, int length)
 	if(length == PARSE_DATASTART) {
 		delete data;
 		data = new String();
-		request.args = (void*)data;
+		request.args = data;
 		return;
 	}
 
-	if(!data) {
+	if(data == nullptr) {
 		debug_e("Invalid request argument");
 		return;
 	}
 
-	if(length == PARSE_DATAEND) {
+	if(length == PARSE_DATAEND || length < 0) {
 		request.setBody(*data);
 		delete data;
 		request.args = nullptr;

Sming/SmingCore/Network/Http/HttpBodyParser.h

@@ -17,35 +17,34 @@
 #include "HttpRequest.h"
 
 /** @brief special length values passed to parse functions */
-#define PARSE_DATASTART -1
-#define PARSE_DATAEND -2
+const int PARSE_DATASTART = -1; ///< Start of incoming data
+const int PARSE_DATAEND = -2;   ///< End of incoming data
 
-typedef Delegate<void(HttpRequest&, const char* at, int length)> HttpBodyParserDelegate;
-typedef HashMap<String, HttpBodyParserDelegate> BodyParsers;
+/**
+ * @brief Body parser callback delegate
+ * @param request
+ * @param at
+ * @param length Negative lengths have special meanings
+ * @see `PARSE_DATASTART`
+ * @see `PARSE_DATAEND`
+ */
+typedef Delegate<void(HttpRequest& request, const char* at, int length)> HttpBodyParserDelegate;
 
-typedef struct {
-	char searchChar = '=';
-	String postName;
-} FormUrlParserState;
+/**
+ * @brief Maps body parsers to a specific content type
+ */
+typedef HashMap<String, HttpBodyParserDelegate> BodyParsers;
 
 /**
  * @brief Parses application/x-www-form-urlencoded body data
- * @param HttpRequest&
- * @param const *char
- * @param int length Negative lengths are used to specify special cases
- * 				-1 - start of incoming data
- * 				-2 - end of incoming data
+ * @see `HttpBodyParserDelegate`
  */
 void formUrlParser(HttpRequest& request, const char* at, int length);
 
 /**
- * @brief Stores the complete body into memory.
- *        The content later can be retrieved by calling request.getBody()
- * @param HttpRequest&
- * @param const *char
- * @param int length Negative lengths are used to specify special cases
- * 				-1 - start of incoming data
- * 				-2 - end of incoming data
+ * @brief Stores the complete body into memory
+ * @see `HttpBodyParserDelegate`
+ * @note The content later can be retrieved by calling request.getBody()
  */
 void bodyToStringParser(HttpRequest& request, const char* at, int length);
 

Sming/SmingCore/Network/Http/HttpClientConnection.cpp

@@ -48,7 +48,7 @@ bool HttpClientConnection::send(HttpRequest* request)
 		return false;
 	}
 
-	bool useSsl = (request->uri.Protocol == HTTPS_URL_PROTOCOL);
+	bool useSsl = (request->uri.Scheme == URI_SCHEME_HTTP_SECURE);
 
 #ifdef ENABLE_SSL
 	// Based on the URL decide if we should reuse the SSL and TCP pool
@@ -62,7 +62,7 @@ bool HttpClientConnection::send(HttpRequest* request)
 	}
 #endif
 
-	return connect(request->uri.Host, request->uri.Port, useSsl);
+	return connect(request->uri.Host, request->uri.getPort(), useSsl);
 }
 
 void HttpClientConnection::reset()
@@ -300,7 +300,7 @@ void HttpClientConnection::sendRequestHeaders(HttpRequest* request)
 	sendString(String(http_method_str(request->method)) + ' ' + request->uri.getPathWithQuery() + _F(" HTTP/1.1\r\n"));
 
 	if(!request->headers.contains(HTTP_HEADER_HOST)) {
-		request->headers[HTTP_HEADER_HOST] = request->uri.Host;
+		request->headers[HTTP_HEADER_HOST] = request->uri.getHostWithPort();
 	}
 
 	request->headers[HTTP_HEADER_CONTENT_LENGTH] = "0";

Sming/SmingCore/Network/Http/HttpCommon.h

@@ -17,7 +17,7 @@
 
 #include "WString.h"
 #include "../WebConstants.h"
-#include "../URL.h"
+#include "../Url.h"
 
 #ifndef HTTP_MAX_HEADER_SIZE
 #define HTTP_MAX_HEADER_SIZE (8 * 1024)

Sming/SmingCore/Network/Http/HttpConnection.cpp

@@ -84,7 +84,7 @@ int HttpConnection::staticOnPath(http_parser* parser, const char* at, size_t len
 {
 	GET_CONNECTION()
 
-	return connection->onPath(URL(String(at, length)));
+	return connection->onPath(String(at, length));
 }
 
 #ifndef COMPACT_MODE

Sming/SmingCore/Network/Http/HttpConnection.h

@@ -151,7 +151,7 @@ class HttpConnection : public TcpClient
 	 * 	@param uri
 	 * 	@retval int 0 on success, non-0 on error
 	 */
-	virtual int onPath(const URL& uri)
+	virtual int onPath(const Url& uri)
 	{
 		return 0;
 	}

Sming/SmingCore/Network/Http/HttpHeaders.h

@@ -96,6 +96,13 @@ enum HttpHeaderFieldName {
 class HttpHeaders : private HashMap<HttpHeaderFieldName, String>
 {
 public:
+	HttpHeaders() = default;
+
+	HttpHeaders(const HttpHeaders& headers)
+	{
+		*this = headers;
+	}
+
 	String toString(HttpHeaderFieldName name) const;
 
 	/** @brief Produce a string for output in the HTTP header, with line ending