SL-18330: Send binary LLSD to LEAP plugins; extend LLSD compatibility.

indra/llcommon/llleap.cpp

@@ -204,30 +204,35 @@ class LLLeapImpl: public LLLeap
         LLSD packet(LLSDMap("pump", pump)("data", data));
 
         std::ostringstream buffer;
-        buffer << LLSDNotationStreamer(packet);
+        // SL-18330: for large data blocks, it's much faster to parse binary
+        // LLSD than notation LLSD. Use serialize(LLSD_BINARY) rather than
+        // directly calling LLSDBinaryFormatter because, unlike the latter,
+        // serialize() prepends the relevant header, needed by a general-
+        // purpose LLSD parser to distinguish binary from notation.
+        LLSDSerialize::serialize(packet, buffer, LLSDSerialize::LLSD_BINARY,
+                                 LLSDFormatter::OPTIONS_NONE);
 
 /*==========================================================================*|
         // DEBUGGING ONLY: don't copy str() if we can avoid it.
         std::string strdata(buffer.str());
         if (std::size_t(buffer.tellp()) != strdata.length())
         {
-            LL_ERRS("LLLeap") << "tellp() -> " << buffer.tellp() << " != "
+            LL_ERRS("LLLeap") << "tellp() -> " << static_cast<U64>(buffer.tellp()) << " != "
                               << "str().length() -> " << strdata.length() << LL_ENDL;
         }
         // DEBUGGING ONLY: reading back is terribly inefficient.
         std::istringstream readback(strdata);
         LLSD echo;
-        LLPointer<LLSDParser> parser(new LLSDNotationParser());
-        S32 parse_status(parser->parse(readback, echo, strdata.length()));
-        if (parse_status == LLSDParser::PARSE_FAILURE)
+        bool parse_status(LLSDSerialize::deserialize(echo, readback, strdata.length()));
+        if (! parse_status)
         {
-            LL_ERRS("LLLeap") << "LLSDNotationParser() cannot parse output of "
-                              << "LLSDNotationStreamer()" << LL_ENDL;
+            LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() cannot parse output of "
+                              << "LLSDSerialize::serialize(LLSD_BINARY)" << LL_ENDL;
         }
         if (! llsd_equals(echo, packet))
         {
-            LL_ERRS("LLLeap") << "LLSDNotationParser() produced different LLSD "
-                              << "than passed to LLSDNotationStreamer()" << LL_ENDL;
+            LL_ERRS("LLLeap") << "LLSDSerialize::deserialize() returned different LLSD "
+                              << "than passed to LLSDSerialize::serialize()" << LL_ENDL;
         }
 |*==========================================================================*/
 
@@ -314,9 +319,17 @@ class LLLeapImpl: public LLLeap
             LL_DEBUGS("LLLeap") << "needed " << mExpect << " bytes, got "
                                 << childout.size() << ", parsing LLSD" << LL_ENDL;
             LLSD data;
+#if 1
+            // specifically require notation LLSD from child
             LLPointer<LLSDParser> parser(new LLSDNotationParser());
             S32 parse_status(parser->parse(childout.get_istream(), data, mExpect));
             if (parse_status == LLSDParser::PARSE_FAILURE)
+#else
+            // SL-18330: accept any valid LLSD serialization format from child
+            // Unfortunately this runs into trouble we have not yet debugged.
+            bool parse_status(LLSDSerialize::deserialize(data, childout.get_istream(), mExpect));
+            if (! parse_status)
+#endif
             {
                 bad_protocol("unparseable LLSD data");
             }

indra/llcommon/llsdserialize.cpp

@@ -45,6 +45,7 @@
 #endif
 
 #include "lldate.h"
+#include "llmemorystream.h"
 #include "llsd.h"
 #include "llstring.h"
 #include "lluri.h"
@@ -61,6 +62,23 @@ const std::string LLSD_NOTATION_HEADER("llsd/notation");
 #define windowBits 15
 #define ENABLE_ZLIB_GZIP 32
 
+// If we published this in llsdserialize.h, we could use it in the
+// implementation of LLSDOStreamer's operator<<().
+template <class Formatter>
+void format_using(const LLSD& data, std::ostream& ostr,
+                  LLSDFormatter::EFormatterOptions options=LLSDFormatter::OPTIONS_PRETTY_BINARY)
+{
+    LLPointer<Formatter> f{ new Formatter };
+    f->format(data, ostr, options);
+}
+
+template <class Parser>
+S32 parse_using(std::istream& istr, LLSD& data, size_t max_bytes, S32 max_depth=-1)
+{
+    LLPointer<Parser> p{ new Parser };
+    return p->parse(istr, data, max_bytes, max_depth);
+}
+
 /**
  * LLSDSerialize
  */
@@ -83,10 +101,10 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize
 		f = new LLSDXMLFormatter;
 		break;
 
-    case LLSD_NOTATION:
-        str << "<? " << LLSD_NOTATION_HEADER << " ?>\n";
-        f = new LLSDNotationFormatter;
-        break;
+	case LLSD_NOTATION:
+		str << "<? " << LLSD_NOTATION_HEADER << " ?>\n";
+		f = new LLSDNotationFormatter;
+		break;
 
 	default:
 		LL_WARNS() << "serialize request for unknown ELLSD_Serialize" << LL_ENDL;
@@ -101,98 +119,129 @@ void LLSDSerialize::serialize(const LLSD& sd, std::ostream& str, ELLSD_Serialize
 // static
 bool LLSDSerialize::deserialize(LLSD& sd, std::istream& str, llssize max_bytes)
 {
-	LLPointer<LLSDParser> p = NULL;
 	char hdr_buf[MAX_HDR_LEN + 1] = ""; /* Flawfinder: ignore */
-	int i;
-	int inbuf = 0;
-	bool legacy_no_header = false;
 	bool fail_if_not_legacy = false;
-	std::string header;
 
 	/*
-	 * Get the first line before anything.
+	 * Get the first line before anything. Don't read more than max_bytes:
+	 * this get() overload reads no more than (count-1) bytes into the
+	 * specified buffer. In the usual case when max_bytes exceeds
+	 * sizeof(hdr_buf), get() will read no more than sizeof(hdr_buf)-2.
 	 */
-	str.get(hdr_buf, MAX_HDR_LEN, '\n');
+	str.get(hdr_buf, llmin(max_bytes+1, sizeof(hdr_buf)-1), '\n');
+	auto inbuf = str.gcount();
+	// https://en.cppreference.com/w/cpp/io/basic_istream/get
+	// When the get() above sees the specified delimiter '\n', it stops there
+	// without pulling it from the stream. If it turns out that the stream
+	// does NOT contain a header, and the content includes meaningful '\n',
+	// it's important to pull that into hdr_buf too.
+	if (inbuf < max_bytes && str.get(hdr_buf[inbuf]))
+	{
+		// got the delimiting '\n'
+		++inbuf;
+		// None of the following requires that hdr_buf contain a final '\0'
+		// byte. We could store one if needed, since even the incremented
+		// inbuf won't exceed sizeof(hdr_buf)-1, but there's no need.
+	}
+	std::string header{ hdr_buf, inbuf };
 	if (str.fail())
 	{
 		str.clear();
 		fail_if_not_legacy = true;
 	}
 
 	if (!strncasecmp(LEGACY_NON_HEADER, hdr_buf, strlen(LEGACY_NON_HEADER))) /* Flawfinder: ignore */
+	{	// Create a LLSD XML parser, and parse the first chunk read above.
+		LLSDXMLParser x;
+		x.parsePart(hdr_buf, inbuf);	// Parse the first part that was already read
+		auto parsed = x.parse(str, sd, max_bytes - inbuf); // Parse the rest of it
+		// Formally we should probably check (parsed != PARSE_FAILURE &&
+		// parsed > 0), but since PARSE_FAILURE is -1, this suffices.
+		return (parsed > 0);
+	}
+
+	if (fail_if_not_legacy)
 	{
-		legacy_no_header = true;
-		inbuf = (int)str.gcount();
+		LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL;
+		return false;
 	}
-	else
+
+	/*
+	* Remove the newline chars
+	*/
+	std::string::size_type lastchar = header.find_last_not_of("\r\n");
+	if (lastchar != std::string::npos)
 	{
-		if (fail_if_not_legacy)
-			goto fail;
-		/*
-		* Remove the newline chars
-		*/
-		for (i = 0; i < MAX_HDR_LEN; i++)
-		{
-			if (hdr_buf[i] == 0 || hdr_buf[i] == '\r' ||
-				hdr_buf[i] == '\n')
-			{
-				hdr_buf[i] = 0;
-				break;
-			}
-		}
-		header = hdr_buf;
+		// It's important that find_last_not_of() returns size_type, which is
+		// why lastchar explicitly declares the type above. erase(size_type)
+		// erases from that offset to the end of the string, whereas
+		// erase(iterator) erases only a single character.
+		header.erase(lastchar+1);
+	}
 
-		std::string::size_type start = std::string::npos;
-		std::string::size_type end = std::string::npos;
-		start = header.find_first_not_of("<? ");
-		if (start != std::string::npos)
+	// trim off the <? ... ?> header syntax
+	auto start = header.find_first_not_of("<? ");
+	if (start != std::string::npos)
+	{
+		auto end = header.find_first_of(" ?", start);
+		if (end != std::string::npos)
 		{
-			end = header.find_first_of(" ?", start);
+			header = header.substr(start, end - start);
+			ws(str);
 		}
-		if ((start == std::string::npos) || (end == std::string::npos))
-			goto fail;
-
-		header = header.substr(start, end - start);
-		ws(str);
 	}
 	/*
 	 * Create the parser as appropriate
 	 */
-	if (legacy_no_header)
-	{	// Create a LLSD XML parser, and parse the first chunk read above
-		LLSDXMLParser* x = new LLSDXMLParser();
-		x->parsePart(hdr_buf, inbuf);	// Parse the first part that was already read
-		x->parseLines(str, sd);			// Parse the rest of it
-		delete x;
-		return true;
-	}
-
-	if (header == LLSD_BINARY_HEADER)
+	if (0 == LLStringUtil::compareInsensitive(header, LLSD_BINARY_HEADER))
 	{
-		p = new LLSDBinaryParser;
+		return (parse_using<LLSDBinaryParser>(str, sd, max_bytes-inbuf) > 0);
 	}
-	else if (header == LLSD_XML_HEADER)
+	else if (0 == LLStringUtil::compareInsensitive(header, LLSD_XML_HEADER))
 	{
-		p = new LLSDXMLParser;
+		return (parse_using<LLSDXMLParser>(str, sd, max_bytes-inbuf) > 0);
 	}
-	else if (header == LLSD_NOTATION_HEADER)
+	else if (0 == LLStringUtil::compareInsensitive(header, LLSD_NOTATION_HEADER))
 	{
-		p = new LLSDNotationParser;
+		return (parse_using<LLSDNotationParser>(str, sd, max_bytes-inbuf) > 0);
 	}
-	else
+	else // no header we recognize
 	{
-		LL_WARNS() << "deserialize request for unknown ELLSD_Serialize" << LL_ENDL;
-	}
-
-	if (p.notNull())
-	{
-		p->parse(str, sd, max_bytes);
-		return true;
+		LLPointer<LLSDParser> p;
+		if (inbuf && hdr_buf[0] == '<')
+		{
+			// looks like XML
+			LL_DEBUGS() << "deserialize request with no header, assuming XML" << LL_ENDL;
+			p = new LLSDXMLParser;
+		}
+		else
+		{
+			// assume notation
+			LL_DEBUGS() << "deserialize request with no header, assuming notation" << LL_ENDL;
+			p = new LLSDNotationParser;
+		}
+		// Since we've already read 'inbuf' bytes into 'hdr_buf', prepend that
+		// data to whatever remains in 'str'.
+		LLMemoryStreamBuf already(reinterpret_cast<const U8*>(hdr_buf), inbuf);
+		cat_streambuf prebuff(&already, str.rdbuf());
+		std::istream  prepend(&prebuff);
+#if 1
+		return (p->parse(prepend, sd, max_bytes) > 0);
+#else
+		// debugging the reconstituted 'prepend' stream
+		// allocate a buffer that we hope is big enough for the whole thing
+		std::vector<char> wholemsg((max_bytes == size_t(SIZE_UNLIMITED))? 1024 : max_bytes);
+		prepend.read(wholemsg.data(), std::min(max_bytes, wholemsg.size()));
+		LLMemoryStream replay(reinterpret_cast<const U8*>(wholemsg.data()), prepend.gcount());
+		auto success{ p->parse(replay, sd, prepend.gcount()) > 0 };
+		{
+			LL_DEBUGS() << (success? "parsed: $$" : "failed: '")
+						<< std::string(wholemsg.data(), llmin(prepend.gcount(), 100)) << "$$"
+						<< LL_ENDL;
+		}
+		return success;
+#endif
 	}
-
-fail:
-	LL_WARNS() << "deserialize LLSD parse failure" << LL_ENDL;
-	return false;
 }
 
 /**
@@ -2394,5 +2443,3 @@ U8* unzip_llsdNavMesh( bool& valid, size_t& outsize, std::istream& is, S32 size
 
 	return result;
 }
-
-

indra/llcommon/llstreamtools.cpp

@@ -513,3 +513,29 @@ std::istream& operator>>(std::istream& str, const char *tocheck)
 	}
 	return str;
 }
+
+int cat_streambuf::underflow()
+{
+    if (gptr() == egptr())
+    {
+        // here because our buffer is empty
+        std::streamsize size = 0;
+        // Until we've run out of mInputs, try reading the first of them
+        // into mBuffer. If that fetches some characters, break the loop.
+        while (! mInputs.empty()
+               && ! (size = mInputs.front()->sgetn(mBuffer.data(), mBuffer.size())))
+        {
+            // We tried to read mInputs.front() but got zero characters.
+            // Discard the first streambuf and try the next one.
+            mInputs.pop_front();
+        }
+        // Either we ran out of mInputs or we succeeded in reading some
+        // characters, that is, size != 0. Tell base class what we have.
+        setg(mBuffer.data(), mBuffer.data(), mBuffer.data() + size);
+    }
+    // If we fell out of the above loop with mBuffer still empty, return
+    // eof(), otherwise return the next character.
+    return (gptr() == egptr())
+        ? std::char_traits<char>::eof()
+        : std::char_traits<char>::to_int_type(*gptr());
+}

indra/llcommon/llstreamtools.h

@@ -27,8 +27,10 @@
 #ifndef LL_STREAM_TOOLS_H
 #define LL_STREAM_TOOLS_H
 
+#include <deque>
 #include <iostream>
 #include <string>
+#include <vector>
 
 // unless specifed otherwise these all return input_stream.good()
 
@@ -113,6 +115,27 @@ LL_COMMON_API std::streamsize fullread(
 
 LL_COMMON_API std::istream& operator>>(std::istream& str, const char *tocheck);
 
-#endif
-
+/**
+ * cat_streambuf is a std::streambuf subclass that accepts a variadic number
+ * of std::streambuf* (e.g. some_istream.rdbuf()) and virtually concatenates
+ * their contents.
+ */
+// derived from https://stackoverflow.com/a/49441066/5533635
+class cat_streambuf: public std::streambuf
+{
+private:
+    std::deque<std::streambuf*> mInputs;
+    std::vector<char> mBuffer;
+
+public:
+    // only valid for std::streambuf* arguments
+    template <typename... Inputs>
+    cat_streambuf(Inputs... inputs):
+        mInputs{inputs...},
+        mBuffer(1024)
+    {}
+
+    int underflow() override;
+};
 
+#endif