LibCpp: Modify Token::to_string() to include more information

Token::to_string() now includes not only the token's type, but also its
text and span in the document.

Userland/Libraries/LibCpp/Parser.cpp

@@ -21,12 +21,7 @@ Parser::Parser(const StringView& program, const String& filename, Preprocessor::
     if constexpr (CPP_DEBUG) {
         dbgln("Tokens:");
         for (auto& token : m_tokens) {
-            StringView text;
-            if (token.start().line != token.end().line || token.start().column > token.end().column)
-                text = {};
-            else
-                text = text_of_token(token);
-            dbgln("{}  {}:{}-{}:{} ({})", token.to_string(), token.start().line, token.start().column, token.end().line, token.end().column, text);
+            dbgln("{}", token.to_string());
         }
     }
 }
@@ -930,7 +925,7 @@ Optional<size_t> Parser::index_of_token_at(Position pos) const
 void Parser::print_tokens() const
 {
     for (auto& token : m_tokens) {
-        dbgln("{}", token.to_string());
+        outln("{}", token.to_string());
     }
 }
 

Userland/Libraries/LibCpp/SyntaxHighlighter.cpp

@@ -63,7 +63,7 @@ void SyntaxHighlighter::rehighlight(const Palette& palette)
 
     Vector<GUI::TextDocumentSpan> spans;
     for (auto& token : tokens) {
-        dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ {}:{} - {}:{}", token.to_string(), token.start().line, token.start().column, token.end().line, token.end().column);
+        dbgln_if(SYNTAX_HIGHLIGHTING_DEBUG, "{} @ {}:{} - {}:{}", token.type_as_string(), token.start().line, token.start().column, token.end().line, token.end().column);
         GUI::TextDocumentSpan span;
         span.range.set_start({ token.start().line, token.start().column });
         span.range.set_end({ token.end().line, token.end().column });

Userland/Libraries/LibCpp/Token.cpp

@@ -5,6 +5,7 @@
  */
 
 #include "Token.h"
+#include <AK/String.h>
 
 namespace Cpp {
 
@@ -24,4 +25,15 @@ bool Position::operator<=(const Position& other) const
 {
     return !(*this > other);
 }
+
+String Token::to_string() const
+{
+    return String::formatted("{}  {}:{}-{}:{} ({})", type_to_string(m_type), start().line, start().column, end().line, end().column, text());
+}
+
+String Token::type_as_string() const
+{
+    return type_to_string(m_type);
+}
+
 }

Userland/Libraries/LibCpp/Token.h

@@ -116,10 +116,9 @@ struct Token {
         VERIFY_NOT_REACHED();
     }
 
-    const char* to_string() const
-    {
-        return type_to_string(m_type);
-    }
+    String to_string() const;
+    String type_as_string() const;
+
     const Position& start() const { return m_start; }
     const Position& end() const { return m_end; }