Kernel: Disable VGA console in graphical mode

Author: https://github.com/deoxxa
Commit: https://github.com/SerenityOS/serenity/commit/879bc28e14f
Pull-request: https://github.com/SerenityOS/serenity/pull/462

Kernel/TTY/VirtualConsole.cpp

@@ -81,12 +81,20 @@ void VirtualConsole::switch_to(unsigned index)
 {
     if ((int)index == s_active_console)
         return;
-    dbgprintf("VC: Switch to %u (%p)\n", index, s_consoles[index]);
     ASSERT(index < 6);
     ASSERT(s_consoles[index]);
+
     InterruptDisabler disabler;
-    if (s_active_console != -1)
-        s_consoles[s_active_console]->set_active(false);
+    if (s_active_console != -1) {
+        auto* active_console = s_consoles[s_active_console];
+        // We won't know how to switch away from a graphical console until we
+        // can set the video mode on our own. Just stop anyone from trying for
+        // now.
+        if (active_console->is_graphical())
+            return;
+        active_console->set_active(false);
+    }
+    dbgprintf("VC: Switch to %u (%p)\n", index, s_consoles[index]);
     s_active_console = index;
     s_consoles[s_active_console]->set_active(true);
     Console::the().set_implementation(s_consoles[s_active_console]);
@@ -420,6 +428,10 @@ void VirtualConsole::put_character_at(unsigned row, unsigned column, u8 ch)
 
 void VirtualConsole::on_char(u8 ch)
 {
+    // ignore writes in graphical mode
+    if (m_graphical)
+        return;
+
     switch (m_escape_state) {
     case ExpectBracket:
         if (ch == '[')
@@ -494,6 +506,10 @@ void VirtualConsole::on_char(u8 ch)
 
 void VirtualConsole::on_key_pressed(KeyboardDevice::Event key)
 {
+    // ignore keyboard in graphical mode
+    if (m_graphical)
+        return;
+
     if (!key.is_press())
         return;
     if (key.ctrl()) {

Kernel/TTY/VirtualConsole.h

@@ -20,6 +20,9 @@ public:
     static void switch_to(unsigned);
     static void initialize();
 
+    bool is_graphical() { return m_graphical; }
+    void set_graphical(bool graphical) { m_graphical = graphical; }
+
 private:
     // ^KeyboardClient
     virtual void on_key_pressed(KeyboardDevice::Event) override;
@@ -43,6 +46,7 @@ private:
     u8* m_buffer;
     unsigned m_index;
     bool m_active { false };
+    bool m_graphical { false };
 
     void scroll_up();
     void set_cursor(unsigned row, unsigned column);

Kernel/init.cpp

@@ -138,6 +138,11 @@ VFS* vfs;
 
     int error;
 
+    // SystemServer will start WindowServer, which will be doing graphics.
+    // From this point on we don't want to touch the VGA text terminal or
+    // accept keyboard input.
+    tty0->set_graphical(true);
+
     auto* system_server_process = Process::create_user_process("/bin/SystemServer", (uid_t)0, (gid_t)0, (pid_t)0, error, {}, {}, tty0);
     if (error != 0) {
         kprintf("init_stage2: error spawning SystemServer: %d\n", error);