Kernel: Implement OffsetDiskDevice to prepare for partition support

This implements a passthrough disk driver that translates the read/write
block addresses by a fixed offset. This could form the basis of MBR
partition support if we were to parse the MBR table at boot and create that
OffsetDiskDevice dynamically, rather than seeking to a fixed offset.

This also introduces a dependency in the form of grub. You'll need to have
32-bit grub binaries installed to build the project now.

As a bonus, divorcing Serenity from qemu's kernel loading means we can now
*technically* boot on real hardware. It just... doesn't get very far yet.
If you write the `_disk_image` file to an IDE hard drive and boot it in a
machine that supports all the basic PC hardware, it *will* start loading
the kernel.

Kernel/.gitignore

@@ -7,3 +7,4 @@ _fs_contents
 sync-local.sh
 *.pcap
 eth_null*
+_disk_image

Kernel/Devices/OffsetDiskDevice.cpp

@@ -0,0 +1,63 @@
+#include <Kernel/Devices/OffsetDiskDevice.h>
+
+// #define OFFD_DEBUG
+
+Retained<OffsetDiskDevice> OffsetDiskDevice::create(Retained<DiskDevice>&& device, unsigned offset)
+{
+    return adopt(*new OffsetDiskDevice(move(device), offset));
+}
+
+OffsetDiskDevice::OffsetDiskDevice(Retained<DiskDevice>&& device, unsigned offset)
+    : m_device(move(device)), m_offset(offset)
+{
+}
+
+OffsetDiskDevice::~OffsetDiskDevice()
+{
+}
+
+unsigned OffsetDiskDevice::block_size() const
+{
+    return m_device->block_size();
+}
+
+bool OffsetDiskDevice::read_block(unsigned index, byte* out) const
+{
+#ifdef OFFD_DEBUG
+    kprintf("OffsetDiskDevice::read_block %u (really: %u)\n", index, m_offset + index);
+#endif
+
+    return m_device->read_block(m_offset + index, out);
+}
+
+bool OffsetDiskDevice::write_block(unsigned index, const byte* data)
+{
+#ifdef OFFD_DEBUG
+    kprintf("OffsetDiskDevice::write_block %u (really: %u)\n", index, m_offset + index);
+#endif
+
+    return m_device->write_block(m_offset + index, data);
+}
+
+bool OffsetDiskDevice::read_blocks(unsigned index, word count, byte* out)
+{
+#ifdef OFFD_DEBUG
+    kprintf("OffsetDiskDevice::read_blocks %u (really: %u) count=%u\n", index, m_offset + index, count);
+#endif
+
+    return m_device->read_blocks(m_offset + index, count, out);
+}
+
+bool OffsetDiskDevice::write_blocks(unsigned index, word count, const byte* data)
+{
+#ifdef OFFD_DEBUG
+    kprintf("OffsetDiskDevice::write_blocks %u (really: %u) count=%u\n", index, m_offset + index, count);
+#endif
+
+    return m_device->write_blocks(m_offset + index, count, data);
+}
+
+const char* OffsetDiskDevice::class_name() const
+{
+    return "OffsetDiskDevice";
+}

Kernel/Devices/OffsetDiskDevice.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include <AK/RetainPtr.h>
+#include <Kernel/Devices/DiskDevice.h>
+
+class OffsetDiskDevice final : public DiskDevice {
+public:
+    static Retained<OffsetDiskDevice> create(Retained<DiskDevice>&& device, unsigned offset);
+    virtual ~OffsetDiskDevice();
+
+    virtual unsigned block_size() const override;
+    virtual bool read_block(unsigned index, byte* out) const override;
+    virtual bool write_block(unsigned index, const byte*) override;
+    virtual bool read_blocks(unsigned index, word count, byte*) override;
+    virtual bool write_blocks(unsigned index, word count, const byte*) override;
+
+private:
+    virtual const char* class_name() const override;
+
+    OffsetDiskDevice(Retained<DiskDevice>&&, unsigned);
+
+    Retained<DiskDevice> m_device;
+    unsigned m_offset;
+};

Kernel/Makefile

@@ -65,6 +65,7 @@ VFS_OBJS = \
     Devices/ZeroDevice.o \
     Devices/RandomDevice.o \
     Devices/DebugLogDevice.o \
+    Devices/OffsetDiskDevice.o \
     FileSystem/FileSystem.o \
     FileSystem/DiskBackedFileSystem.o \
     FileSystem/Ext2FileSystem.o \

Kernel/grub.cfg

@@ -0,0 +1,6 @@
+timeout=1
+
+menuentry 'SerenityOS' {
+  root=hd0,1
+  multiboot /boot/kernel Hello from grub!
+}

Kernel/init.cpp

@@ -6,6 +6,7 @@
 #include "Process.h"
 #include "PIC.h"
 #include <Kernel/Devices/IDEDiskDevice.h>
+#include <Kernel/Devices/OffsetDiskDevice.h>
 #include "KSyms.h"
 #include <Kernel/Devices/NullDevice.h>
 #include <Kernel/Devices/ZeroDevice.h>
@@ -57,6 +58,11 @@ VFS* vfs;
 }
 #endif
 
+// TODO: delete this magic number. this block offset corresponds to a
+// partition that starts at 32k into an MBR disk. this value is also specified
+// in sync.sh, but should ideally be read from the MBR header at startup.
+#define PARTITION_OFFSET 62
+
 [[noreturn]] static void init_stage2()
 {
     Syscall::initialize();
@@ -66,7 +72,8 @@ VFS* vfs;
     auto dev_random = make<RandomDevice>();
     auto dev_ptmx = make<PTYMultiplexer>();
     auto dev_hd0 = IDEDiskDevice::create();
-    auto e2fs = Ext2FS::create(dev_hd0.copy_ref());
+    auto dev_hd0p1 = OffsetDiskDevice::create(dev_hd0.copy_ref(), PARTITION_OFFSET);
+    auto e2fs = Ext2FS::create(dev_hd0p1.copy_ref());
     e2fs->initialize();
 
     vfs->mount_root(e2fs.copy_ref());

Kernel/run

@@ -18,9 +18,7 @@ elif [ "$1" = "qn" ]; then
         -device VGA,vgamem_mb=64 \
         -debugcon stdio \
         -device e1000 \
-        -kernel kernel \
-        -append ${SERENITY_KERNEL_CMDLINE} \
-        -hda _fs_contents \
+        -hda _disk_image \
         -soundhw pcspk
 elif [ "$1" = "qtap" ]; then
     # ./run qtap: qemu with tap
@@ -32,9 +30,7 @@ elif [ "$1" = "qtap" ]; then
         -object filter-dump,id=hue,netdev=br0,file=e1000.pcap \
         -netdev tap,ifname=tap0,id=br0 \
         -device e1000,netdev=br0 \
-        -kernel kernel \
-        -append ${SERENITY_KERNEL_CMDLINE} \
-        -hda _fs_contents \
+        -hda _disk_image \
         -soundhw pcspk
 else
     # ./run: qemu with user networking
@@ -46,9 +42,7 @@ else
         -object filter-dump,id=hue,netdev=breh,file=e1000.pcap \
         -netdev user,id=breh,hostfwd=tcp:127.0.0.1:8888-192.168.5.2:8888 \
         -device e1000,netdev=breh \
-        -kernel kernel \
-        -append ${SERENITY_KERNEL_CMDLINE} \
-        -hda _fs_contents \
+        -hda _disk_image \
         -soundhw pcspk
 fi
 

Kernel/sync.sh

@@ -1,33 +1,68 @@
 #!/bin/bash
 
-if [ "$1" = "-f" ]; then
-    rm -vf _fs_contents
-fi
+set -e
+
+die() {
+    echo "die: $@"
+    exit 1
+}
 
 if [ $(id -u) != 0 ]; then
-    echo "This needs to be run as root"
-    exit 1
+    die "this script needs to run as root"
 fi
 
-rm -vf _fs_contents.lock
-
-# If target filesystem image doesn't exist, create it.
-if [ ! -f _fs_contents ]; then
-    dd if=/dev/zero of=_fs_contents bs=1M count=512
+echo "setting up disk image..."
+if [ ! -f _disk_image ]; then
+    echo "not found; creating a new one"
+    dd if=/dev/zero of=_disk_image bs=1M count=100 || die "couldn't create disk image"
+    parted -s _disk_image mklabel msdos mkpart primary ext2 32k 100% -a minimal set 1 boot on || die "couldn't partition disk image"
+    chown 1000:1000 _disk_image || die "couldn't adjust permissions on disk image"
+else
+    echo "already exists, nothing to do"
 fi
+echo "done"
 
-mke2fs -F -I 128 _fs_contents
+echo "checking for and removing old loopback devices..."
+losetup -j _disk_image | cut -d : -f 1 | while read old_dev; do
+    echo "removing $dev"
+    losetup -d ${old_dev}
+done
+echo "done"
 
-chown 1000:1000 _fs_contents
-mkdir -vp mnt
-mount -o loop _fs_contents mnt/
-mkdir -vp mnt/bin
-mkdir -vp mnt/etc
-mkdir -vp mnt/proc
-mkdir -vp mnt/tmp
+echo -n "creating loopback device... "
+dev=$(losetup --find --partscan --show _disk_image)
+if [ -z $dev ]; then
+    die "couldn't mount loopback device"
+fi
+echo "loopback device is at ${dev}"
+
+echo -n "destroying old filesystem... "
+dd if=/dev/zero of=${dev}p1 bs=1M count=1 status=none
+echo "done"
+
+echo -n "creating new filesystem... "
+mke2fs -q -I 128 ${dev}p1 || die "couldn't create filesystem"
+echo "done"
+
+echo -n "mounting loopback device... "
+mkdir -p mnt
+mount ${dev}p1 mnt/ || die "couldn't mount loopback device"
+echo "done"
+
+echo -n "creating initial filesystem structure... "
+mkdir -p mnt/{boot,bin,etc,proc,tmp}
 chmod 1777 mnt/tmp
-mkdir -vp mnt/dev
-mkdir -vp mnt/dev/pts
+echo "done"
+
+echo "installing grub..."
+mkdir -p mnt/boot/grub
+cp grub.cfg mnt/boot/grub/grub.cfg
+grub-install --boot-directory=mnt/boot --target=i386-pc --modules="ext2 part_msdos" ${dev}
+echo "done"
+
+echo -n "setting up device nodes... "
+mkdir -p mnt/dev
+mkdir -p mnt/dev/pts
 mknod -m 666 mnt/dev/bxvga b 82 413
 mknod mnt/dev/tty0 c 4 0
 mknod mnt/dev/tty1 c 4 1
@@ -44,50 +79,73 @@ mknod -m 666 mnt/dev/ptmx c 5 2
 ln -s /proc/self/fd/0 mnt/dev/stdin
 ln -s /proc/self/fd/1 mnt/dev/stdout
 ln -s /proc/self/fd/2 mnt/dev/stderr
-cp -vR ../Base/* mnt/
-cp -vR ../Root/* mnt/
-mkdir -vp mnt/home/anon
-mkdir -vp mnt/home/nona
+echo "done"
+
+echo -n "installing base system... "
+cp -R ../Base/* mnt/
+cp -R ../Root/* mnt/
+cp kernel mnt/boot
+cp kernel.map mnt/
+echo "done"
+
+echo -n "installing users... "
+mkdir -p mnt/home/anon
+mkdir -p mnt/home/nona
 cp ../ReadMe.md mnt/home/anon/
-chown -vR 100:100 mnt/home/anon
-chown -vR 200:200 mnt/home/nona
-find ../Userland/ -type f -executable -exec cp -v {} mnt/bin/ \;
+chown -R 100:100 mnt/home/anon
+chown -R 200:200 mnt/home/nona
+echo "done"
+
+echo -n "installing userland... "
+find ../Userland/ -type f -executable -exec cp {} mnt/bin/ \;
 chmod 4755 mnt/bin/su
-cp -v ../Applications/Terminal/Terminal mnt/bin/Terminal
-cp -v ../Applications/FontEditor/FontEditor mnt/bin/FontEditor
-cp -v ../Applications/Launcher/Launcher mnt/bin/Launcher
-cp -v ../Applications/FileManager/FileManager mnt/bin/FileManager
-cp -v ../Applications/ProcessManager/ProcessManager mnt/bin/ProcessManager
-cp -v ../Applications/About/About mnt/bin/About
-cp -v ../Applications/TextEditor/TextEditor mnt/bin/TextEditor
-cp -v ../Applications/IRCClient/IRCClient mnt/bin/IRCClient
-ln -s IRCClient mnt/bin/irc
-ln -s FileManager mnt/bin/fm
-cp -v ../Servers/SystemServer/SystemServer mnt/bin/SystemServer
-cp -v ../Servers/LookupServer/LookupServer mnt/bin/LookupServer
-cp -v ../Servers/WindowServer/WindowServer mnt/bin/WindowServer
-cp -v ../Applications/Taskbar/Taskbar mnt/bin/Taskbar
-ln -s Taskbar mnt/bin/tb
-cp -v ../Applications/Downloader/Downloader mnt/bin/Downloader
+echo "done"
+
+echo -n "installing applications... "
+cp ../Applications/About/About mnt/bin/About
+cp ../Applications/Downloader/Downloader mnt/bin/Downloader
+cp ../Applications/FileManager/FileManager mnt/bin/FileManager
+cp ../Applications/FontEditor/FontEditor mnt/bin/FontEditor
+cp ../Applications/IRCClient/IRCClient mnt/bin/IRCClient
+cp ../Applications/Launcher/Launcher mnt/bin/Launcher
+cp ../Applications/ProcessManager/ProcessManager mnt/bin/ProcessManager
+cp ../Applications/Taskbar/Taskbar mnt/bin/Taskbar
+cp ../Applications/Terminal/Terminal mnt/bin/Terminal
+cp ../Applications/TextEditor/TextEditor mnt/bin/TextEditor
+cp ../Demos/HelloWorld/HelloWorld mnt/bin/HelloWorld
+cp ../Demos/RetroFetch/RetroFetch mnt/bin/RetroFetch
+cp ../Demos/WidgetGallery/WidgetGallery mnt/bin/WidgetGallery
+cp ../DevTools/VisualBuilder/VisualBuilder mnt/bin/VisualBuilder
+cp ../Games/Minesweeper/Minesweeper mnt/bin/Minesweeper
+cp ../Games/Snake/Snake mnt/bin/Snake
+cp ../Servers/LookupServer/LookupServer mnt/bin/LookupServer
+cp ../Servers/SystemServer/SystemServer mnt/bin/SystemServer
+cp ../Servers/WindowServer/WindowServer mnt/bin/WindowServer
+cp ../Shell/Shell mnt/bin/Shell
+echo "done"
+
+echo -n "installing shortcuts... "
 ln -s Downloader mnt/bin/dl
-cp -v ../DevTools/VisualBuilder/VisualBuilder mnt/bin/VisualBuilder
-ln -s VisualBuilder mnt/bin/vb
-cp -v ../Games/Minesweeper/Minesweeper mnt/bin/Minesweeper
-ln -s Minesweeper mnt/bin/ms
-cp -v ../Games/Snake/Snake mnt/bin/Snake
-ln -s Snake mnt/bin/sn
-cp -v ../Shell/Shell mnt/bin/Shell
-ln -s Shell mnt/bin/sh
-cp -v kernel.map mnt/
-cp -v ../Demos/HelloWorld/HelloWorld mnt/bin/HelloWorld
+ln -s FileManager mnt/bin/fm
 ln -s HelloWorld mnt/bin/hw
-cp -v ../Demos/RetroFetch/RetroFetch mnt/bin/RetroFetch
-cp -v ../Demos/WidgetGallery/WidgetGallery mnt/bin/WidgetGallery
+ln -s IRCClient mnt/bin/irc
+ln -s Minesweeper mnt/bin/ms
+ln -s Shell mnt/bin/sh
+ln -s Snake mnt/bin/sn
+ln -s Taskbar mnt/bin/tb
+ln -s VisualBuilder mnt/bin/vb
 ln -s WidgetGallery mnt/bin/wg
+echo "done"
 
 # Run local sync script, if it exists
 if [ -f sync-local.sh ]; then
     sh sync-local.sh
 fi
 
-umount mnt || ( sleep 0.5 && sync && umount mnt )
+echo -n "unmounting filesystem... "
+umount mnt || ( sleep 1 && sync && umount mnt )
+echo "done"
+
+echo -n "removing loopback device... "
+losetup -d ${dev}
+echo "done"