Allow use of file images for LXC container filesystems

A previous commit gave the LXC driver the ability to mount
block devices for the container filesystem. Through use of
the loopback device functionality, we can build on this to
support use of plain file images for LXC filesytems.

By setting the LO_FLAGS_AUTOCLEAR flag we can ensure that
the loop device automatically disappears when the container
dies / shuts down

* src/lxc/lxc_container.c: Raise error if we see a file
  based filesystem, since it should have been turned into
  a loopback device already
* src/lxc/lxc_controller.c: Rewrite any filesystems of
  type=file, into type=block, by binding the file image
  to a free loop device

src/lxc/lxc_container.c

@@ -808,6 +808,11 @@ static int lxcContainerMountFS(virDomainFSDefPtr fs,
         if (lxcContainerMountFSBlock(fs, srcprefix) < 0)
             return -1;
         break;
+    case VIR_DOMAIN_FS_TYPE_FILE:
+        lxcError(VIR_ERR_INTERNAL_ERROR,
+                 _("Unexpected filesystem type %s"),
+                 virDomainFSTypeToString(fs->type));
+        break;
     default:
         lxcError(VIR_ERR_CONFIG_UNSUPPORTED,
                  _("Cannot mount filesystem type %s"),

src/lxc/lxc_controller.c

@@ -39,6 +39,8 @@
 #include <getopt.h>
 #include <sys/mount.h>
 #include <locale.h>
+#include <linux/loop.h>
+#include <dirent.h>
 
 #if HAVE_CAPNG
 # include <cap-ng.h>
@@ -63,6 +65,160 @@ struct cgroup_device_policy {
     int minor;
 };
 
+
+static int lxcGetLoopFD(char **devname)
+{
+    int fd = -1;
+    DIR *dh = NULL;
+    struct dirent *de;
+    char *looppath;
+    struct loop_info64 lo;
+
+    VIR_DEBUG("Looking for loop devices in /dev");
+
+    if (!(dh = opendir("/dev"))) {
+        virReportSystemError(errno, "%s",
+                             _("Unable to read /dev"));
+        goto cleanup;
+    }
+
+    while ((de = readdir(dh)) != NULL) {
+        if (!STRPREFIX(de->d_name, "loop"))
+            continue;
+
+        if (virAsprintf(&looppath, "/dev/%s", de->d_name) < 0) {
+            virReportOOMError();
+            goto cleanup;
+        }
+
+        VIR_DEBUG("Checking up on device %s", looppath);
+        if ((fd = open(looppath, O_RDWR)) < 0) {
+            virReportSystemError(errno,
+                                 _("Unable to open %s"), looppath);
+            goto cleanup;
+        }
+
+        if (ioctl(fd, LOOP_GET_STATUS64, &lo) < 0) {
+            /* Got a free device, return the fd */
+            if (errno == ENXIO)
+                goto cleanup;
+
+            VIR_FORCE_CLOSE(fd);
+            virReportSystemError(errno,
+                                 _("Unable to get loop status on %s"),
+                                 looppath);
+            goto cleanup;
+        }
+
+        /* Oh well, try the next device */
+        VIR_FORCE_CLOSE(fd);
+        VIR_FREE(looppath);
+    }
+
+    lxcError(VIR_ERR_INTERNAL_ERROR, "%s",
+             _("Unable to find a free loop device in /dev"));
+
+cleanup:
+    if (fd != -1) {
+        VIR_DEBUG("Got free loop device %s %d", looppath, fd);
+        *devname = looppath;
+    } else {
+        VIR_DEBUG("No free loop devices available");
+        VIR_FREE(looppath);
+    }
+    if (dh)
+        closedir(dh);
+    return fd;
+}
+
+static int lxcSetupLoopDevice(virDomainFSDefPtr fs)
+{
+    int lofd = -1;
+    int fsfd = -1;
+    struct loop_info64 lo;
+    char *loname = NULL;
+    int ret = -1;
+
+    if ((lofd = lxcGetLoopFD(&loname)) < 0)
+        return -1;
+
+    memset(&lo, 0, sizeof(lo));
+    lo.lo_flags = LO_FLAGS_AUTOCLEAR;
+
+    if ((fsfd = open(fs->src, O_RDWR)) < 0) {
+        virReportSystemError(errno,
+                             _("Unable to open %s"), fs->src);
+        goto cleanup;
+    }
+
+    if (ioctl(lofd, LOOP_SET_FD, fsfd) < 0) {
+        virReportSystemError(errno,
+                             _("Unable to attach %s to loop device"),
+                             fs->src);
+        goto cleanup;
+    }
+
+    if (ioctl(lofd, LOOP_SET_STATUS64, &lo) < 0) {
+        virReportSystemError(errno, "%s",
+                             _("Unable to mark loop device as autoclear"));
+
+        if (ioctl(lofd, LOOP_CLR_FD, 0) < 0)
+            VIR_WARN("Unable to detach %s from loop device", fs->src);
+        goto cleanup;
+    }
+
+    VIR_DEBUG("Attached loop device  %s %d to %s", fs->src, lofd, loname);
+    /*
+     * We now change it into a block device type, so that
+     * the rest of container setup 'just works'
+     */
+    fs->type = VIR_DOMAIN_FS_TYPE_BLOCK;
+    VIR_FREE(fs->src);
+    fs->src = loname;
+    loname = NULL;
+
+    ret = 0;
+
+cleanup:
+    VIR_FREE(loname);
+    VIR_FORCE_CLOSE(fsfd);
+    if (ret == -1)
+        VIR_FORCE_CLOSE(lofd);
+    return lofd;
+}
+
+
+static int lxcSetupLoopDevices(virDomainDefPtr def, size_t *nloopDevs, int **loopDevs)
+{
+    size_t i;
+    int ret = -1;
+
+    for (i = 0 ; i < def->nfss ; i++) {
+        int fd;
+
+        if (def->fss[i]->type != VIR_DOMAIN_FS_TYPE_FILE)
+            continue;
+
+        fd = lxcSetupLoopDevice(def->fss[i]);
+        if (fd < 0)
+            goto cleanup;
+
+        VIR_DEBUG("Saving loop fd %d", fd);
+        if (VIR_REALLOC_N(*loopDevs, *nloopDevs+1) < 0) {
+            VIR_FORCE_CLOSE(fd);
+            virReportOOMError();
+            goto cleanup;
+        }
+        (*loopDevs)[*nloopDevs++] = fd;
+    }
+
+    VIR_DEBUG("Setup all loop devices");
+    ret = 0;
+
+cleanup:
+    return ret;
+}
+
 /**
  * lxcSetContainerResources
  * @def: pointer to virtual machine structure
@@ -641,6 +797,9 @@ lxcControllerRun(virDomainDefPtr def,
     virDomainFSDefPtr root;
     char *devpts = NULL;
     char *devptmx = NULL;
+    size_t nloopDevs = 0;
+    int *loopDevs = NULL;
+    size_t i;
 
     if (socketpair(PF_UNIX, SOCK_STREAM, 0, control) < 0) {
         virReportSystemError(errno, "%s",
@@ -654,6 +813,9 @@ lxcControllerRun(virDomainDefPtr def,
         goto cleanup;
     }
 
+    if (lxcSetupLoopDevices(def, &nloopDevs, &loopDevs) < 0)
+        goto cleanup;
+
     root = virDomainGetRootFilesystem(def);
 
     if (lxcSetContainerResources(def) < 0)
@@ -778,8 +940,14 @@ lxcControllerRun(virDomainDefPtr def,
         goto cleanup;
     }
 
-    /* Now the container is running, there's no need for us to keep
-       any elevated capabilities */
+    /* Now the container is fully setup... */
+
+    /* ...we can close the loop devices... */
+
+    for (i = 0 ; i < nloopDevs ; i++)
+        VIR_FORCE_CLOSE(loopDevs[i]);
+
+    /* ...and reduce our privileges */
     if (lxcControllerClearCapabilities() < 0)
         goto cleanup;
 
@@ -803,6 +971,10 @@ cleanup:
     VIR_FORCE_CLOSE(containerhandshake[0]);
     VIR_FORCE_CLOSE(containerhandshake[1]);
 
+    for (i = 0 ; i < nloopDevs ; i++)
+        VIR_FORCE_CLOSE(loopDevs[i]);
+    VIR_FREE(loopDevs);
+
     if (container > 1) {
         int status;
         kill(container, SIGTERM);