nbd patches for 2018-07-02

Bug fixes and iotest exposure of fleecing via NBD (serving a
 read-only point-in-time view via blockdev-backup sync:none,
 as well as serving dirty bitmaps over NBD), including a new
 x-dirty-bitmap parameter when opening NBD clients as the
 counterpart to x-nbd-server-add-bitmap. Also a random fix
 for iscsi block_status spotted by Coverity that missed other
 miscellaneous trees.
 
 - Eric Blake: nbd/server: Fix dirty bitmap logic regression
 - Eric Blake: iscsi: Avoid potential for get_status overflow
 - John Snow/Vladimir Sementsov-Ogievskiy: 0/2 block: formalize and test fleecing
 - Eric Blake: 0/2 test NBD bitmap export
 -----BEGIN PGP SIGNATURE-----
 Comment: Public key at http://people.redhat.com/eblake/eblake.gpg
 
 iQEcBAABCAAGBQJbOtJPAAoJEKeha0olJ0NqEvwH/3FwWnlBdBvdYGgPjzGE1Atm
 ofCKcyxE/2VJtxeWlZQHzs1VqSq81s7am5SdzOrIWnQekvHFcLu6/71RABiauzMd
 neCvVOrXOVdktj1i1Z2Gg4BgjDmqbTDlo5ssVh/oXP0Zebi6OZFfQrB7y3cGBvui
 4XI7lW9qJxt6F1FlKloXnofWRDENyo5vgdz6QjQXfauthw2T5045RIPTfiz03FCp
 fbs+6K0+bKxfPdNLrqxxOZo/loYnEXbDYv6VBAIWBqztXVnMHxCqnh0YN05jwsfF
 TRW0/YT8lpWarOZ1soIC6a/OGXQZbxgRhZ+Zr+Wa2jw0YNHJanU9isxi37aUqQo=
 =Xivx
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/ericb/tags/pull-nbd-2018-07-02' into staging

nbd patches for 2018-07-02

Bug fixes and iotest exposure of fleecing via NBD (serving a
read-only point-in-time view via blockdev-backup sync:none,
as well as serving dirty bitmaps over NBD), including a new
x-dirty-bitmap parameter when opening NBD clients as the
counterpart to x-nbd-server-add-bitmap. Also a random fix
for iscsi block_status spotted by Coverity that missed other
miscellaneous trees.

- Eric Blake: nbd/server: Fix dirty bitmap logic regression
- Eric Blake: iscsi: Avoid potential for get_status overflow
- John Snow/Vladimir Sementsov-Ogievskiy: 0/2 block: formalize and test fleecing
- Eric Blake: 0/2 test NBD bitmap export

# gpg: Signature made Tue 03 Jul 2018 02:33:03 BST
# gpg:                using RSA key A7A16B4A2527436A
# gpg: Good signature from "Eric Blake <eblake@redhat.com>"
# gpg:                 aka "Eric Blake (Free Software Programmer) <ebb9@byu.net>"
# gpg:                 aka "[jpeg image of size 6874]"
# Primary key fingerprint: 71C2 CC22 B1C4 6029 27D2  F3AA A7A1 6B4A 2527 436A

* remotes/ericb/tags/pull-nbd-2018-07-02:
  iotests: New test 223 for exporting dirty bitmap over NBD
  nbd/client: Add x-dirty-bitmap to query bitmap from server
  iotests: add 222 to test basic fleecing
  blockdev: enable non-root nodes for backup source
  iscsi: Avoid potential for get_status overflow
  nbd/server: Fix dirty bitmap logic regression

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2018-07-03 10:47:02 +01:00
commit 9b75dcb15f
14 changed files with 436 additions and 9 deletions

View File

@ -734,7 +734,7 @@ retry:
goto out_unlock;
}
*pnum = lbasd->num_blocks * iscsilun->block_size;
*pnum = (int64_t) lbasd->num_blocks * iscsilun->block_size;
if (lbasd->provisioning == SCSI_PROVISIONING_TYPE_DEALLOCATED ||
lbasd->provisioning == SCSI_PROVISIONING_TYPE_ANCHORED) {

View File

@ -970,6 +970,7 @@ int nbd_client_init(BlockDriverState *bs,
const char *export,
QCryptoTLSCreds *tlscreds,
const char *hostname,
const char *x_dirty_bitmap,
Error **errp)
{
NBDClientSession *client = nbd_get_client_session(bs);
@ -982,9 +983,11 @@ int nbd_client_init(BlockDriverState *bs,
client->info.request_sizes = true;
client->info.structured_reply = true;
client->info.base_allocation = true;
client->info.x_dirty_bitmap = g_strdup(x_dirty_bitmap);
ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), export,
tlscreds, hostname,
&client->ioc, &client->info, errp);
g_free(client->info.x_dirty_bitmap);
if (ret < 0) {
logout("Failed to negotiate with the NBD server\n");
return ret;

View File

@ -45,6 +45,7 @@ int nbd_client_init(BlockDriverState *bs,
const char *export_name,
QCryptoTLSCreds *tlscreds,
const char *hostname,
const char *x_dirty_bitmap,
Error **errp);
void nbd_client_close(BlockDriverState *bs);

View File

@ -378,6 +378,12 @@ static QemuOptsList nbd_runtime_opts = {
.type = QEMU_OPT_STRING,
.help = "ID of the TLS credentials to use",
},
{
.name = "x-dirty-bitmap",
.type = QEMU_OPT_STRING,
.help = "experimental: expose named dirty bitmap in place of "
"block status",
},
{ /* end of list */ }
},
};
@ -438,8 +444,8 @@ static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
}
/* NBD handshake */
ret = nbd_client_init(bs, sioc, s->export,
tlscreds, hostname, errp);
ret = nbd_client_init(bs, sioc, s->export, tlscreds, hostname,
qemu_opt_get(opts, "x-dirty-bitmap"), errp);
error:
if (sioc) {
object_unref(OBJECT(sioc));

View File

@ -1859,7 +1859,7 @@ static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
backup = common->action->u.blockdev_backup.data;
bs = qmp_get_root_bs(backup->device, errp);
bs = bdrv_lookup_bs(backup->device, backup->device, errp);
if (!bs) {
return;
}
@ -3517,7 +3517,7 @@ BlockJob *do_blockdev_backup(BlockdevBackup *backup, JobTxn *txn,
backup->compress = false;
}
bs = qmp_get_root_bs(backup->device, errp);
bs = bdrv_lookup_bs(backup->device, backup->device, errp);
if (!bs) {
return NULL;
}

View File

@ -259,6 +259,7 @@ static inline bool nbd_reply_type_is_error(int type)
struct NBDExportInfo {
/* Set by client before nbd_receive_negotiate() */
bool request_sizes;
char *x_dirty_bitmap;
/* In-out fields, set by client before nbd_receive_negotiate() and
* updated by server results during nbd_receive_negotiate() */

View File

@ -1,5 +1,5 @@
/*
* Copyright (C) 2016-2017 Red Hat, Inc.
* Copyright (C) 2016-2018 Red Hat, Inc.
* Copyright (C) 2005 Anthony Liguori <anthony@codemonkey.ws>
*
* Network Block Device Client Side
@ -831,7 +831,7 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name,
if (info->structured_reply && base_allocation) {
result = nbd_negotiate_simple_meta_context(
ioc, name, "base:allocation",
ioc, name, info->x_dirty_bitmap ?: "base:allocation",
&info->meta_base_allocation_id, errp);
if (result < 0) {
goto fail;

View File

@ -1977,7 +1977,7 @@ static unsigned int bitmap_to_extents(BdrvDirtyBitmap *bitmap, uint64_t offset,
bdrv_dirty_bitmap_unlock(bitmap);
assert(offset > end);
assert(offset < end);
*length = end - offset;
return i;
}

View File

@ -3471,12 +3471,17 @@
#
# @tls-creds: TLS credentials ID
#
# @x-dirty-bitmap: A "qemu:dirty-bitmap:NAME" string to query in place of
# traditional "base:allocation" block status (see
# NBD_OPT_LIST_META_CONTEXT in the NBD protocol) (since 3.0)
#
# Since: 2.9
##
{ 'struct': 'BlockdevOptionsNbd',
'data': { 'server': 'SocketAddress',
'*export': 'str',
'*tls-creds': 'str' } }
'*tls-creds': 'str',
'*x-dirty-bitmap': 'str' } }
##
# @BlockdevOptionsRaw:

155
tests/qemu-iotests/222 Normal file
View File

@ -0,0 +1,155 @@
#!/usr/bin/env python
#
# This test covers the basic fleecing workflow, which provides a
# point-in-time snapshot of a node that can be queried over NBD.
#
# Copyright (C) 2018 Red Hat, Inc.
# John helped, too.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Creator/Owner: John Snow <jsnow@redhat.com>
import iotests
from iotests import log, qemu_img, qemu_io, qemu_io_silent
iotests.verify_platform(['linux'])
patterns = [("0x5d", "0", "64k"),
("0xd5", "1M", "64k"),
("0xdc", "32M", "64k"),
("0xcd", "0x3ff0000", "64k")] # 64M - 64K
overwrite = [("0xab", "0", "64k"), # Full overwrite
("0xad", "0x00f8000", "64k"), # Partial-left (1M-32K)
("0x1d", "0x2008000", "64k"), # Partial-right (32M+32K)
("0xea", "0x3fe0000", "64k")] # Adjacent-left (64M - 128K)
zeroes = [("0", "0x00f8000", "32k"), # Left-end of partial-left (1M-32K)
("0", "0x2010000", "32k"), # Right-end of partial-right (32M+64K)
("0", "0x3fe0000", "64k")] # overwrite[3]
remainder = [("0xd5", "0x108000", "32k"), # Right-end of partial-left [1]
("0xdc", "32M", "32k"), # Left-end of partial-right [2]
("0xcd", "0x3ff0000", "64k")] # patterns[3]
with iotests.FilePath('base.img') as base_img_path, \
iotests.FilePath('fleece.img') as fleece_img_path, \
iotests.FilePath('nbd.sock') as nbd_sock_path, \
iotests.VM() as vm:
log('--- Setting up images ---')
log('')
assert qemu_img('create', '-f', iotests.imgfmt, base_img_path, '64M') == 0
assert qemu_img('create', '-f', "qcow2", fleece_img_path, '64M') == 0
for p in patterns:
qemu_io('-f', iotests.imgfmt,
'-c', 'write -P%s %s %s' % p, base_img_path)
log('Done')
log('')
log('--- Launching VM ---')
log('')
vm.add_drive(base_img_path)
vm.launch()
log('Done')
log('')
log('--- Setting up Fleecing Graph ---')
log('')
src_node = "drive0"
tgt_node = "fleeceNode"
# create tgt_node backed by src_node
log(vm.qmp("blockdev-add", **{
"driver": "qcow2",
"node-name": tgt_node,
"file": {
"driver": "file",
"filename": fleece_img_path,
},
"backing": src_node,
}))
# Establish COW from source to fleecing node
log(vm.qmp("blockdev-backup",
device=src_node,
target=tgt_node,
sync="none"))
log('')
log('--- Setting up NBD Export ---')
log('')
nbd_uri = 'nbd+unix:///%s?socket=%s' % (tgt_node, nbd_sock_path)
log(vm.qmp("nbd-server-start",
**{"addr": { "type": "unix",
"data": { "path": nbd_sock_path } } }))
log(vm.qmp("nbd-server-add", device=tgt_node))
log('')
log('--- Sanity Check ---')
log('')
for p in (patterns + zeroes):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Testing COW ---')
log('')
for p in overwrite:
cmd = "write -P%s %s %s" % p
log(cmd)
log(vm.hmp_qemu_io(src_node, cmd))
log('')
log('--- Verifying Data ---')
log('')
for p in (patterns + zeroes):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent('-r', '-f', 'raw', '-c', cmd, nbd_uri) == 0
log('')
log('--- Cleanup ---')
log('')
log(vm.qmp('block-job-cancel', device=src_node))
log(vm.event_wait('BLOCK_JOB_CANCELLED'),
filters=[iotests.filter_qmp_event])
log(vm.qmp('nbd-server-stop'))
log(vm.qmp('blockdev-del', node_name=tgt_node))
vm.shutdown()
log('')
log('--- Confirming writes ---')
log('')
for p in (overwrite + remainder):
cmd = "read -P%s %s %s" % p
log(cmd)
assert qemu_io_silent(base_img_path, '-c', cmd) == 0
log('')
log('Done')

View File

@ -0,0 +1,67 @@
--- Setting up images ---
Done
--- Launching VM ---
Done
--- Setting up Fleecing Graph ---
{u'return': {}}
{u'return': {}}
--- Setting up NBD Export ---
{u'return': {}}
{u'return': {}}
--- Sanity Check ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Testing COW ---
write -P0xab 0 64k
{u'return': u''}
write -P0xad 0x00f8000 64k
{u'return': u''}
write -P0x1d 0x2008000 64k
{u'return': u''}
write -P0xea 0x3fe0000 64k
{u'return': u''}
--- Verifying Data ---
read -P0x5d 0 64k
read -P0xd5 1M 64k
read -P0xdc 32M 64k
read -P0xcd 0x3ff0000 64k
read -P0 0x00f8000 32k
read -P0 0x2010000 32k
read -P0 0x3fe0000 64k
--- Cleanup ---
{u'return': {}}
{u'timestamp': {u'seconds': 'SECS', u'microseconds': 'USECS'}, u'data': {u'device': u'drive0', u'type': u'backup', u'speed': 0, u'len': 67108864, u'offset': 393216}, u'event': u'BLOCK_JOB_CANCELLED'}
{u'return': {}}
{u'return': {}}
--- Confirming writes ---
read -P0xab 0 64k
read -P0xad 0x00f8000 64k
read -P0x1d 0x2008000 64k
read -P0xea 0x3fe0000 64k
read -P0xd5 0x108000 32k
read -P0xdc 32M 32k
read -P0xcd 0x3ff0000 64k
Done

138
tests/qemu-iotests/223 Executable file
View File

@ -0,0 +1,138 @@
#!/bin/bash
#
# Test reading dirty bitmap over NBD
#
# Copyright (C) 2018 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
seq="$(basename $0)"
echo "QA output created by $seq"
here="$PWD"
status=1 # failure is the default!
_cleanup()
{
_cleanup_test_img
_cleanup_qemu
rm -f "$TEST_DIR/nbd"
}
trap "_cleanup; exit \$status" 0 1 2 3 15
# get standard environment, filters and checks
. ./common.rc
. ./common.filter
. ./common.qemu
_supported_fmt qcow2
_supported_proto file # uses NBD as well
_supported_os Linux
function do_run_qemu()
{
echo Testing: "$@"
$QEMU -nographic -qmp stdio -serial none "$@"
echo
}
function run_qemu()
{
do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
| _filter_qemu | _filter_imgfmt \
| _filter_actual_image_size
}
echo
echo "=== Create partially sparse image, then add dirty bitmap ==="
echo
_make_test_img 4M
$QEMU_IO -c 'w -P 0x11 1M 2M' "$TEST_IMG" | _filter_qemu_io
run_qemu <<EOF
{ "execute": "qmp_capabilities" }
{ "execute": "blockdev-add",
"arguments": {
"driver": "$IMGFMT",
"node-name": "n",
"file": {
"driver": "file",
"filename": "$TEST_IMG"
}
}
}
{ "execute": "block-dirty-bitmap-add",
"arguments": {
"node": "n",
"name": "b",
"persistent": true
}
}
{ "execute": "quit" }
EOF
echo
echo "=== Write part of the file under active bitmap ==="
echo
$QEMU_IO -c 'w -P 0x22 2M 2M' "$TEST_IMG" | _filter_qemu_io
echo
echo "=== End dirty bitmap, and start serving image over NBD ==="
echo
_launch_qemu 2> >(_filter_nbd)
silent=
_send_qemu_cmd $QEMU_HANDLE '{"execute":"qmp_capabilities"}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"blockdev-add",
"arguments":{"driver":"qcow2", "node-name":"n",
"file":{"driver":"file", "filename":"'"$TEST_IMG"'"}}}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"x-block-dirty-bitmap-disable",
"arguments":{"node":"n", "name":"b"}}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-start",
"arguments":{"addr":{"type":"unix",
"data":{"path":"'"$TEST_DIR/nbd"'"}}}}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-add",
"arguments":{"device":"n"}}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"x-nbd-server-add-bitmap",
"arguments":{"name":"n", "bitmap":"b"}}' "return"
echo
echo "=== Contrast normal status with dirty-bitmap status ==="
echo
QEMU_IO_OPTIONS=$QEMU_IO_OPTIONS_NO_FMT
IMG="driver=nbd,export=n,server.type=unix,server.path=$TEST_DIR/nbd"
$QEMU_IO -r -c 'r -P 0 0 1m' -c 'r -P 0x11 1m 1m' \
-c 'r -P 0x22 2m 2m' --image-opts "$IMG" | _filter_qemu_io
$QEMU_IMG map --output=json --image-opts \
"$IMG" | _filter_qemu_img_map
$QEMU_IMG map --output=json --image-opts \
"$IMG,x-dirty-bitmap=qemu:dirty-bitmap:b" | _filter_qemu_img_map
echo
echo "=== End NBD server ==="
echo
_send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-remove",
"arguments":{"name":"n"}}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"nbd-server-stop"}' "return"
_send_qemu_cmd $QEMU_HANDLE '{"execute":"quit"}' "return"
# success, all done
echo '*** done'
rm -f $seq.full
status=0

View File

@ -0,0 +1,49 @@
QA output created by 223
=== Create partially sparse image, then add dirty bitmap ===
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4194304
wrote 2097152/2097152 bytes at offset 1048576
2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
Testing:
QMP_VERSION
{"return": {}}
{"return": {}}
{"return": {}}
{"return": {}}
{"timestamp": {"seconds": TIMESTAMP, "microseconds": TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
=== Write part of the file under active bitmap ===
wrote 2097152/2097152 bytes at offset 2097152
2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== End dirty bitmap, and start serving image over NBD ===
{"return": {}}
{"return": {}}
{"return": {}}
{"return": {}}
{"return": {}}
{"return": {}}
=== Contrast normal status with dirty-bitmap status ===
read 1048576/1048576 bytes at offset 0
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
read 1048576/1048576 bytes at offset 1048576
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
read 2097152/2097152 bytes at offset 2097152
2 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false},
{ "start": 1048576, "length": 3145728, "depth": 0, "zero": false, "data": true}]
[{ "start": 0, "length": 2097152, "depth": 0, "zero": false, "data": true},
{ "start": 2097152, "length": 2097152, "depth": 0, "zero": false, "data": false}]
=== End NBD server ===
{"return": {}}
{"return": {}}
{"return": {}}
*** done

View File

@ -220,3 +220,5 @@
218 rw auto quick
219 rw auto
221 rw auto quick
222 rw auto quick
223 rw auto quick