diff --git a/net/netfilter/xt_policy.c b/net/netfilter/xt_policy.c
index d57a611ae0d3..1099cb005fcc 100644
--- a/net/netfilter/xt_policy.c
+++ b/net/netfilter/xt_policy.c
@@ -27,9 +27,9 @@ xt_addr_cmp(const union xt_policy_addr *a1, const union xt_policy_addr *m,
 {
 	switch (family) {
 	case AF_INET:
-		return (a1->a4.s_addr ^ a2->a4.s_addr) & m->a4.s_addr;
+		return !((a1->a4.s_addr ^ a2->a4.s_addr) & m->a4.s_addr);
 	case AF_INET6:
-		return ipv6_masked_addr_cmp(&a1->a6, &m->a6, &a2->a6);
+		return !ipv6_masked_addr_cmp(&a1->a6, &m->a6, &a2->a6);
 	}
 	return 0;
 }
@@ -44,7 +44,7 @@ match_xfrm_state(struct xfrm_state *x, const struct xt_policy_elem *e,
 #define MATCH(x,y)		(!e->match.x || ((e->x == (y)) ^ e->invert.x))
 
 	return MATCH_ADDR(saddr, smask, (union xt_policy_addr *)&x->props.saddr) &&
-	       MATCH_ADDR(daddr, dmask, (union xt_policy_addr *)&x->id.daddr.a4) &&
+	       MATCH_ADDR(daddr, dmask, (union xt_policy_addr *)&x->id.daddr) &&
 	       MATCH(proto, x->id.proto) &&
 	       MATCH(mode, x->props.mode) &&
 	       MATCH(spi, x->id.spi) &&