| @@ -25,6 +25,8 @@ · 24 unmodified lines |
| 25 | 25 | #include <linux/hugetlb.h> |
| 26 | 26 | #include <linux/vmalloc.h> |
| 27 | 27 | #include <linux/srcu.h> |
| 28 | +#include <linux/anon_inodes.h> |
| 29 | +#include <linux/file.h> |
| 28 | 30 | |
| 29 | 31 | #include <asm/tlbflush.h> |
| 30 | 32 | #include <asm/kvm_ppc.h> |
| @@ -1145,6 +1147,348 @@ void kvmppc_unpin_guest_page(struct kvm *kvm, void *va) · 1114 unmodified lines |
| 1145 | 1147 | put_page(page); |
| 1146 | 1148 | } |
| 1147 | 1149 | |
| 1150 | +/* |
| 1151 | + * Functions for reading and writing the hash table via reads and |
| 1152 | + * writes on a file descriptor. |
| 1153 | + * |
| 1154 | + * Reads return the guest view of the hash table, which has to be |
| 1155 | + * pieced together from the real hash table and the guest_rpte |
| 1156 | + * values in the revmap array. |
| 1157 | + * |
| 1158 | + * On writes, each HPTE written is considered in turn, and if it |
| 1159 | + * is valid, it is written to the HPT as if an H_ENTER with the |
| 1160 | + * exact flag set was done. When the invalid count is non-zero |
| 1161 | + * in the header written to the stream, the kernel will make |
| 1162 | + * sure that that many HPTEs are invalid, and invalidate them |
| 1163 | + * if not. |
| 1164 | + */ |
| 1165 | + |
| 1166 | +struct kvm_htab_ctx { |
| 1167 | + unsigned long index; |
| 1168 | + unsigned long flags; |
| 1169 | + struct kvm *kvm; |
| 1170 | + int first_pass; |
| 1171 | +}; |
| 1172 | + |
| 1173 | +#define HPTE_SIZE (2 * sizeof(unsigned long)) |
| 1174 | + |
| 1175 | +static long record_hpte(unsigned long flags, unsigned long *hptp, |
| 1176 | + unsigned long *hpte, struct revmap_entry *revp, |
| 1177 | + int want_valid, int first_pass) |
| 1178 | +{ |
| 1179 | + unsigned long v, r; |
| 1180 | + int ok = 1; |
| 1181 | + int valid, dirty; |
| 1182 | + |
| 1183 | + /* Unmodified entries are uninteresting except on the first pass */ |
| 1184 | + dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED); |
| 1185 | + if (!first_pass && !dirty) |
| 1186 | + return 0; |
| 1187 | + |
| 1188 | + valid = 0; |
| 1189 | + if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) { |
| 1190 | + valid = 1; |
| 1191 | + if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && |
| 1192 | + !(hptp[0] & HPTE_V_BOLTED)) |
| 1193 | + valid = 0; |
| 1194 | + } |
| 1195 | + if (valid != want_valid) |
| 1196 | + return 0; |
| 1197 | + |
| 1198 | + v = r = 0; |
| 1199 | + if (valid || dirty) { |
| 1200 | + /* lock the HPTE so it's stable and read it */ |
| 1201 | + preempt_disable(); |
| 1202 | + while (!try_lock_hpte(hptp, HPTE_V_HVLOCK)) |
| 1203 | + cpu_relax(); |
| 1204 | + v = hptp[0]; |
| 1205 | + if (v & HPTE_V_ABSENT) { |
| 1206 | + v &= ~HPTE_V_ABSENT; |
| 1207 | + v |= HPTE_V_VALID; |
| 1208 | + } |
| 1209 | + /* re-evaluate valid and dirty from synchronized HPTE value */ |
| 1210 | + valid = !!(v & HPTE_V_VALID); |
| 1211 | + if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED)) |
| 1212 | + valid = 0; |
| 1213 | + r = revp->guest_rpte | (hptp[1] & (HPTE_R_R | HPTE_R_C)); |
| 1214 | + dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED); |
| 1215 | + /* only clear modified if this is the right sort of entry */ |
| 1216 | + if (valid == want_valid && dirty) { |
| 1217 | + r &= ~HPTE_GR_MODIFIED; |
| 1218 | + revp->guest_rpte = r; |
| 1219 | + } |
| 1220 | + asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); |
| 1221 | + hptp[0] &= ~HPTE_V_HVLOCK; |
| 1222 | + preempt_enable(); |
| 1223 | + if (!(valid == want_valid && (first_pass || dirty))) |
| 1224 | + ok = 0; |
| 1225 | + } |
| 1226 | + hpte[0] = v; |
| 1227 | + hpte[1] = r; |
| 1228 | + return ok; |
| 1229 | +} |
| 1230 | + |
| 1231 | +static ssize_t kvm_htab_read(struct file *file, char __user *buf, |
| 1232 | + size_t count, loff_t *ppos) |
| 1233 | +{ |
| 1234 | + struct kvm_htab_ctx *ctx = file->private_data; |
| 1235 | + struct kvm *kvm = ctx->kvm; |
| 1236 | + struct kvm_get_htab_header hdr; |
| 1237 | + unsigned long *hptp; |
| 1238 | + struct revmap_entry *revp; |
| 1239 | + unsigned long i, nb, nw; |
| 1240 | + unsigned long __user *lbuf; |
| 1241 | + struct kvm_get_htab_header __user *hptr; |
| 1242 | + unsigned long flags; |
| 1243 | + int first_pass; |
| 1244 | + unsigned long hpte[2]; |
| 1245 | + |
| 1246 | + if (!access_ok(VERIFY_WRITE, buf, count)) |
| 1247 | + return -EFAULT; |
| 1248 | + |
| 1249 | + first_pass = ctx->first_pass; |
| 1250 | + flags = ctx->flags; |
| 1251 | + |
| 1252 | + i = ctx->index; |
| 1253 | + hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); |
| 1254 | + revp = kvm->arch.revmap + i; |
| 1255 | + lbuf = (unsigned long __user *)buf; |
| 1256 | + |
| 1257 | + nb = 0; |
| 1258 | + while (nb + sizeof(hdr) + HPTE_SIZE < count) { |
| 1259 | + /* Initialize header */ |
| 1260 | + hptr = (struct kvm_get_htab_header __user *)buf; |
| 1261 | + hdr.index = i; |
| 1262 | + hdr.n_valid = 0; |
| 1263 | + hdr.n_invalid = 0; |
| 1264 | + nw = nb; |
| 1265 | + nb += sizeof(hdr); |
| 1266 | + lbuf = (unsigned long __user *)(buf + sizeof(hdr)); |
| 1267 | + |
| 1268 | + /* Skip uninteresting entries, i.e. clean on not-first pass */ |
| 1269 | + if (!first_pass) { |
| 1270 | + while (i < kvm->arch.hpt_npte && |
| 1271 | + !(revp->guest_rpte & HPTE_GR_MODIFIED)) { |
| 1272 | + ++i; |
| 1273 | + hptp += 2; |
| 1274 | + ++revp; |
| 1275 | + } |
| 1276 | + } |
| 1277 | + |
| 1278 | + /* Grab a series of valid entries */ |
| 1279 | + while (i < kvm->arch.hpt_npte && |
| 1280 | + hdr.n_valid < 0xffff && |
| 1281 | + nb + HPTE_SIZE < count && |
| 1282 | + record_hpte(flags, hptp, hpte, revp, 1, first_pass)) { |
| 1283 | + /* valid entry, write it out */ |
| 1284 | + ++hdr.n_valid; |
| 1285 | + if (__put_user(hpte[0], lbuf) || |
| 1286 | + __put_user(hpte[1], lbuf + 1)) |
| 1287 | + return -EFAULT; |
| 1288 | + nb += HPTE_SIZE; |
| 1289 | + lbuf += 2; |
| 1290 | + ++i; |
| 1291 | + hptp += 2; |
| 1292 | + ++revp; |
| 1293 | + } |
| 1294 | + /* Now skip invalid entries while we can */ |
| 1295 | + while (i < kvm->arch.hpt_npte && |
| 1296 | + hdr.n_invalid < 0xffff && |
| 1297 | + record_hpte(flags, hptp, hpte, revp, 0, first_pass)) { |
| 1298 | + /* found an invalid entry */ |
| 1299 | + ++hdr.n_invalid; |
| 1300 | + ++i; |
| 1301 | + hptp += 2; |
| 1302 | + ++revp; |
| 1303 | + } |
| 1304 | + |
| 1305 | + if (hdr.n_valid || hdr.n_invalid) { |
| 1306 | + /* write back the header */ |
| 1307 | + if (__copy_to_user(hptr, &hdr, sizeof(hdr))) |
| 1308 | + return -EFAULT; |
| 1309 | + nw = nb; |
| 1310 | + buf = (char __user *)lbuf; |
| 1311 | + } else { |
| 1312 | + nb = nw; |
| 1313 | + } |
| 1314 | + |
| 1315 | + /* Check if we've wrapped around the hash table */ |
| 1316 | + if (i >= kvm->arch.hpt_npte) { |
| 1317 | + i = 0; |
| 1318 | + ctx->first_pass = 0; |
| 1319 | + break; |
| 1320 | + } |
| 1321 | + } |
| 1322 | + |
| 1323 | + ctx->index = i; |
| 1324 | + |
| 1325 | + return nb; |
| 1326 | +} |
| 1327 | + |
| 1328 | +static ssize_t kvm_htab_write(struct file *file, const char __user *buf, |
| 1329 | + size_t count, loff_t *ppos) |
| 1330 | +{ |
| 1331 | + struct kvm_htab_ctx *ctx = file->private_data; |
| 1332 | + struct kvm *kvm = ctx->kvm; |
| 1333 | + struct kvm_get_htab_header hdr; |
| 1334 | + unsigned long i, j; |
| 1335 | + unsigned long v, r; |
| 1336 | + unsigned long __user *lbuf; |
| 1337 | + unsigned long *hptp; |
| 1338 | + unsigned long tmp[2]; |
| 1339 | + ssize_t nb; |
| 1340 | + long int err, ret; |
| 1341 | + int rma_setup; |
| 1342 | + |
| 1343 | + if (!access_ok(VERIFY_READ, buf, count)) |
| 1344 | + return -EFAULT; |
| 1345 | + |
| 1346 | + /* lock out vcpus from running while we're doing this */ |
| 1347 | + mutex_lock(&kvm->lock); |
| 1348 | + rma_setup = kvm->arch.rma_setup_done; |
| 1349 | + if (rma_setup) { |
| 1350 | + kvm->arch.rma_setup_done = 0; /* temporarily */ |
| 1351 | + /* order rma_setup_done vs. vcpus_running */ |
| 1352 | + smp_mb(); |
| 1353 | + if (atomic_read(&kvm->arch.vcpus_running)) { |
| 1354 | + kvm->arch.rma_setup_done = 1; |
| 1355 | + mutex_unlock(&kvm->lock); |
| 1356 | + return -EBUSY; |
| 1357 | + } |
| 1358 | + } |
| 1359 | + |
| 1360 | + err = 0; |
| 1361 | + for (nb = 0; nb + sizeof(hdr) <= count; ) { |
| 1362 | + err = -EFAULT; |
| 1363 | + if (__copy_from_user(&hdr, buf, sizeof(hdr))) |
| 1364 | + break; |
| 1365 | + |
| 1366 | + err = 0; |
| 1367 | + if (nb + hdr.n_valid * HPTE_SIZE > count) |
| 1368 | + break; |
| 1369 | + |
| 1370 | + nb += sizeof(hdr); |
| 1371 | + buf += sizeof(hdr); |
| 1372 | + |
| 1373 | + err = -EINVAL; |
| 1374 | + i = hdr.index; |
| 1375 | + if (i >= kvm->arch.hpt_npte || |
| 1376 | + i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte) |
| 1377 | + break; |
| 1378 | + |
| 1379 | + hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); |
| 1380 | + lbuf = (unsigned long __user *)buf; |
| 1381 | + for (j = 0; j < hdr.n_valid; ++j) { |
| 1382 | + err = -EFAULT; |
| 1383 | + if (__get_user(v, lbuf) || __get_user(r, lbuf + 1)) |
| 1384 | + goto out; |
| 1385 | + err = -EINVAL; |
| 1386 | + if (!(v & HPTE_V_VALID)) |
| 1387 | + goto out; |
| 1388 | + lbuf += 2; |
| 1389 | + nb += HPTE_SIZE; |
| 1390 | + |
| 1391 | + if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) |
| 1392 | + kvmppc_do_h_remove(kvm, 0, i, 0, tmp); |
| 1393 | + err = -EIO; |
| 1394 | + ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r, |
| 1395 | + tmp); |
| 1396 | + if (ret != H_SUCCESS) { |
| 1397 | + pr_err("kvm_htab_write ret %ld i=%ld v=%lx " |
| 1398 | + "r=%lx\n", ret, i, v, r); |
| 1399 | + goto out; |
| 1400 | + } |
| 1401 | + if (!rma_setup && is_vrma_hpte(v)) { |
| 1402 | + unsigned long psize = hpte_page_size(v, r); |
| 1403 | + unsigned long senc = slb_pgsize_encoding(psize); |
| 1404 | + unsigned long lpcr; |
| 1405 | + |
| 1406 | + kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | |
| 1407 | + (VRMA_VSID << SLB_VSID_SHIFT_1T); |
| 1408 | + lpcr = kvm->arch.lpcr & ~LPCR_VRMASD; |
| 1409 | + lpcr |= senc << (LPCR_VRMASD_SH - 4); |
| 1410 | + kvm->arch.lpcr = lpcr; |
| 1411 | + rma_setup = 1; |
| 1412 | + } |
| 1413 | + ++i; |
| 1414 | + hptp += 2; |
| 1415 | + } |
| 1416 | + |
| 1417 | + for (j = 0; j < hdr.n_invalid; ++j) { |
| 1418 | + if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) |
| 1419 | + kvmppc_do_h_remove(kvm, 0, i, 0, tmp); |
| 1420 | + ++i; |
| 1421 | + hptp += 2; |
| 1422 | + } |
| 1423 | + err = 0; |
| 1424 | + } |
| 1425 | + |
| 1426 | + out: |
| 1427 | + /* Order HPTE updates vs. rma_setup_done */ |
| 1428 | + smp_wmb(); |
| 1429 | + kvm->arch.rma_setup_done = rma_setup; |
| 1430 | + mutex_unlock(&kvm->lock); |
| 1431 | + |
| 1432 | + if (err) |
| 1433 | + return err; |
| 1434 | + return nb; |
| 1435 | +} |
| 1436 | + |
| 1437 | +static int kvm_htab_release(struct inode *inode, struct file *filp) |
| 1438 | +{ |
| 1439 | + struct kvm_htab_ctx *ctx = filp->private_data; |
| 1440 | + |
| 1441 | + filp->private_data = NULL; |
| 1442 | + if (!(ctx->flags & KVM_GET_HTAB_WRITE)) |
| 1443 | + atomic_dec(&ctx->kvm->arch.hpte_mod_interest); |
| 1444 | + kvm_put_kvm(ctx->kvm); |
| 1445 | + kfree(ctx); |
| 1446 | + return 0; |
| 1447 | +} |
| 1448 | + |
| 1449 | +static struct file_operations kvm_htab_fops = { |
| 1450 | + .read = kvm_htab_read, |
| 1451 | + .write = kvm_htab_write, |
| 1452 | + .llseek = default_llseek, |
| 1453 | + .release = kvm_htab_release, |
| 1454 | +}; |
| 1455 | + |
| 1456 | +int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf) |
| 1457 | +{ |
| 1458 | + int ret; |
| 1459 | + struct kvm_htab_ctx *ctx; |
| 1460 | + int rwflag; |
| 1461 | + |
| 1462 | + /* reject flags we don't recognize */ |
| 1463 | + if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE)) |
| 1464 | + return -EINVAL; |
| 1465 | + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); |
| 1466 | + if (!ctx) |
| 1467 | + return -ENOMEM; |
| 1468 | + kvm_get_kvm(kvm); |
| 1469 | + ctx->kvm = kvm; |
| 1470 | + ctx->index = ghf->start_index; |
| 1471 | + ctx->flags = ghf->flags; |
| 1472 | + ctx->first_pass = 1; |
| 1473 | + |
| 1474 | + rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY; |
| 1475 | + ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag); |
| 1476 | + if (ret < 0) { |
| 1477 | + kvm_put_kvm(kvm); |
| 1478 | + return ret; |
| 1479 | + } |
| 1480 | + |
| 1481 | + if (rwflag == O_RDONLY) { |
| 1482 | + mutex_lock(&kvm->slots_lock); |
| 1483 | + atomic_inc(&kvm->arch.hpte_mod_interest); |
| 1484 | + /* make sure kvmppc_do_h_enter etc. see the increment */ |
| 1485 | + synchronize_srcu_expedited(&kvm->srcu); |
| 1486 | + mutex_unlock(&kvm->slots_lock); |
| 1487 | + } |
| 1488 | + |
| 1489 | + return ret; |
| 1490 | +} |
| 1491 | + |
| 1148 | 1492 | void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu) |
| 1149 | 1493 | { |
| 1150 | 1494 | struct kvmppc_mmu *mmu = &vcpu->arch.mmu; |