| @@ -25,6 +25,8 @@ · 24 unmodified lines |
| 25 | #include <linux/hugetlb.h> | 25 | #include <linux/hugetlb.h> |
| 26 | #include <linux/vmalloc.h> | 26 | #include <linux/vmalloc.h> |
| 27 | #include <linux/srcu.h> | 27 | #include <linux/srcu.h> |
| | 28 | #include <linux/anon_inodes.h> |
| | 29 | #include <linux/file.h> |
| 28 | | 30 | |
| 29 | #include <asm/tlbflush.h> | 31 | #include <asm/tlbflush.h> |
| 30 | #include <asm/kvm_ppc.h> | 32 | #include <asm/kvm_ppc.h> |
| @@ -1145,6 +1147,348 @@ void kvmppc_unpin_guest_page(struct kvm *kvm, void *va) · 1114 unmodified lines |
| 1145 | put_page(page); | 1147 | put_page(page); |
| 1146 | } | 1148 | } |
| 1147 | | 1149 | |
| | 1150 | /* |
| | 1151 | * Functions for reading and writing the hash table via reads and |
| | 1152 | * writes on a file descriptor. |
| | 1153 | * |
| | 1154 | * Reads return the guest view of the hash table, which has to be |
| | 1155 | * pieced together from the real hash table and the guest_rpte |
| | 1156 | * values in the revmap array. |
| | 1157 | * |
| | 1158 | * On writes, each HPTE written is considered in turn, and if it |
| | 1159 | * is valid, it is written to the HPT as if an H_ENTER with the |
| | 1160 | * exact flag set was done. When the invalid count is non-zero |
| | 1161 | * in the header written to the stream, the kernel will make |
| | 1162 | * sure that that many HPTEs are invalid, and invalidate them |
| | 1163 | * if not. |
| | 1164 | */ |
| | 1165 | |
| | 1166 | struct kvm_htab_ctx { |
| | 1167 | unsigned long index; |
| | 1168 | unsigned long flags; |
| | 1169 | struct kvm *kvm; |
| | 1170 | int first_pass; |
| | 1171 | }; |
| | 1172 | |
| | 1173 | #define HPTE_SIZE (2 * sizeof(unsigned long)) |
| | 1174 | |
| | 1175 | static long record_hpte(unsigned long flags, unsigned long *hptp, |
| | 1176 | unsigned long *hpte, struct revmap_entry *revp, |
| | 1177 | int want_valid, int first_pass) |
| | 1178 | { |
| | 1179 | unsigned long v, r; |
| | 1180 | int ok = 1; |
| | 1181 | int valid, dirty; |
| | 1182 | |
| | 1183 | /* Unmodified entries are uninteresting except on the first pass */ |
| | 1184 | dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED); |
| | 1185 | if (!first_pass && !dirty) |
| | 1186 | return 0; |
| | 1187 | |
| | 1188 | valid = 0; |
| | 1189 | if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) { |
| | 1190 | valid = 1; |
| | 1191 | if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && |
| | 1192 | !(hptp[0] & HPTE_V_BOLTED)) |
| | 1193 | valid = 0; |
| | 1194 | } |
| | 1195 | if (valid != want_valid) |
| | 1196 | return 0; |
| | 1197 | |
| | 1198 | v = r = 0; |
| | 1199 | if (valid || dirty) { |
| | 1200 | /* lock the HPTE so it's stable and read it */ |
| | 1201 | preempt_disable(); |
| | 1202 | while (!try_lock_hpte(hptp, HPTE_V_HVLOCK)) |
| | 1203 | cpu_relax(); |
| | 1204 | v = hptp[0]; |
| | 1205 | if (v & HPTE_V_ABSENT) { |
| | 1206 | v &= ~HPTE_V_ABSENT; |
| | 1207 | v |= HPTE_V_VALID; |
| | 1208 | } |
| | 1209 | /* re-evaluate valid and dirty from synchronized HPTE value */ |
| | 1210 | valid = !!(v & HPTE_V_VALID); |
| | 1211 | if ((flags & KVM_GET_HTAB_BOLTED_ONLY) && !(v & HPTE_V_BOLTED)) |
| | 1212 | valid = 0; |
| | 1213 | r = revp->guest_rpte | (hptp[1] & (HPTE_R_R | HPTE_R_C)); |
| | 1214 | dirty = !!(revp->guest_rpte & HPTE_GR_MODIFIED); |
| | 1215 | /* only clear modified if this is the right sort of entry */ |
| | 1216 | if (valid == want_valid && dirty) { |
| | 1217 | r &= ~HPTE_GR_MODIFIED; |
| | 1218 | revp->guest_rpte = r; |
| | 1219 | } |
| | 1220 | asm volatile(PPC_RELEASE_BARRIER "" : : : "memory"); |
| | 1221 | hptp[0] &= ~HPTE_V_HVLOCK; |
| | 1222 | preempt_enable(); |
| | 1223 | if (!(valid == want_valid && (first_pass || dirty))) |
| | 1224 | ok = 0; |
| | 1225 | } |
| | 1226 | hpte[0] = v; |
| | 1227 | hpte[1] = r; |
| | 1228 | return ok; |
| | 1229 | } |
| | 1230 | |
| | 1231 | static ssize_t kvm_htab_read(struct file *file, char __user *buf, |
| | 1232 | size_t count, loff_t *ppos) |
| | 1233 | { |
| | 1234 | struct kvm_htab_ctx *ctx = file->private_data; |
| | 1235 | struct kvm *kvm = ctx->kvm; |
| | 1236 | struct kvm_get_htab_header hdr; |
| | 1237 | unsigned long *hptp; |
| | 1238 | struct revmap_entry *revp; |
| | 1239 | unsigned long i, nb, nw; |
| | 1240 | unsigned long __user *lbuf; |
| | 1241 | struct kvm_get_htab_header __user *hptr; |
| | 1242 | unsigned long flags; |
| | 1243 | int first_pass; |
| | 1244 | unsigned long hpte[2]; |
| | 1245 | |
| | 1246 | if (!access_ok(VERIFY_WRITE, buf, count)) |
| | 1247 | return -EFAULT; |
| | 1248 | |
| | 1249 | first_pass = ctx->first_pass; |
| | 1250 | flags = ctx->flags; |
| | 1251 | |
| | 1252 | i = ctx->index; |
| | 1253 | hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); |
| | 1254 | revp = kvm->arch.revmap + i; |
| | 1255 | lbuf = (unsigned long __user *)buf; |
| | 1256 | |
| | 1257 | nb = 0; |
| | 1258 | while (nb + sizeof(hdr) + HPTE_SIZE < count) { |
| | 1259 | /* Initialize header */ |
| | 1260 | hptr = (struct kvm_get_htab_header __user *)buf; |
| | 1261 | hdr.index = i; |
| | 1262 | hdr.n_valid = 0; |
| | 1263 | hdr.n_invalid = 0; |
| | 1264 | nw = nb; |
| | 1265 | nb += sizeof(hdr); |
| | 1266 | lbuf = (unsigned long __user *)(buf + sizeof(hdr)); |
| | 1267 | |
| | 1268 | /* Skip uninteresting entries, i.e. clean on not-first pass */ |
| | 1269 | if (!first_pass) { |
| | 1270 | while (i < kvm->arch.hpt_npte && |
| | 1271 | !(revp->guest_rpte & HPTE_GR_MODIFIED)) { |
| | 1272 | ++i; |
| | 1273 | hptp += 2; |
| | 1274 | ++revp; |
| | 1275 | } |
| | 1276 | } |
| | 1277 | |
| | 1278 | /* Grab a series of valid entries */ |
| | 1279 | while (i < kvm->arch.hpt_npte && |
| | 1280 | hdr.n_valid < 0xffff && |
| | 1281 | nb + HPTE_SIZE < count && |
| | 1282 | record_hpte(flags, hptp, hpte, revp, 1, first_pass)) { |
| | 1283 | /* valid entry, write it out */ |
| | 1284 | ++hdr.n_valid; |
| | 1285 | if (__put_user(hpte[0], lbuf) || |
| | 1286 | __put_user(hpte[1], lbuf + 1)) |
| | 1287 | return -EFAULT; |
| | 1288 | nb += HPTE_SIZE; |
| | 1289 | lbuf += 2; |
| | 1290 | ++i; |
| | 1291 | hptp += 2; |
| | 1292 | ++revp; |
| | 1293 | } |
| | 1294 | /* Now skip invalid entries while we can */ |
| | 1295 | while (i < kvm->arch.hpt_npte && |
| | 1296 | hdr.n_invalid < 0xffff && |
| | 1297 | record_hpte(flags, hptp, hpte, revp, 0, first_pass)) { |
| | 1298 | /* found an invalid entry */ |
| | 1299 | ++hdr.n_invalid; |
| | 1300 | ++i; |
| | 1301 | hptp += 2; |
| | 1302 | ++revp; |
| | 1303 | } |
| | 1304 | |
| | 1305 | if (hdr.n_valid || hdr.n_invalid) { |
| | 1306 | /* write back the header */ |
| | 1307 | if (__copy_to_user(hptr, &hdr, sizeof(hdr))) |
| | 1308 | return -EFAULT; |
| | 1309 | nw = nb; |
| | 1310 | buf = (char __user *)lbuf; |
| | 1311 | } else { |
| | 1312 | nb = nw; |
| | 1313 | } |
| | 1314 | |
| | 1315 | /* Check if we've wrapped around the hash table */ |
| | 1316 | if (i >= kvm->arch.hpt_npte) { |
| | 1317 | i = 0; |
| | 1318 | ctx->first_pass = 0; |
| | 1319 | break; |
| | 1320 | } |
| | 1321 | } |
| | 1322 | |
| | 1323 | ctx->index = i; |
| | 1324 | |
| | 1325 | return nb; |
| | 1326 | } |
| | 1327 | |
| | 1328 | static ssize_t kvm_htab_write(struct file *file, const char __user *buf, |
| | 1329 | size_t count, loff_t *ppos) |
| | 1330 | { |
| | 1331 | struct kvm_htab_ctx *ctx = file->private_data; |
| | 1332 | struct kvm *kvm = ctx->kvm; |
| | 1333 | struct kvm_get_htab_header hdr; |
| | 1334 | unsigned long i, j; |
| | 1335 | unsigned long v, r; |
| | 1336 | unsigned long __user *lbuf; |
| | 1337 | unsigned long *hptp; |
| | 1338 | unsigned long tmp[2]; |
| | 1339 | ssize_t nb; |
| | 1340 | long int err, ret; |
| | 1341 | int rma_setup; |
| | 1342 | |
| | 1343 | if (!access_ok(VERIFY_READ, buf, count)) |
| | 1344 | return -EFAULT; |
| | 1345 | |
| | 1346 | /* lock out vcpus from running while we're doing this */ |
| | 1347 | mutex_lock(&kvm->lock); |
| | 1348 | rma_setup = kvm->arch.rma_setup_done; |
| | 1349 | if (rma_setup) { |
| | 1350 | kvm->arch.rma_setup_done = 0; /* temporarily */ |
| | 1351 | /* order rma_setup_done vs. vcpus_running */ |
| | 1352 | smp_mb(); |
| | 1353 | if (atomic_read(&kvm->arch.vcpus_running)) { |
| | 1354 | kvm->arch.rma_setup_done = 1; |
| | 1355 | mutex_unlock(&kvm->lock); |
| | 1356 | return -EBUSY; |
| | 1357 | } |
| | 1358 | } |
| | 1359 | |
| | 1360 | err = 0; |
| | 1361 | for (nb = 0; nb + sizeof(hdr) <= count; ) { |
| | 1362 | err = -EFAULT; |
| | 1363 | if (__copy_from_user(&hdr, buf, sizeof(hdr))) |
| | 1364 | break; |
| | 1365 | |
| | 1366 | err = 0; |
| | 1367 | if (nb + hdr.n_valid * HPTE_SIZE > count) |
| | 1368 | break; |
| | 1369 | |
| | 1370 | nb += sizeof(hdr); |
| | 1371 | buf += sizeof(hdr); |
| | 1372 | |
| | 1373 | err = -EINVAL; |
| | 1374 | i = hdr.index; |
| | 1375 | if (i >= kvm->arch.hpt_npte || |
| | 1376 | i + hdr.n_valid + hdr.n_invalid > kvm->arch.hpt_npte) |
| | 1377 | break; |
| | 1378 | |
| | 1379 | hptp = (unsigned long *)(kvm->arch.hpt_virt + (i * HPTE_SIZE)); |
| | 1380 | lbuf = (unsigned long __user *)buf; |
| | 1381 | for (j = 0; j < hdr.n_valid; ++j) { |
| | 1382 | err = -EFAULT; |
| | 1383 | if (__get_user(v, lbuf) || __get_user(r, lbuf + 1)) |
| | 1384 | goto out; |
| | 1385 | err = -EINVAL; |
| | 1386 | if (!(v & HPTE_V_VALID)) |
| | 1387 | goto out; |
| | 1388 | lbuf += 2; |
| | 1389 | nb += HPTE_SIZE; |
| | 1390 | |
| | 1391 | if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) |
| | 1392 | kvmppc_do_h_remove(kvm, 0, i, 0, tmp); |
| | 1393 | err = -EIO; |
| | 1394 | ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r, |
| | 1395 | tmp); |
| | 1396 | if (ret != H_SUCCESS) { |
| | 1397 | pr_err("kvm_htab_write ret %ld i=%ld v=%lx " |
| | 1398 | "r=%lx\n", ret, i, v, r); |
| | 1399 | goto out; |
| | 1400 | } |
| | 1401 | if (!rma_setup && is_vrma_hpte(v)) { |
| | 1402 | unsigned long psize = hpte_page_size(v, r); |
| | 1403 | unsigned long senc = slb_pgsize_encoding(psize); |
| | 1404 | unsigned long lpcr; |
| | 1405 | |
| | 1406 | kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T | |
| | 1407 | (VRMA_VSID << SLB_VSID_SHIFT_1T); |
| | 1408 | lpcr = kvm->arch.lpcr & ~LPCR_VRMASD; |
| | 1409 | lpcr |= senc << (LPCR_VRMASD_SH - 4); |
| | 1410 | kvm->arch.lpcr = lpcr; |
| | 1411 | rma_setup = 1; |
| | 1412 | } |
| | 1413 | ++i; |
| | 1414 | hptp += 2; |
| | 1415 | } |
| | 1416 | |
| | 1417 | for (j = 0; j < hdr.n_invalid; ++j) { |
| | 1418 | if (hptp[0] & (HPTE_V_VALID | HPTE_V_ABSENT)) |
| | 1419 | kvmppc_do_h_remove(kvm, 0, i, 0, tmp); |
| | 1420 | ++i; |
| | 1421 | hptp += 2; |
| | 1422 | } |
| | 1423 | err = 0; |
| | 1424 | } |
| | 1425 | |
| | 1426 | out: |
| | 1427 | /* Order HPTE updates vs. rma_setup_done */ |
| | 1428 | smp_wmb(); |
| | 1429 | kvm->arch.rma_setup_done = rma_setup; |
| | 1430 | mutex_unlock(&kvm->lock); |
| | 1431 | |
| | 1432 | if (err) |
| | 1433 | return err; |
| | 1434 | return nb; |
| | 1435 | } |
| | 1436 | |
| | 1437 | static int kvm_htab_release(struct inode *inode, struct file *filp) |
| | 1438 | { |
| | 1439 | struct kvm_htab_ctx *ctx = filp->private_data; |
| | 1440 | |
| | 1441 | filp->private_data = NULL; |
| | 1442 | if (!(ctx->flags & KVM_GET_HTAB_WRITE)) |
| | 1443 | atomic_dec(&ctx->kvm->arch.hpte_mod_interest); |
| | 1444 | kvm_put_kvm(ctx->kvm); |
| | 1445 | kfree(ctx); |
| | 1446 | return 0; |
| | 1447 | } |
| | 1448 | |
| | 1449 | static struct file_operations kvm_htab_fops = { |
| | 1450 | .read = kvm_htab_read, |
| | 1451 | .write = kvm_htab_write, |
| | 1452 | .llseek = default_llseek, |
| | 1453 | .release = kvm_htab_release, |
| | 1454 | }; |
| | 1455 | |
| | 1456 | int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf) |
| | 1457 | { |
| | 1458 | int ret; |
| | 1459 | struct kvm_htab_ctx *ctx; |
| | 1460 | int rwflag; |
| | 1461 | |
| | 1462 | /* reject flags we don't recognize */ |
| | 1463 | if (ghf->flags & ~(KVM_GET_HTAB_BOLTED_ONLY | KVM_GET_HTAB_WRITE)) |
| | 1464 | return -EINVAL; |
| | 1465 | ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); |
| | 1466 | if (!ctx) |
| | 1467 | return -ENOMEM; |
| | 1468 | kvm_get_kvm(kvm); |
| | 1469 | ctx->kvm = kvm; |
| | 1470 | ctx->index = ghf->start_index; |
| | 1471 | ctx->flags = ghf->flags; |
| | 1472 | ctx->first_pass = 1; |
| | 1473 | |
| | 1474 | rwflag = (ghf->flags & KVM_GET_HTAB_WRITE) ? O_WRONLY : O_RDONLY; |
| | 1475 | ret = anon_inode_getfd("kvm-htab", &kvm_htab_fops, ctx, rwflag); |
| | 1476 | if (ret < 0) { |
| | 1477 | kvm_put_kvm(kvm); |
| | 1478 | return ret; |
| | 1479 | } |
| | 1480 | |
| | 1481 | if (rwflag == O_RDONLY) { |
| | 1482 | mutex_lock(&kvm->slots_lock); |
| | 1483 | atomic_inc(&kvm->arch.hpte_mod_interest); |
| | 1484 | /* make sure kvmppc_do_h_enter etc. see the increment */ |
| | 1485 | synchronize_srcu_expedited(&kvm->srcu); |
| | 1486 | mutex_unlock(&kvm->slots_lock); |
| | 1487 | } |
| | 1488 | |
| | 1489 | return ret; |
| | 1490 | } |
| | 1491 | |
| 1148 | void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu) | 1492 | void kvmppc_mmu_book3s_hv_init(struct kvm_vcpu *vcpu) |
| 1149 | { | 1493 | { |
| 1150 | struct kvmppc_mmu *mmu = &vcpu->arch.mmu; | 1494 | struct kvmppc_mmu *mmu = &vcpu->arch.mmu; |