Introduce task_vma bpf_iter to print memory information of a process. It can be used to print customized information similar to /proc/<pid>/maps. Current /proc/<pid>/maps and /proc/<pid>/smaps provide information of vma's of a process. However, these information are not flexible enough to cover all use cases. For example, if a vma cover mixed 2MB pages and 4kB pages (x86_64), there is no easy way to tell which address ranges are backed by 2MB pages. task_vma solves the problem by enabling the user to generate customize information based on the vma (and vma->vm_mm, vma->vm_file, etc.). To access the vma safely in the BPF program, task_vma iterator holds target mmap_lock while calling the BPF program. If the mmap_lock is contended, task_vma unlocks mmap_lock between iterations to unblock the writer(s). This lock contention avoidance mechanism is similar to the one used in show_smaps_rollup().
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines | ||
| 286 | 286 | .show = task_file_seq_show, |
| 287 | 287 | }; |
| 288 | 288 | |
| 289 | +struct bpf_iter_seq_task_vma_info { | |
| 290 | + /* The first field must be struct bpf_iter_seq_task_common. | |
| 291 | + * this is assumed by {init, fini}_seq_pidns() callback functions. | |
| 292 | + */ | |
| 293 | + struct bpf_iter_seq_task_common common; | |
| 294 | + struct task_struct *task; | |
| 295 | + struct vm_area_struct *vma; | |
| 296 | + u32 tid; | |
| 297 | + unsigned long prev_vm_start; | |
| 298 | + unsigned long prev_vm_end; | |
| 299 | +}; | |
| 300 | + | |
| 301 | +enum bpf_task_vma_iter_find_op { | |
| 302 | + task_vma_iter_first_vma, /* use mm->mmap */ | |
| 303 | + task_vma_iter_next_vma, /* use curr_vma->vm_next */ | |
| 304 | + task_vma_iter_find_vma, /* use find_vma() to find next vma */ | |
| 305 | +}; | |
| 306 | + | |
| 307 | +static struct vm_area_struct * | |
| 308 | +task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) | |
| 309 | +{ | |
| 310 | + struct pid_namespace *ns = info->common.ns; | |
| 311 | + enum bpf_task_vma_iter_find_op op; | |
| 312 | + struct vm_area_struct *curr_vma; | |
| 313 | + struct task_struct *curr_task; | |
| 314 | + u32 curr_tid = info->tid; | |
| 315 | + | |
| 316 | + /* If this function returns a non-NULL vma, it holds a reference to | |
| 317 | + * the task_struct, and holds read lock on vma->mm->mmap_lock. | |
| 318 | + * If this function returns NULL, it does not hold any reference or | |
| 319 | + * lock. | |
| 320 | + */ | |
| 321 | + if (info->task) { | |
| 322 | + curr_task = info->task; | |
| 323 | + curr_vma = info->vma; | |
| 324 | + /* In case of lock contention, drop mmap_lock to unblock | |
| 325 | + * the writer. | |
| 326 | + * | |
| 327 | + * After relock, call find(mm, prev_vm_end - 1) to find | |
| 328 | + * new vma to process. | |
| 329 | + * | |
| 330 | + * +------+------+-----------+ | |
| 331 | + * | VMA1 | VMA2 | VMA3 | | |
| 332 | + * +------+------+-----------+ | |
| 333 | + * | | | | | |
| 334 | + * 4k 8k 16k 400k | |
| 335 | + * | |
| 336 | + * For example, curr_vma == VMA2. Before unlock, we set | |
| 337 | + * | |
| 338 | + * prev_vm_start = 8k | |
| 339 | + * prev_vm_end = 16k | |
| 340 | + * | |
| 341 | + * There are a few cases: | |
| 342 | + * | |
| 343 | + * 1) VMA2 is freed, but VMA3 exists. | |
| 344 | + * | |
| 345 | + * find_vma() will return VMA3, just process VMA3. | |
| 346 | + * | |
| 347 | + * 2) VMA2 still exists. | |
| 348 | + * | |
| 349 | + * find_vma() will return VMA2, process VMA2->next. | |
| 350 | + * | |
| 351 | + * 3) no more vma in this mm. | |
| 352 | + * | |
| 353 | + * Process the next task. | |
| 354 | + * | |
| 355 | + * 4) find_vma() returns a different vma, VMA2'. | |
| 356 | + * | |
| 357 | + * 4.1) If VMA2 covers same range as VMA2', skip VMA2', | |
| 358 | + * because we already covered the range; | |
| 359 | + * 4.2) VMA2 and VMA2' covers different ranges, process | |
| 360 | + * VMA2'. | |
| 361 | + */ | |
| 362 | + if (mmap_lock_is_contended(curr_task->mm)) { | |
| 363 | + info->prev_vm_start = curr_vma->vm_start; | |
| 364 | + info->prev_vm_end = curr_vma->vm_end; | |
| 365 | + op = task_vma_iter_find_vma; | |
| 366 | + mmap_read_unlock(curr_task->mm); | |
| 367 | + if (mmap_read_lock_killable(curr_task->mm)) | |
| 368 | + goto finish; | |
| 369 | + } else { | |
| 370 | + op = task_vma_iter_next_vma; | |
| 371 | + } | |
| 372 | + } else { | |
| 373 | +again: | |
| 374 | + curr_task = task_seq_get_next(ns, &curr_tid, true); | |
| 375 | + if (!curr_task) { | |
| 376 | + info->tid = curr_tid + 1; | |
| 377 | + goto finish; | |
| 378 | + } | |
| 379 | + | |
| 380 | + if (curr_tid != info->tid) { | |
| 381 | + info->tid = curr_tid; | |
| 382 | + /* new task, process the first vma */ | |
| 383 | + op = task_vma_iter_first_vma; | |
| 384 | + } else { | |
| 385 | + /* Found the same tid, which means the user space | |
| 386 | + * finished data in previous buffer and read more. | |
| 387 | + * We dropped mmap_lock before returning to user | |
| 388 | + * space, so it is necessary to use find_vma() to | |
| 389 | + * find the next vma to process. | |
| 390 | + */ | |
| 391 | + op = task_vma_iter_find_vma; | |
| 392 | + } | |
| 393 | + | |
| 394 | + if (!curr_task->mm) | |
| 395 | + goto next_task; | |
| 396 | + | |
| 397 | + if (mmap_read_lock_killable(curr_task->mm)) | |
| 398 | + goto finish; | |
| 399 | + } | |
| 400 | + | |
| 401 | + switch (op) { | |
| 402 | + case task_vma_iter_first_vma: | |
| 403 | + curr_vma = curr_task->mm->mmap; | |
| 404 | + break; | |
| 405 | + case task_vma_iter_next_vma: | |
| 406 | + curr_vma = curr_vma->vm_next; | |
| 407 | + break; | |
| 408 | + case task_vma_iter_find_vma: | |
| 409 | + /* We dropped mmap_lock so it is necessary to use find_vma | |
| 410 | + * to find the next vma. This is similar to the mechanism | |
| 411 | + * in show_smaps_rollup(). | |
| 412 | + */ | |
| 413 | + curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1); | |
| 414 | + /* case 1) and 4.2) above just use curr_vma */ | |
| 415 | + | |
| 416 | + /* check for case 2) or case 4.1) above */ | |
| 417 | + if (curr_vma && | |
| 418 | + curr_vma->vm_start == info->prev_vm_start && | |
| 419 | + curr_vma->vm_end == info->prev_vm_end) | |
| 420 | + curr_vma = curr_vma->vm_next; | |
| 421 | + break; | |
| 422 | + } | |
| 423 | + if (!curr_vma) { | |
| 424 | + /* case 3) above, or case 2) 4.1) with vma->next == NULL */ | |
| 425 | + mmap_read_unlock(curr_task->mm); | |
| 426 | + goto next_task; | |
| 427 | + } | |
| 428 | + info->task = curr_task; | |
| 429 | + info->vma = curr_vma; | |
| 430 | + return curr_vma; | |
| 431 | + | |
| 432 | +next_task: | |
| 433 | + put_task_struct(curr_task); | |
| 434 | + info->task = NULL; | |
| 435 | + curr_tid++; | |
| 436 | + goto again; | |
| 437 | + | |
| 438 | +finish: | |
| 439 | + if (curr_task) | |
| 440 | + put_task_struct(curr_task); | |
| 441 | + info->task = NULL; | |
| 442 | + info->vma = NULL; | |
| 443 | + return NULL; | |
| 444 | +} | |
| 445 | + | |
| 446 | +static void *task_vma_seq_start(struct seq_file *seq, loff_t *pos) | |
| 447 | +{ | |
| 448 | + struct bpf_iter_seq_task_vma_info *info = seq->private; | |
| 449 | + struct vm_area_struct *vma; | |
| 450 | + | |
| 451 | + vma = task_vma_seq_get_next(info); | |
| 452 | + if (vma && *pos == 0) | |
| 453 | + ++*pos; | |
| 454 | + | |
| 455 | + return vma; | |
| 456 | +} | |
| 457 | + | |
| 458 | +static void *task_vma_seq_next(struct seq_file *seq, void *v, loff_t *pos) | |
| 459 | +{ | |
| 460 | + struct bpf_iter_seq_task_vma_info *info = seq->private; | |
| 461 | + | |
| 462 | + ++*pos; | |
| 463 | + return task_vma_seq_get_next(info); | |
| 464 | +} | |
| 465 | + | |
| 466 | +struct bpf_iter__task_vma { | |
| 467 | + __bpf_md_ptr(struct bpf_iter_meta *, meta); | |
| 468 | + __bpf_md_ptr(struct task_struct *, task); | |
| 469 | + __bpf_md_ptr(struct vm_area_struct *, vma); | |
| 470 | +}; | |
| 471 | + | |
| 472 | +DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta, | |
| 473 | + struct task_struct *task, struct vm_area_struct *vma) | |
| 474 | + | |
| 475 | +static int __task_vma_seq_show(struct seq_file *seq, bool in_stop) | |
| 476 | +{ | |
| 477 | + struct bpf_iter_seq_task_vma_info *info = seq->private; | |
| 478 | + struct bpf_iter__task_vma ctx; | |
| 479 | + struct bpf_iter_meta meta; | |
| 480 | + struct bpf_prog *prog; | |
| 481 | + | |
| 482 | + meta.seq = seq; | |
| 483 | + prog = bpf_iter_get_info(&meta, in_stop); | |
| 484 | + if (!prog) | |
| 485 | + return 0; | |
| 486 | + | |
| 487 | + ctx.meta = &meta; | |
| 488 | + ctx.task = info->task; | |
| 489 | + ctx.vma = info->vma; | |
| 490 | + return bpf_iter_run_prog(prog, &ctx); | |
| 491 | +} | |
| 492 | + | |
| 493 | +static int task_vma_seq_show(struct seq_file *seq, void *v) | |
| 494 | +{ | |
| 495 | + return __task_vma_seq_show(seq, false); | |
| 496 | +} | |
| 497 | + | |
| 498 | +static void task_vma_seq_stop(struct seq_file *seq, void *v) | |
| 499 | +{ | |
| 500 | + struct bpf_iter_seq_task_vma_info *info = seq->private; | |
| 501 | + | |
| 502 | + if (!v) { | |
| 503 | + (void)__task_vma_seq_show(seq, true); | |
| 504 | + } else { | |
| 505 | + /* info->vma has not been seen by the BPF program. If the | |
| 506 | + * user space reads more, task_vma_seq_get_next should | |
| 507 | + * return this vma again. Set prev_vm_start to ~0UL, | |
| 508 | + * so that we don't skip the vma returned by the next | |
| 509 | + * find_vma() (case task_vma_iter_find_vma in | |
| 510 | + * task_vma_seq_get_next()). | |
| 511 | + */ | |
| 512 | + info->prev_vm_start = ~0UL; | |
| 513 | + info->prev_vm_end = info->vma->vm_end; | |
| 514 | + mmap_read_unlock(info->task->mm); | |
| 515 | + put_task_struct(info->task); | |
| 516 | + info->task = NULL; | |
| 517 | + } | |
| 518 | +} | |
| 519 | + | |
| 520 | +static const struct seq_operations task_vma_seq_ops = { | |
| 521 | + .start = task_vma_seq_start, | |
| 522 | + .next = task_vma_seq_next, | |
| 523 | + .stop = task_vma_seq_stop, | |
| 524 | + .show = task_vma_seq_show, | |
| 525 | +}; | |
| 526 | + | |
| 289 | 527 | BTF_ID_LIST(btf_task_file_ids) |
| 290 | 528 | BTF_ID(struct, task_struct) |
| 291 | 529 | BTF_ID(struct, file) |
| 530 | +BTF_ID(struct, vm_area_struct) | |
| 292 | 531 | |
| 293 | 532 | static const struct bpf_iter_seq_info task_seq_info = { |
| 294 | 533 | .seq_ops = &task_seq_ops, |
| @@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines | ||
| 328 | 567 | .seq_info = &task_file_seq_info, |
| 329 | 568 | }; |
| 330 | 569 | |
| 570 | +static const struct bpf_iter_seq_info task_vma_seq_info = { | |
| 571 | + .seq_ops = &task_vma_seq_ops, | |
| 572 | + .init_seq_private = init_seq_pidns, | |
| 573 | + .fini_seq_private = fini_seq_pidns, | |
| 574 | + .seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info), | |
| 575 | +}; | |
| 576 | + | |
| 577 | +static struct bpf_iter_reg task_vma_reg_info = { | |
| 578 | + .target = "task_vma", | |
| 579 | + .feature = BPF_ITER_RESCHED, | |
| 580 | + .ctx_arg_info_size = 2, | |
| 581 | + .ctx_arg_info = { | |
| 582 | + { offsetof(struct bpf_iter__task_vma, task), | |
| 583 | + PTR_TO_BTF_ID_OR_NULL }, | |
| 584 | + { offsetof(struct bpf_iter__task_vma, vma), | |
| 585 | + PTR_TO_BTF_ID_OR_NULL }, | |
| 586 | + }, | |
| 587 | + .seq_info = &task_vma_seq_info, | |
| 588 | +}; | |
| 589 | + | |
| 331 | 590 | static int __init task_iter_init(void) |
| 332 | 591 | { |
| 333 | 592 | int ret; |
| @@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines | ||
| 339 | 598 | |
| 340 | 599 | task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0]; |
| 341 | 600 | task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1]; |
| 342 | − return bpf_iter_reg_target(&task_file_reg_info); | |
| 601 | + ret = bpf_iter_reg_target(&task_file_reg_info); | |
| 602 | + if (ret) | |
| 603 | + return ret; | |
| 604 | + | |
| 605 | + task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0]; | |
| 606 | + task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2]; | |
| 607 | + return bpf_iter_reg_target(&task_vma_reg_info); | |
| 343 | 608 | } |
| 344 | 609 | late_initcall(task_iter_init); |