Introduce task_vma bpf_iter to print memory information of a process. It can be used to print customized information similar to /proc/<pid>/maps. Current /proc/<pid>/maps and /proc/<pid>/smaps provide information of vma's of a process. However, these information are not flexible enough to cover all use cases. For example, if a vma cover mixed 2MB pages and 4kB pages (x86_64), there is no easy way to tell which address ranges are backed by 2MB pages. task_vma solves the problem by enabling the user to generate customize information based on the vma (and vma->vm_mm, vma->vm_file, etc.). To access the vma safely in the BPF program, task_vma iterator holds target mmap_lock while calling the BPF program. If the mmap_lock is contended, task_vma unlocks mmap_lock between iterations to unblock the writer(s). This lock contention avoidance mechanism is similar to the one used in show_smaps_rollup().
Unified · Split
| @@ -286,9 +286,248 @@ static const struct seq_operations task_file_seq_ops = { · 285 unmodified lines | |||
| 286 | .show = task_file_seq_show, | 286 | .show = task_file_seq_show, |
| 287 | }; | 287 | }; |
| 288 | 288 | ||
| 289 | struct bpf_iter_seq_task_vma_info { | ||
| 290 | /* The first field must be struct bpf_iter_seq_task_common. | ||
| 291 | * this is assumed by {init, fini}_seq_pidns() callback functions. | ||
| 292 | */ | ||
| 293 | struct bpf_iter_seq_task_common common; | ||
| 294 | struct task_struct *task; | ||
| 295 | struct vm_area_struct *vma; | ||
| 296 | u32 tid; | ||
| 297 | unsigned long prev_vm_start; | ||
| 298 | unsigned long prev_vm_end; | ||
| 299 | }; | ||
| 300 | |||
| 301 | enum bpf_task_vma_iter_find_op { | ||
| 302 | task_vma_iter_first_vma, /* use mm->mmap */ | ||
| 303 | task_vma_iter_next_vma, /* use curr_vma->vm_next */ | ||
| 304 | task_vma_iter_find_vma, /* use find_vma() to find next vma */ | ||
| 305 | }; | ||
| 306 | |||
| 307 | static struct vm_area_struct * | ||
| 308 | task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info) | ||
| 309 | { | ||
| 310 | struct pid_namespace *ns = info->common.ns; | ||
| 311 | enum bpf_task_vma_iter_find_op op; | ||
| 312 | struct vm_area_struct *curr_vma; | ||
| 313 | struct task_struct *curr_task; | ||
| 314 | u32 curr_tid = info->tid; | ||
| 315 | |||
| 316 | /* If this function returns a non-NULL vma, it holds a reference to | ||
| 317 | * the task_struct, and holds read lock on vma->mm->mmap_lock. | ||
| 318 | * If this function returns NULL, it does not hold any reference or | ||
| 319 | * lock. | ||
| 320 | */ | ||
| 321 | if (info->task) { | ||
| 322 | curr_task = info->task; | ||
| 323 | curr_vma = info->vma; | ||
| 324 | /* In case of lock contention, drop mmap_lock to unblock | ||
| 325 | * the writer. | ||
| 326 | * | ||
| 327 | * After relock, call find(mm, prev_vm_end - 1) to find | ||
| 328 | * new vma to process. | ||
| 329 | * | ||
| 330 | * +------+------+-----------+ | ||
| 331 | * | VMA1 | VMA2 | VMA3 | | ||
| 332 | * +------+------+-----------+ | ||
| 333 | * | | | | | ||
| 334 | * 4k 8k 16k 400k | ||
| 335 | * | ||
| 336 | * For example, curr_vma == VMA2. Before unlock, we set | ||
| 337 | * | ||
| 338 | * prev_vm_start = 8k | ||
| 339 | * prev_vm_end = 16k | ||
| 340 | * | ||
| 341 | * There are a few cases: | ||
| 342 | * | ||
| 343 | * 1) VMA2 is freed, but VMA3 exists. | ||
| 344 | * | ||
| 345 | * find_vma() will return VMA3, just process VMA3. | ||
| 346 | * | ||
| 347 | * 2) VMA2 still exists. | ||
| 348 | * | ||
| 349 | * find_vma() will return VMA2, process VMA2->next. | ||
| 350 | * | ||
| 351 | * 3) no more vma in this mm. | ||
| 352 | * | ||
| 353 | * Process the next task. | ||
| 354 | * | ||
| 355 | * 4) find_vma() returns a different vma, VMA2'. | ||
| 356 | * | ||
| 357 | * 4.1) If VMA2 covers same range as VMA2', skip VMA2', | ||
| 358 | * because we already covered the range; | ||
| 359 | * 4.2) VMA2 and VMA2' covers different ranges, process | ||
| 360 | * VMA2'. | ||
| 361 | */ | ||
| 362 | if (mmap_lock_is_contended(curr_task->mm)) { | ||
| 363 | info->prev_vm_start = curr_vma->vm_start; | ||
| 364 | info->prev_vm_end = curr_vma->vm_end; | ||
| 365 | op = task_vma_iter_find_vma; | ||
| 366 | mmap_read_unlock(curr_task->mm); | ||
| 367 | if (mmap_read_lock_killable(curr_task->mm)) | ||
| 368 | goto finish; | ||
| 369 | } else { | ||
| 370 | op = task_vma_iter_next_vma; | ||
| 371 | } | ||
| 372 | } else { | ||
| 373 | again: | ||
| 374 | curr_task = task_seq_get_next(ns, &curr_tid, true); | ||
| 375 | if (!curr_task) { | ||
| 376 | info->tid = curr_tid + 1; | ||
| 377 | goto finish; | ||
| 378 | } | ||
| 379 | |||
| 380 | if (curr_tid != info->tid) { | ||
| 381 | info->tid = curr_tid; | ||
| 382 | /* new task, process the first vma */ | ||
| 383 | op = task_vma_iter_first_vma; | ||
| 384 | } else { | ||
| 385 | /* Found the same tid, which means the user space | ||
| 386 | * finished data in previous buffer and read more. | ||
| 387 | * We dropped mmap_lock before returning to user | ||
| 388 | * space, so it is necessary to use find_vma() to | ||
| 389 | * find the next vma to process. | ||
| 390 | */ | ||
| 391 | op = task_vma_iter_find_vma; | ||
| 392 | } | ||
| 393 | |||
| 394 | if (!curr_task->mm) | ||
| 395 | goto next_task; | ||
| 396 | |||
| 397 | if (mmap_read_lock_killable(curr_task->mm)) | ||
| 398 | goto finish; | ||
| 399 | } | ||
| 400 | |||
| 401 | switch (op) { | ||
| 402 | case task_vma_iter_first_vma: | ||
| 403 | curr_vma = curr_task->mm->mmap; | ||
| 404 | break; | ||
| 405 | case task_vma_iter_next_vma: | ||
| 406 | curr_vma = curr_vma->vm_next; | ||
| 407 | break; | ||
| 408 | case task_vma_iter_find_vma: | ||
| 409 | /* We dropped mmap_lock so it is necessary to use find_vma | ||
| 410 | * to find the next vma. This is similar to the mechanism | ||
| 411 | * in show_smaps_rollup(). | ||
| 412 | */ | ||
| 413 | curr_vma = find_vma(curr_task->mm, info->prev_vm_end - 1); | ||
| 414 | /* case 1) and 4.2) above just use curr_vma */ | ||
| 415 | |||
| 416 | /* check for case 2) or case 4.1) above */ | ||
| 417 | if (curr_vma && | ||
| 418 | curr_vma->vm_start == info->prev_vm_start && | ||
| 419 | curr_vma->vm_end == info->prev_vm_end) | ||
| 420 | curr_vma = curr_vma->vm_next; | ||
| 421 | break; | ||
| 422 | } | ||
| 423 | if (!curr_vma) { | ||
| 424 | /* case 3) above, or case 2) 4.1) with vma->next == NULL */ | ||
| 425 | mmap_read_unlock(curr_task->mm); | ||
| 426 | goto next_task; | ||
| 427 | } | ||
| 428 | info->task = curr_task; | ||
| 429 | info->vma = curr_vma; | ||
| 430 | return curr_vma; | ||
| 431 | |||
| 432 | next_task: | ||
| 433 | put_task_struct(curr_task); | ||
| 434 | info->task = NULL; | ||
| 435 | curr_tid++; | ||
| 436 | goto again; | ||
| 437 | |||
| 438 | finish: | ||
| 439 | if (curr_task) | ||
| 440 | put_task_struct(curr_task); | ||
| 441 | info->task = NULL; | ||
| 442 | info->vma = NULL; | ||
| 443 | return NULL; | ||
| 444 | } | ||
| 445 | |||
| 446 | static void *task_vma_seq_start(struct seq_file *seq, loff_t *pos) | ||
| 447 | { | ||
| 448 | struct bpf_iter_seq_task_vma_info *info = seq->private; | ||
| 449 | struct vm_area_struct *vma; | ||
| 450 | |||
| 451 | vma = task_vma_seq_get_next(info); | ||
| 452 | if (vma && *pos == 0) | ||
| 453 | ++*pos; | ||
| 454 | |||
| 455 | return vma; | ||
| 456 | } | ||
| 457 | |||
| 458 | static void *task_vma_seq_next(struct seq_file *seq, void *v, loff_t *pos) | ||
| 459 | { | ||
| 460 | struct bpf_iter_seq_task_vma_info *info = seq->private; | ||
| 461 | |||
| 462 | ++*pos; | ||
| 463 | return task_vma_seq_get_next(info); | ||
| 464 | } | ||
| 465 | |||
| 466 | struct bpf_iter__task_vma { | ||
| 467 | __bpf_md_ptr(struct bpf_iter_meta *, meta); | ||
| 468 | __bpf_md_ptr(struct task_struct *, task); | ||
| 469 | __bpf_md_ptr(struct vm_area_struct *, vma); | ||
| 470 | }; | ||
| 471 | |||
| 472 | DEFINE_BPF_ITER_FUNC(task_vma, struct bpf_iter_meta *meta, | ||
| 473 | struct task_struct *task, struct vm_area_struct *vma) | ||
| 474 | |||
| 475 | static int __task_vma_seq_show(struct seq_file *seq, bool in_stop) | ||
| 476 | { | ||
| 477 | struct bpf_iter_seq_task_vma_info *info = seq->private; | ||
| 478 | struct bpf_iter__task_vma ctx; | ||
| 479 | struct bpf_iter_meta meta; | ||
| 480 | struct bpf_prog *prog; | ||
| 481 | |||
| 482 | meta.seq = seq; | ||
| 483 | prog = bpf_iter_get_info(&meta, in_stop); | ||
| 484 | if (!prog) | ||
| 485 | return 0; | ||
| 486 | |||
| 487 | ctx.meta = &meta; | ||
| 488 | ctx.task = info->task; | ||
| 489 | ctx.vma = info->vma; | ||
| 490 | return bpf_iter_run_prog(prog, &ctx); | ||
| 491 | } | ||
| 492 | |||
| 493 | static int task_vma_seq_show(struct seq_file *seq, void *v) | ||
| 494 | { | ||
| 495 | return __task_vma_seq_show(seq, false); | ||
| 496 | } | ||
| 497 | |||
| 498 | static void task_vma_seq_stop(struct seq_file *seq, void *v) | ||
| 499 | { | ||
| 500 | struct bpf_iter_seq_task_vma_info *info = seq->private; | ||
| 501 | |||
| 502 | if (!v) { | ||
| 503 | (void)__task_vma_seq_show(seq, true); | ||
| 504 | } else { | ||
| 505 | /* info->vma has not been seen by the BPF program. If the | ||
| 506 | * user space reads more, task_vma_seq_get_next should | ||
| 507 | * return this vma again. Set prev_vm_start to ~0UL, | ||
| 508 | * so that we don't skip the vma returned by the next | ||
| 509 | * find_vma() (case task_vma_iter_find_vma in | ||
| 510 | * task_vma_seq_get_next()). | ||
| 511 | */ | ||
| 512 | info->prev_vm_start = ~0UL; | ||
| 513 | info->prev_vm_end = info->vma->vm_end; | ||
| 514 | mmap_read_unlock(info->task->mm); | ||
| 515 | put_task_struct(info->task); | ||
| 516 | info->task = NULL; | ||
| 517 | } | ||
| 518 | } | ||
| 519 | |||
| 520 | static const struct seq_operations task_vma_seq_ops = { | ||
| 521 | .start = task_vma_seq_start, | ||
| 522 | .next = task_vma_seq_next, | ||
| 523 | .stop = task_vma_seq_stop, | ||
| 524 | .show = task_vma_seq_show, | ||
| 525 | }; | ||
| 526 | |||
| 289 | BTF_ID_LIST(btf_task_file_ids) | 527 | BTF_ID_LIST(btf_task_file_ids) |
| 290 | BTF_ID(struct, task_struct) | 528 | BTF_ID(struct, task_struct) |
| 291 | BTF_ID(struct, file) | 529 | BTF_ID(struct, file) |
| 530 | BTF_ID(struct, vm_area_struct) | ||
| 292 | 531 | ||
| 293 | static const struct bpf_iter_seq_info task_seq_info = { | 532 | static const struct bpf_iter_seq_info task_seq_info = { |
| 294 | .seq_ops = &task_seq_ops, | 533 | .seq_ops = &task_seq_ops, |
| @@ -328,6 +567,26 @@ static struct bpf_iter_reg task_file_reg_info = { · 33 unmodified lines | |||
| 328 | .seq_info = &task_file_seq_info, | 567 | .seq_info = &task_file_seq_info, |
| 329 | }; | 568 | }; |
| 330 | 569 | ||
| 570 | static const struct bpf_iter_seq_info task_vma_seq_info = { | ||
| 571 | .seq_ops = &task_vma_seq_ops, | ||
| 572 | .init_seq_private = init_seq_pidns, | ||
| 573 | .fini_seq_private = fini_seq_pidns, | ||
| 574 | .seq_priv_size = sizeof(struct bpf_iter_seq_task_vma_info), | ||
| 575 | }; | ||
| 576 | |||
| 577 | static struct bpf_iter_reg task_vma_reg_info = { | ||
| 578 | .target = "task_vma", | ||
| 579 | .feature = BPF_ITER_RESCHED, | ||
| 580 | .ctx_arg_info_size = 2, | ||
| 581 | .ctx_arg_info = { | ||
| 582 | { offsetof(struct bpf_iter__task_vma, task), | ||
| 583 | PTR_TO_BTF_ID_OR_NULL }, | ||
| 584 | { offsetof(struct bpf_iter__task_vma, vma), | ||
| 585 | PTR_TO_BTF_ID_OR_NULL }, | ||
| 586 | }, | ||
| 587 | .seq_info = &task_vma_seq_info, | ||
| 588 | }; | ||
| 589 | |||
| 331 | static int __init task_iter_init(void) | 590 | static int __init task_iter_init(void) |
| 332 | { | 591 | { |
| 333 | int ret; | 592 | int ret; |
| @@ -339,6 +598,12 @@ static int __init task_iter_init(void) · 5 unmodified lines | |||
| 339 | 598 | ||
| 340 | task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0]; | 599 | task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0]; |
| 341 | task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1]; | 600 | task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1]; |
| 342 | return bpf_iter_reg_target(&task_file_reg_info); | 601 | ret = bpf_iter_reg_target(&task_file_reg_info); |
| 602 | if (ret) | ||
| 603 | return ret; | ||
| 604 | |||
| 605 | task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_file_ids[0]; | ||
| 606 | task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[2]; | ||
| 607 | return bpf_iter_reg_target(&task_vma_reg_info); | ||
| 343 | } | 608 | } |
| 344 | late_initcall(task_iter_init); | 609 | late_initcall(task_iter_init); |