| @@ -3,6 +3,10 @@ · 2 unmodified lines |
| 3 | * Copyright IBM Corporation, 2012 | 3 | * Copyright IBM Corporation, 2012 |
| 4 | * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | 4 | * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> |
| 5 | * | 5 | * |
| | 6 | * Cgroup v2 |
| | 7 | * Copyright (C) 2019 Red Hat, Inc. |
| | 8 | * Author: Giuseppe Scrivano <gscrivan@redhat.com> |
| | 9 | * |
| 6 | * This program is free software; you can redistribute it and/or modify it | 10 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of version 2.1 of the GNU Lesser General Public License | 11 | * under the terms of version 2.1 of the GNU Lesser General Public License |
| 8 | * as published by the Free Software Foundation. | 12 | * as published by the Free Software Foundation. |
| @@ -19,18 +23,36 @@ · 10 unmodified lines |
| 19 | #include <linux/hugetlb.h> | 23 | #include <linux/hugetlb.h> |
| 20 | #include <linux/hugetlb_cgroup.h> | 24 | #include <linux/hugetlb_cgroup.h> |
| 21 | | 25 | |
| | 26 | enum hugetlb_memory_event { |
| | 27 | HUGETLB_MAX, |
| | 28 | HUGETLB_NR_MEMORY_EVENTS, |
| | 29 | }; |
| | 30 | |
| 22 | struct hugetlb_cgroup { | 31 | struct hugetlb_cgroup { |
| 23 | struct cgroup_subsys_state css; | 32 | struct cgroup_subsys_state css; |
| | 33 | |
| 24 | /* | 34 | /* |
| 25 | * the counter to account for hugepages from hugetlb. | 35 | * the counter to account for hugepages from hugetlb. |
| 26 | */ | 36 | */ |
| 27 | struct page_counter hugepage[HUGE_MAX_HSTATE]; | 37 | struct page_counter hugepage[HUGE_MAX_HSTATE]; |
| | 38 | |
| | 39 | atomic_long_t events[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; |
| | 40 | atomic_long_t events_local[HUGE_MAX_HSTATE][HUGETLB_NR_MEMORY_EVENTS]; |
| | 41 | |
| | 42 | /* Handle for "hugetlb.events" */ |
| | 43 | struct cgroup_file events_file[HUGE_MAX_HSTATE]; |
| | 44 | |
| | 45 | /* Handle for "hugetlb.events.local" */ |
| | 46 | struct cgroup_file events_local_file[HUGE_MAX_HSTATE]; |
| 28 | }; | 47 | }; |
| 29 | | 48 | |
| 30 | #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) | 49 | #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) |
| 31 | #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) | 50 | #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) |
| 32 | #define MEMFILE_ATTR(val) ((val) & 0xffff) | 51 | #define MEMFILE_ATTR(val) ((val) & 0xffff) |
| 33 | | 52 | |
| | 53 | #define hugetlb_cgroup_from_counter(counter, idx) \ |
| | 54 | container_of(counter, struct hugetlb_cgroup, hugepage[idx]) |
| | 55 | |
| 34 | static struct hugetlb_cgroup *root_h_cgroup __read_mostly; | 56 | static struct hugetlb_cgroup *root_h_cgroup __read_mostly; |
| 35 | | 57 | |
| 36 | static inline | 58 | static inline |
| @@ -178,6 +200,19 @@ static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) · 141 unmodified lines |
| 178 | } while (hugetlb_cgroup_have_usage(h_cg)); | 200 | } while (hugetlb_cgroup_have_usage(h_cg)); |
| 179 | } | 201 | } |
| 180 | | 202 | |
| | 203 | static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx, |
| | 204 | enum hugetlb_memory_event event) |
| | 205 | { |
| | 206 | atomic_long_inc(&hugetlb->events_local[idx][event]); |
| | 207 | cgroup_file_notify(&hugetlb->events_local_file[idx]); |
| | 208 | |
| | 209 | do { |
| | 210 | atomic_long_inc(&hugetlb->events[idx][event]); |
| | 211 | cgroup_file_notify(&hugetlb->events_file[idx]); |
| | 212 | } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) && |
| | 213 | !hugetlb_cgroup_is_root(hugetlb)); |
| | 214 | } |
| | 215 | |
| 181 | int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, | 216 | int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, |
| 182 | struct hugetlb_cgroup **ptr) | 217 | struct hugetlb_cgroup **ptr) |
| 183 | { | 218 | { |
| @@ -202,8 +237,12 @@ again: · 18 unmodified lines |
| 202 | } | 237 | } |
| 203 | rcu_read_unlock(); | 238 | rcu_read_unlock(); |
| 204 | | 239 | |
| 205 | if (!page_counter_try_charge(&h_cg->hugepage[idx], nr_pages, &counter)) | 240 | if (!page_counter_try_charge(&h_cg->hugepage[idx], nr_pages, |
| | 241 | &counter)) { |
| 206 | ret = -ENOMEM; | 242 | ret = -ENOMEM; |
| | 243 | hugetlb_event(hugetlb_cgroup_from_counter(counter, idx), idx, |
| | 244 | HUGETLB_MAX); |
| | 245 | } |
| 207 | css_put(&h_cg->css); | 246 | css_put(&h_cg->css); |
| 208 | done: | 247 | done: |
| 209 | *ptr = h_cg; | 248 | *ptr = h_cg; |
| @@ -283,10 +322,45 @@ static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, · 73 unmodified lines |
| 283 | } | 322 | } |
| 284 | } | 323 | } |
| 285 | | 324 | |
| | 325 | static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v) |
| | 326 | { |
| | 327 | int idx; |
| | 328 | u64 val; |
| | 329 | struct cftype *cft = seq_cft(seq); |
| | 330 | unsigned long limit; |
| | 331 | struct page_counter *counter; |
| | 332 | struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); |
| | 333 | |
| | 334 | idx = MEMFILE_IDX(cft->private); |
| | 335 | counter = &h_cg->hugepage[idx]; |
| | 336 | |
| | 337 | limit = round_down(PAGE_COUNTER_MAX, |
| | 338 | 1 << huge_page_order(&hstates[idx])); |
| | 339 | |
| | 340 | switch (MEMFILE_ATTR(cft->private)) { |
| | 341 | case RES_USAGE: |
| | 342 | val = (u64)page_counter_read(counter); |
| | 343 | seq_printf(seq, "%llu\n", val * PAGE_SIZE); |
| | 344 | break; |
| | 345 | case RES_LIMIT: |
| | 346 | val = (u64)counter->max; |
| | 347 | if (val == limit) |
| | 348 | seq_puts(seq, "max\n"); |
| | 349 | else |
| | 350 | seq_printf(seq, "%llu\n", val * PAGE_SIZE); |
| | 351 | break; |
| | 352 | default: |
| | 353 | BUG(); |
| | 354 | } |
| | 355 | |
| | 356 | return 0; |
| | 357 | } |
| | 358 | |
| 286 | static DEFINE_MUTEX(hugetlb_limit_mutex); | 359 | static DEFINE_MUTEX(hugetlb_limit_mutex); |
| 287 | | 360 | |
| 288 | static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, | 361 | static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, |
| 289 | char *buf, size_t nbytes, loff_t off) | 362 | char *buf, size_t nbytes, loff_t off, |
| | 363 | const char *max) |
| 290 | { | 364 | { |
| 291 | int ret, idx; | 365 | int ret, idx; |
| 292 | unsigned long nr_pages; | 366 | unsigned long nr_pages; |
| @@ -296,7 +370,7 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, · 3 unmodified lines |
| 296 | return -EINVAL; | 370 | return -EINVAL; |
| 297 | | 371 | |
| 298 | buf = strstrip(buf); | 372 | buf = strstrip(buf); |
| 299 | ret = page_counter_memparse(buf, "-1", &nr_pages); | 373 | ret = page_counter_memparse(buf, max, &nr_pages); |
| 300 | if (ret) | 374 | if (ret) |
| 301 | return ret; | 375 | return ret; |
| 302 | | 376 | |
| @@ -316,6 +390,18 @@ static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, · 13 unmodified lines |
| 316 | return ret ?: nbytes; | 390 | return ret ?: nbytes; |
| 317 | } | 391 | } |
| 318 | | 392 | |
| | 393 | static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of, |
| | 394 | char *buf, size_t nbytes, loff_t off) |
| | 395 | { |
| | 396 | return hugetlb_cgroup_write(of, buf, nbytes, off, "-1"); |
| | 397 | } |
| | 398 | |
| | 399 | static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of, |
| | 400 | char *buf, size_t nbytes, loff_t off) |
| | 401 | { |
| | 402 | return hugetlb_cgroup_write(of, buf, nbytes, off, "max"); |
| | 403 | } |
| | 404 | |
| 319 | static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, | 405 | static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, |
| 320 | char *buf, size_t nbytes, loff_t off) | 406 | char *buf, size_t nbytes, loff_t off) |
| 321 | { | 407 | { |
| @@ -350,7 +436,36 @@ static char *mem_fmt(char *buf, int size, unsigned long hsize) · 28 unmodified lines |
| 350 | return buf; | 436 | return buf; |
| 351 | } | 437 | } |
| 352 | | 438 | |
| 353 | static void __init __hugetlb_cgroup_file_init(int idx) | 439 | static int __hugetlb_events_show(struct seq_file *seq, bool local) |
| | 440 | { |
| | 441 | int idx; |
| | 442 | long max; |
| | 443 | struct cftype *cft = seq_cft(seq); |
| | 444 | struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq)); |
| | 445 | |
| | 446 | idx = MEMFILE_IDX(cft->private); |
| | 447 | |
| | 448 | if (local) |
| | 449 | max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]); |
| | 450 | else |
| | 451 | max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]); |
| | 452 | |
| | 453 | seq_printf(seq, "max %lu\n", max); |
| | 454 | |
| | 455 | return 0; |
| | 456 | } |
| | 457 | |
| | 458 | static int hugetlb_events_show(struct seq_file *seq, void *v) |
| | 459 | { |
| | 460 | return __hugetlb_events_show(seq, false); |
| | 461 | } |
| | 462 | |
| | 463 | static int hugetlb_events_local_show(struct seq_file *seq, void *v) |
| | 464 | { |
| | 465 | return __hugetlb_events_show(seq, true); |
| | 466 | } |
| | 467 | |
| | 468 | static void __init __hugetlb_cgroup_file_dfl_init(int idx) |
| 354 | { | 469 | { |
| 355 | char buf[32]; | 470 | char buf[32]; |
| 356 | struct cftype *cft; | 471 | struct cftype *cft; |
| @@ -360,38 +475,93 @@ static void __init __hugetlb_cgroup_file_init(int idx) · 3 unmodified lines |
| 360 | mem_fmt(buf, 32, huge_page_size(h)); | 475 | mem_fmt(buf, 32, huge_page_size(h)); |
| 361 | | 476 | |
| 362 | /* Add the limit file */ | 477 | /* Add the limit file */ |
| 363 | cft = &h->cgroup_files[0]; | 478 | cft = &h->cgroup_files_dfl[0]; |
| | 479 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max", buf); |
| | 480 | cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); |
| | 481 | cft->seq_show = hugetlb_cgroup_read_u64_max; |
| | 482 | cft->write = hugetlb_cgroup_write_dfl; |
| | 483 | cft->flags = CFTYPE_NOT_ON_ROOT; |
| | 484 | |
| | 485 | /* Add the current usage file */ |
| | 486 | cft = &h->cgroup_files_dfl[1]; |
| | 487 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.current", buf); |
| | 488 | cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); |
| | 489 | cft->seq_show = hugetlb_cgroup_read_u64_max; |
| | 490 | cft->flags = CFTYPE_NOT_ON_ROOT; |
| | 491 | |
| | 492 | /* Add the events file */ |
| | 493 | cft = &h->cgroup_files_dfl[2]; |
| | 494 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events", buf); |
| | 495 | cft->private = MEMFILE_PRIVATE(idx, 0); |
| | 496 | cft->seq_show = hugetlb_events_show; |
| | 497 | cft->file_offset = offsetof(struct hugetlb_cgroup, events_file[idx]), |
| | 498 | cft->flags = CFTYPE_NOT_ON_ROOT; |
| | 499 | |
| | 500 | /* Add the events.local file */ |
| | 501 | cft = &h->cgroup_files_dfl[3]; |
| | 502 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events.local", buf); |
| | 503 | cft->private = MEMFILE_PRIVATE(idx, 0); |
| | 504 | cft->seq_show = hugetlb_events_local_show; |
| | 505 | cft->file_offset = offsetof(struct hugetlb_cgroup, |
| | 506 | events_local_file[idx]), |
| | 507 | cft->flags = CFTYPE_NOT_ON_ROOT; |
| | 508 | |
| | 509 | /* NULL terminate the last cft */ |
| | 510 | cft = &h->cgroup_files_dfl[4]; |
| | 511 | memset(cft, 0, sizeof(*cft)); |
| | 512 | |
| | 513 | WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys, |
| | 514 | h->cgroup_files_dfl)); |
| | 515 | } |
| | 516 | |
| | 517 | static void __init __hugetlb_cgroup_file_legacy_init(int idx) |
| | 518 | { |
| | 519 | char buf[32]; |
| | 520 | struct cftype *cft; |
| | 521 | struct hstate *h = &hstates[idx]; |
| | 522 | |
| | 523 | /* format the size */ |
| | 524 | mem_fmt(buf, 32, huge_page_size(h)); |
| | 525 | |
| | 526 | /* Add the limit file */ |
| | 527 | cft = &h->cgroup_files_legacy[0]; |
| 364 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf); | 528 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf); |
| 365 | cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); | 529 | cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); |
| 366 | cft->read_u64 = hugetlb_cgroup_read_u64; | 530 | cft->read_u64 = hugetlb_cgroup_read_u64; |
| 367 | cft->write = hugetlb_cgroup_write; | 531 | cft->write = hugetlb_cgroup_write_legacy; |
| 368 | | 532 | |
| 369 | /* Add the usage file */ | 533 | /* Add the usage file */ |
| 370 | cft = &h->cgroup_files[1]; | 534 | cft = &h->cgroup_files_legacy[1]; |
| 371 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf); | 535 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf); |
| 372 | cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); | 536 | cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); |
| 373 | cft->read_u64 = hugetlb_cgroup_read_u64; | 537 | cft->read_u64 = hugetlb_cgroup_read_u64; |
| 374 | | 538 | |
| 375 | /* Add the MAX usage file */ | 539 | /* Add the MAX usage file */ |
| 376 | cft = &h->cgroup_files[2]; | 540 | cft = &h->cgroup_files_legacy[2]; |
| 377 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf); | 541 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf); |
| 378 | cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE); | 542 | cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE); |
| 379 | cft->write = hugetlb_cgroup_reset; | 543 | cft->write = hugetlb_cgroup_reset; |
| 380 | cft->read_u64 = hugetlb_cgroup_read_u64; | 544 | cft->read_u64 = hugetlb_cgroup_read_u64; |
| 381 | | 545 | |
| 382 | /* Add the failcntfile */ | 546 | /* Add the failcntfile */ |
| 383 | cft = &h->cgroup_files[3]; | 547 | cft = &h->cgroup_files_legacy[3]; |
| 384 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf); | 548 | snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf); |
| 385 | cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); | 549 | cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); |
| 386 | cft->write = hugetlb_cgroup_reset; | 550 | cft->write = hugetlb_cgroup_reset; |
| 387 | cft->read_u64 = hugetlb_cgroup_read_u64; | 551 | cft->read_u64 = hugetlb_cgroup_read_u64; |
| 388 | | 552 | |
| 389 | /* NULL terminate the last cft */ | 553 | /* NULL terminate the last cft */ |
| 390 | cft = &h->cgroup_files[4]; | 554 | cft = &h->cgroup_files_legacy[4]; |
| 391 | memset(cft, 0, sizeof(*cft)); | 555 | memset(cft, 0, sizeof(*cft)); |
| 392 | | 556 | |
| 393 | WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, | 557 | WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, |
| 394 | h->cgroup_files)); | 558 | h->cgroup_files_legacy)); |
| | 559 | } |
| | 560 | |
| | 561 | static void __init __hugetlb_cgroup_file_init(int idx) |
| | 562 | { |
| | 563 | __hugetlb_cgroup_file_dfl_init(idx); |
| | 564 | __hugetlb_cgroup_file_legacy_init(idx); |
| 395 | } | 565 | } |
| 396 | | 566 | |
| 397 | void __init hugetlb_cgroup_file_init(void) | 567 | void __init hugetlb_cgroup_file_init(void) |
| @@ -433,8 +603,14 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) · 35 unmodified lines |
| 433 | return; | 603 | return; |
| 434 | } | 604 | } |
| 435 | | 605 | |
| | 606 | static struct cftype hugetlb_files[] = { |
| | 607 | {} /* terminate */ |
| | 608 | }; |
| | 609 | |
| 436 | struct cgroup_subsys hugetlb_cgrp_subsys = { | 610 | struct cgroup_subsys hugetlb_cgrp_subsys = { |
| 437 | .css_alloc = hugetlb_cgroup_css_alloc, | 611 | .css_alloc = hugetlb_cgroup_css_alloc, |
| 438 | .css_offline = hugetlb_cgroup_css_offline, | 612 | .css_offline = hugetlb_cgroup_css_offline, |
| 439 | .css_free = hugetlb_cgroup_css_free, | 613 | .css_free = hugetlb_cgroup_css_free, |
| | 614 | .dfl_cftypes = hugetlb_files, |
| | 615 | .legacy_cftypes = hugetlb_files, |
| 440 | }; | 616 | }; |