Showing the changes to kernel (19 of 906 files). Show all
Unified · Split
| @@ -1101,13 +1101,11 @@ static void audit_log_feature_change(int which, u32 old_feature, u32 new_feature · 1100 unmodified lines | |||
| 1101 | audit_log_end(ab); | 1101 | audit_log_end(ab); |
| 1102 | } | 1102 | } |
| 1103 | 1103 | ||
| 1104 | static int audit_set_feature(struct sk_buff *skb) | 1104 | static int audit_set_feature(struct audit_features *uaf) |
| 1105 | { | 1105 | { |
| 1106 | struct audit_features *uaf; | ||
| 1107 | int i; | 1106 | int i; |
| 1108 | 1107 | ||
| 1109 | BUILD_BUG_ON(AUDIT_LAST_FEATURE + 1 > ARRAY_SIZE(audit_feature_names)); | 1108 | BUILD_BUG_ON(AUDIT_LAST_FEATURE + 1 > ARRAY_SIZE(audit_feature_names)); |
| 1110 | uaf = nlmsg_data(nlmsg_hdr(skb)); | ||
| 1111 | 1109 | ||
| 1112 | /* if there is ever a version 2 we should handle that here */ | 1110 | /* if there is ever a version 2 we should handle that here */ |
| 1113 | 1111 | ||
| @@ -1175,6 +1173,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 61 unmodified lines | |||
| 1175 | { | 1173 | { |
| 1176 | u32 seq; | 1174 | u32 seq; |
| 1177 | void *data; | 1175 | void *data; |
| 1176 | int data_len; | ||
| 1178 | int err; | 1177 | int err; |
| 1179 | struct audit_buffer *ab; | 1178 | struct audit_buffer *ab; |
| 1180 | u16 msg_type = nlh->nlmsg_type; | 1179 | u16 msg_type = nlh->nlmsg_type; |
| @@ -1188,6 +1187,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 7 unmodified lines | |||
| 1188 | 1187 | ||
| 1189 | seq = nlh->nlmsg_seq; | 1188 | seq = nlh->nlmsg_seq; |
| 1190 | data = nlmsg_data(nlh); | 1189 | data = nlmsg_data(nlh); |
| 1190 | data_len = nlmsg_len(nlh); | ||
| 1191 | 1191 | ||
| 1192 | switch (msg_type) { | 1192 | switch (msg_type) { |
| 1193 | case AUDIT_GET: { | 1193 | case AUDIT_GET: { |
| @@ -1211,7 +1211,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 17 unmodified lines | |||
| 1211 | struct audit_status s; | 1211 | struct audit_status s; |
| 1212 | memset(&s, 0, sizeof(s)); | 1212 | memset(&s, 0, sizeof(s)); |
| 1213 | /* guard against past and future API changes */ | 1213 | /* guard against past and future API changes */ |
| 1214 | memcpy(&s, data, min_t(size_t, sizeof(s), nlmsg_len(nlh))); | 1214 | memcpy(&s, data, min_t(size_t, sizeof(s), data_len)); |
| 1215 | if (s.mask & AUDIT_STATUS_ENABLED) { | 1215 | if (s.mask & AUDIT_STATUS_ENABLED) { |
| 1216 | err = audit_set_enabled(s.enabled); | 1216 | err = audit_set_enabled(s.enabled); |
| 1217 | if (err < 0) | 1217 | if (err < 0) |
| @@ -1315,7 +1315,9 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 97 unmodified lines | |||
| 1315 | return err; | 1315 | return err; |
| 1316 | break; | 1316 | break; |
| 1317 | case AUDIT_SET_FEATURE: | 1317 | case AUDIT_SET_FEATURE: |
| 1318 | err = audit_set_feature(skb); | 1318 | if (data_len < sizeof(struct audit_features)) |
| 1319 | return -EINVAL; | ||
| 1320 | err = audit_set_feature(data); | ||
| 1319 | if (err) | 1321 | if (err) |
| 1320 | return err; | 1322 | return err; |
| 1321 | break; | 1323 | break; |
| @@ -1327,6 +1329,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 5 unmodified lines | |||
| 1327 | 1329 | ||
| 1328 | err = audit_filter(msg_type, AUDIT_FILTER_USER); | 1330 | err = audit_filter(msg_type, AUDIT_FILTER_USER); |
| 1329 | if (err == 1) { /* match or error */ | 1331 | if (err == 1) { /* match or error */ |
| 1332 | char *str = data; | ||
| 1333 | |||
| 1330 | err = 0; | 1334 | err = 0; |
| 1331 | if (msg_type == AUDIT_USER_TTY) { | 1335 | if (msg_type == AUDIT_USER_TTY) { |
| 1332 | err = tty_audit_push(); | 1336 | err = tty_audit_push(); |
| @@ -1334,26 +1338,24 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 1 unmodified line | |||
| 1334 | break; | 1338 | break; |
| 1335 | } | 1339 | } |
| 1336 | audit_log_user_recv_msg(&ab, msg_type); | 1340 | audit_log_user_recv_msg(&ab, msg_type); |
| 1337 | if (msg_type != AUDIT_USER_TTY) | 1341 | if (msg_type != AUDIT_USER_TTY) { |
| 1342 | /* ensure NULL termination */ | ||
| 1343 | str[data_len - 1] = '\0'; | ||
| 1338 | audit_log_format(ab, " msg='%.*s'", | 1344 | audit_log_format(ab, " msg='%.*s'", |
| 1339 | AUDIT_MESSAGE_TEXT_MAX, | 1345 | AUDIT_MESSAGE_TEXT_MAX, |
| 1340 | (char *)data); | 1346 | str); |
| 1341 | else { | 1347 | } else { |
| 1342 | int size; | ||
| 1343 | |||
| 1344 | audit_log_format(ab, " data="); | 1348 | audit_log_format(ab, " data="); |
| 1345 | size = nlmsg_len(nlh); | 1349 | if (data_len > 0 && str[data_len - 1] == '\0') |
| 1346 | if (size > 0 && | 1350 | data_len--; |
| 1347 | ((unsigned char *)data)[size - 1] == '\0') | 1351 | audit_log_n_untrustedstring(ab, str, data_len); |
| 1348 | size--; | ||
| 1349 | audit_log_n_untrustedstring(ab, data, size); | ||
| 1350 | } | 1352 | } |
| 1351 | audit_log_end(ab); | 1353 | audit_log_end(ab); |
| 1352 | } | 1354 | } |
| 1353 | break; | 1355 | break; |
| 1354 | case AUDIT_ADD_RULE: | 1356 | case AUDIT_ADD_RULE: |
| 1355 | case AUDIT_DEL_RULE: | 1357 | case AUDIT_DEL_RULE: |
| 1356 | if (nlmsg_len(nlh) < sizeof(struct audit_rule_data)) | 1358 | if (data_len < sizeof(struct audit_rule_data)) |
| 1357 | return -EINVAL; | 1359 | return -EINVAL; |
| 1358 | if (audit_enabled == AUDIT_LOCKED) { | 1360 | if (audit_enabled == AUDIT_LOCKED) { |
| 1359 | audit_log_common_recv_msg(audit_context(), &ab, | 1361 | audit_log_common_recv_msg(audit_context(), &ab, |
| @@ -1365,7 +1367,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 5 unmodified lines | |||
| 1365 | audit_log_end(ab); | 1367 | audit_log_end(ab); |
| 1366 | return -EPERM; | 1368 | return -EPERM; |
| 1367 | } | 1369 | } |
| 1368 | err = audit_rule_change(msg_type, seq, data, nlmsg_len(nlh)); | 1370 | err = audit_rule_change(msg_type, seq, data, data_len); |
| 1369 | break; | 1371 | break; |
| 1370 | case AUDIT_LIST_RULES: | 1372 | case AUDIT_LIST_RULES: |
| 1371 | err = audit_list_rules_send(skb, seq); | 1373 | err = audit_list_rules_send(skb, seq); |
| @@ -1380,7 +1382,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 8 unmodified lines | |||
| 1380 | case AUDIT_MAKE_EQUIV: { | 1382 | case AUDIT_MAKE_EQUIV: { |
| 1381 | void *bufp = data; | 1383 | void *bufp = data; |
| 1382 | u32 sizes[2]; | 1384 | u32 sizes[2]; |
| 1383 | size_t msglen = nlmsg_len(nlh); | 1385 | size_t msglen = data_len; |
| 1384 | char *old, *new; | 1386 | char *old, *new; |
| 1385 | 1387 | ||
| 1386 | err = -EINVAL; | 1388 | err = -EINVAL; |
| @@ -1456,7 +1458,7 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh) · 69 unmodified lines | |||
| 1456 | 1458 | ||
| 1457 | memset(&s, 0, sizeof(s)); | 1459 | memset(&s, 0, sizeof(s)); |
| 1458 | /* guard against past and future API changes */ | 1460 | /* guard against past and future API changes */ |
| 1459 | memcpy(&s, data, min_t(size_t, sizeof(s), nlmsg_len(nlh))); | 1461 | memcpy(&s, data, min_t(size_t, sizeof(s), data_len)); |
| 1460 | /* check if new data is valid */ | 1462 | /* check if new data is valid */ |
| 1461 | if ((s.enabled != 0 && s.enabled != 1) || | 1463 | if ((s.enabled != 0 && s.enabled != 1) || |
| 1462 | (s.log_passwd != 0 && s.log_passwd != 1)) | 1464 | (s.log_passwd != 0 && s.log_passwd != 1)) |
| @@ -456,6 +456,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 455 unmodified lines | |||
| 456 | bufp = data->buf; | 456 | bufp = data->buf; |
| 457 | for (i = 0; i < data->field_count; i++) { | 457 | for (i = 0; i < data->field_count; i++) { |
| 458 | struct audit_field *f = &entry->rule.fields[i]; | 458 | struct audit_field *f = &entry->rule.fields[i]; |
| 459 | u32 f_val; | ||
| 459 | 460 | ||
| 460 | err = -EINVAL; | 461 | err = -EINVAL; |
| 461 | 462 | ||
| @@ -464,12 +465,12 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 2 unmodified lines | |||
| 464 | goto exit_free; | 465 | goto exit_free; |
| 465 | 466 | ||
| 466 | f->type = data->fields[i]; | 467 | f->type = data->fields[i]; |
| 467 | f->val = data->values[i]; | 468 | f_val = data->values[i]; |
| 468 | 469 | ||
| 469 | /* Support legacy tests for a valid loginuid */ | 470 | /* Support legacy tests for a valid loginuid */ |
| 470 | if ((f->type == AUDIT_LOGINUID) && (f->val == AUDIT_UID_UNSET)) { | 471 | if ((f->type == AUDIT_LOGINUID) && (f_val == AUDIT_UID_UNSET)) { |
| 471 | f->type = AUDIT_LOGINUID_SET; | 472 | f->type = AUDIT_LOGINUID_SET; |
| 472 | f->val = 0; | 473 | f_val = 0; |
| 473 | entry->rule.pflags |= AUDIT_LOGINUID_LEGACY; | 474 | entry->rule.pflags |= AUDIT_LOGINUID_LEGACY; |
| 474 | } | 475 | } |
| 475 | 476 | ||
| @@ -485,7 +486,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 9 unmodified lines | |||
| 485 | case AUDIT_SUID: | 486 | case AUDIT_SUID: |
| 486 | case AUDIT_FSUID: | 487 | case AUDIT_FSUID: |
| 487 | case AUDIT_OBJ_UID: | 488 | case AUDIT_OBJ_UID: |
| 488 | f->uid = make_kuid(current_user_ns(), f->val); | 489 | f->uid = make_kuid(current_user_ns(), f_val); |
| 489 | if (!uid_valid(f->uid)) | 490 | if (!uid_valid(f->uid)) |
| 490 | goto exit_free; | 491 | goto exit_free; |
| 491 | break; | 492 | break; |
| @@ -494,11 +495,12 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 2 unmodified lines | |||
| 494 | case AUDIT_SGID: | 495 | case AUDIT_SGID: |
| 495 | case AUDIT_FSGID: | 496 | case AUDIT_FSGID: |
| 496 | case AUDIT_OBJ_GID: | 497 | case AUDIT_OBJ_GID: |
| 497 | f->gid = make_kgid(current_user_ns(), f->val); | 498 | f->gid = make_kgid(current_user_ns(), f_val); |
| 498 | if (!gid_valid(f->gid)) | 499 | if (!gid_valid(f->gid)) |
| 499 | goto exit_free; | 500 | goto exit_free; |
| 500 | break; | 501 | break; |
| 501 | case AUDIT_ARCH: | 502 | case AUDIT_ARCH: |
| 503 | f->val = f_val; | ||
| 502 | entry->rule.arch_f = f; | 504 | entry->rule.arch_f = f; |
| 503 | break; | 505 | break; |
| 504 | case AUDIT_SUBJ_USER: | 506 | case AUDIT_SUBJ_USER: |
| @@ -511,11 +513,13 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 6 unmodified lines | |||
| 511 | case AUDIT_OBJ_TYPE: | 513 | case AUDIT_OBJ_TYPE: |
| 512 | case AUDIT_OBJ_LEV_LOW: | 514 | case AUDIT_OBJ_LEV_LOW: |
| 513 | case AUDIT_OBJ_LEV_HIGH: | 515 | case AUDIT_OBJ_LEV_HIGH: |
| 514 | str = audit_unpack_string(&bufp, &remain, f->val); | 516 | str = audit_unpack_string(&bufp, &remain, f_val); |
| 515 | if (IS_ERR(str)) | 517 | if (IS_ERR(str)) { |
| 518 | err = PTR_ERR(str); | ||
| 516 | goto exit_free; | 519 | goto exit_free; |
| 517 | entry->rule.buflen += f->val; | 520 | } |
| 518 | 521 | entry->rule.buflen += f_val; | |
| 522 | f->lsm_str = str; | ||
| 519 | err = security_audit_rule_init(f->type, f->op, str, | 523 | err = security_audit_rule_init(f->type, f->op, str, |
| 520 | (void **)&f->lsm_rule); | 524 | (void **)&f->lsm_rule); |
| 521 | /* Keep currently invalid fields around in case they | 525 | /* Keep currently invalid fields around in case they |
| @@ -524,68 +528,71 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, · 2 unmodified lines | |||
| 524 | pr_warn("audit rule for LSM \'%s\' is invalid\n", | 528 | pr_warn("audit rule for LSM \'%s\' is invalid\n", |
| 525 | str); | 529 | str); |
| 526 | err = 0; | 530 | err = 0; |
| 527 | } | 531 | } else if (err) |
| 528 | if (err) { | ||
| 529 | kfree(str); | ||
| 530 | goto exit_free; | 532 | goto exit_free; |
| 531 | } else | ||
| 532 | f->lsm_str = str; | ||
| 533 | break; | 533 | break; |
| 534 | case AUDIT_WATCH: | 534 | case AUDIT_WATCH: |
| 535 | str = audit_unpack_string(&bufp, &remain, f->val); | 535 | str = audit_unpack_string(&bufp, &remain, f_val); |
| 536 | if (IS_ERR(str)) | 536 | if (IS_ERR(str)) { |
| 537 | err = PTR_ERR(str); | ||
| 537 | goto exit_free; | 538 | goto exit_free; |
| 538 | entry->rule.buflen += f->val; | 539 | } |
| 539 | 540 | err = audit_to_watch(&entry->rule, str, f_val, f->op); | |
| 540 | err = audit_to_watch(&entry->rule, str, f->val, f->op); | ||
| 541 | if (err) { | 541 | if (err) { |
| 542 | kfree(str); | 542 | kfree(str); |
| 543 | goto exit_free; | 543 | goto exit_free; |
| 544 | } | 544 | } |
| 545 | entry->rule.buflen += f_val; | ||
| 545 | break; | 546 | break; |
| 546 | case AUDIT_DIR: | 547 | case AUDIT_DIR: |
| 547 | str = audit_unpack_string(&bufp, &remain, f->val); | 548 | str = audit_unpack_string(&bufp, &remain, f_val); |
| 548 | if (IS_ERR(str)) | 549 | if (IS_ERR(str)) { |
| 550 | err = PTR_ERR(str); | ||
| 549 | goto exit_free; | 551 | goto exit_free; |
| 550 | entry->rule.buflen += f->val; | 552 | } |
| 551 | |||
| 552 | err = audit_make_tree(&entry->rule, str, f->op); | 553 | err = audit_make_tree(&entry->rule, str, f->op); |
| 553 | kfree(str); | 554 | kfree(str); |
| 554 | if (err) | 555 | if (err) |
| 555 | goto exit_free; | 556 | goto exit_free; |
| 557 | entry->rule.buflen += f_val; | ||
| 556 | break; | 558 | break; |
| 557 | case AUDIT_INODE: | 559 | case AUDIT_INODE: |
| 560 | f->val = f_val; | ||
| 558 | err = audit_to_inode(&entry->rule, f); | 561 | err = audit_to_inode(&entry->rule, f); |
| 559 | if (err) | 562 | if (err) |
| 560 | goto exit_free; | 563 | goto exit_free; |
| 561 | break; | 564 | break; |
| 562 | case AUDIT_FILTERKEY: | 565 | case AUDIT_FILTERKEY: |
| 563 | if (entry->rule.filterkey || f->val > AUDIT_MAX_KEY_LEN) | 566 | if (entry->rule.filterkey || f_val > AUDIT_MAX_KEY_LEN) |
| 564 | goto exit_free; | 567 | goto exit_free; |
| 565 | str = audit_unpack_string(&bufp, &remain, f->val); | 568 | str = audit_unpack_string(&bufp, &remain, f_val); |
| 566 | if (IS_ERR(str)) | 569 | if (IS_ERR(str)) { |
| 570 | err = PTR_ERR(str); | ||
| 567 | goto exit_free; | 571 | goto exit_free; |
| 568 | entry->rule.buflen += f->val; | 572 | } |
| 573 | entry->rule.buflen += f_val; | ||
| 569 | entry->rule.filterkey = str; | 574 | entry->rule.filterkey = str; |
| 570 | break; | 575 | break; |
| 571 | case AUDIT_EXE: | 576 | case AUDIT_EXE: |
| 572 | if (entry->rule.exe || f->val > PATH_MAX) | 577 | if (entry->rule.exe || f_val > PATH_MAX) |
| 573 | goto exit_free; | 578 | goto exit_free; |
| 574 | str = audit_unpack_string(&bufp, &remain, f->val); | 579 | str = audit_unpack_string(&bufp, &remain, f_val); |
| 575 | if (IS_ERR(str)) { | 580 | if (IS_ERR(str)) { |
| 576 | err = PTR_ERR(str); | 581 | err = PTR_ERR(str); |
| 577 | goto exit_free; | 582 | goto exit_free; |
| 578 | } | 583 | } |
| 579 | entry->rule.buflen += f->val; | 584 | audit_mark = audit_alloc_mark(&entry->rule, str, f_val); |
| 580 | |||
| 581 | audit_mark = audit_alloc_mark(&entry->rule, str, f->val); | ||
| 582 | if (IS_ERR(audit_mark)) { | 585 | if (IS_ERR(audit_mark)) { |
| 583 | kfree(str); | 586 | kfree(str); |
| 584 | err = PTR_ERR(audit_mark); | 587 | err = PTR_ERR(audit_mark); |
| 585 | goto exit_free; | 588 | goto exit_free; |
| 586 | } | 589 | } |
| 590 | entry->rule.buflen += f_val; | ||
| 587 | entry->rule.exe = audit_mark; | 591 | entry->rule.exe = audit_mark; |
| 588 | break; | 592 | break; |
| 593 | default: | ||
| 594 | f->val = f_val; | ||
| 595 | break; | ||
| 589 | } | 596 | } |
| 590 | } | 597 | } |
| 591 | 598 | ||
| @@ -471,6 +471,7 @@ static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) · 470 unmodified lines | |||
| 471 | */ | 471 | */ |
| 472 | p++; | 472 | p++; |
| 473 | if (p >= end) { | 473 | if (p >= end) { |
| 474 | (*pos)++; | ||
| 474 | return NULL; | 475 | return NULL; |
| 475 | } else { | 476 | } else { |
| 476 | *pos = *p; | 477 | *pos = *p; |
| @@ -782,7 +783,7 @@ void cgroup1_release_agent(struct work_struct *work) · 305 unmodified lines | |||
| 782 | 783 | ||
| 783 | pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); | 784 | pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); |
| 784 | agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); | 785 | agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); |
| 785 | if (!pathbuf || !agentbuf) | 786 | if (!pathbuf || !agentbuf || !strlen(agentbuf)) |
| 786 | goto out; | 787 | goto out; |
| 787 | 788 | ||
| 788 | spin_lock_irq(&css_set_lock); | 789 | spin_lock_irq(&css_set_lock); |
| @@ -3542,21 +3542,21 @@ static int cpu_stat_show(struct seq_file *seq, void *v) · 3541 unmodified lines | |||
| 3542 | static int cgroup_io_pressure_show(struct seq_file *seq, void *v) | 3542 | static int cgroup_io_pressure_show(struct seq_file *seq, void *v) |
| 3543 | { | 3543 | { |
| 3544 | struct cgroup *cgrp = seq_css(seq)->cgroup; | 3544 | struct cgroup *cgrp = seq_css(seq)->cgroup; |
| 3545 | struct psi_group *psi = cgroup_id(cgrp) == 1 ? &psi_system : &cgrp->psi; | 3545 | struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; |
| 3546 | 3546 | ||
| 3547 | return psi_show(seq, psi, PSI_IO); | 3547 | return psi_show(seq, psi, PSI_IO); |
| 3548 | } | 3548 | } |
| 3549 | static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) | 3549 | static int cgroup_memory_pressure_show(struct seq_file *seq, void *v) |
| 3550 | { | 3550 | { |
| 3551 | struct cgroup *cgrp = seq_css(seq)->cgroup; | 3551 | struct cgroup *cgrp = seq_css(seq)->cgroup; |
| 3552 | struct psi_group *psi = cgroup_id(cgrp) == 1 ? &psi_system : &cgrp->psi; | 3552 | struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; |
| 3553 | 3553 | ||
| 3554 | return psi_show(seq, psi, PSI_MEM); | 3554 | return psi_show(seq, psi, PSI_MEM); |
| 3555 | } | 3555 | } |
| 3556 | static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) | 3556 | static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) |
| 3557 | { | 3557 | { |
| 3558 | struct cgroup *cgrp = seq_css(seq)->cgroup; | 3558 | struct cgroup *cgrp = seq_css(seq)->cgroup; |
| 3559 | struct psi_group *psi = cgroup_id(cgrp) == 1 ? &psi_system : &cgrp->psi; | 3559 | struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : &cgrp->psi; |
| 3560 | 3560 | ||
| 3561 | return psi_show(seq, psi, PSI_CPU); | 3561 | return psi_show(seq, psi, PSI_CPU); |
| 3562 | } | 3562 | } |
| @@ -4400,12 +4400,16 @@ static void css_task_iter_advance_css_set(struct css_task_iter *it) · 837 unmodified lines | |||
| 4400 | } | 4400 | } |
| 4401 | } while (!css_set_populated(cset) && list_empty(&cset->dying_tasks)); | 4401 | } while (!css_set_populated(cset) && list_empty(&cset->dying_tasks)); |
| 4402 | 4402 | ||
| 4403 | if (!list_empty(&cset->tasks)) | 4403 | if (!list_empty(&cset->tasks)) { |
| 4404 | it->task_pos = cset->tasks.next; | 4404 | it->task_pos = cset->tasks.next; |
| 4405 | else if (!list_empty(&cset->mg_tasks)) | 4405 | it->cur_tasks_head = &cset->tasks; |
| 4406 | } else if (!list_empty(&cset->mg_tasks)) { | ||
| 4406 | it->task_pos = cset->mg_tasks.next; | 4407 | it->task_pos = cset->mg_tasks.next; |
| 4407 | else | 4408 | it->cur_tasks_head = &cset->mg_tasks; |
| 4409 | } else { | ||
| 4408 | it->task_pos = cset->dying_tasks.next; | 4410 | it->task_pos = cset->dying_tasks.next; |
| 4411 | it->cur_tasks_head = &cset->dying_tasks; | ||
| 4412 | } | ||
| 4409 | 4413 | ||
| 4410 | it->tasks_head = &cset->tasks; | 4414 | it->tasks_head = &cset->tasks; |
| 4411 | it->mg_tasks_head = &cset->mg_tasks; | 4415 | it->mg_tasks_head = &cset->mg_tasks; |
| @@ -4463,10 +4467,14 @@ repeat: · 51 unmodified lines | |||
| 4463 | else | 4467 | else |
| 4464 | it->task_pos = it->task_pos->next; | 4468 | it->task_pos = it->task_pos->next; |
| 4465 | 4469 | ||
| 4466 | if (it->task_pos == it->tasks_head) | 4470 | if (it->task_pos == it->tasks_head) { |
| 4467 | it->task_pos = it->mg_tasks_head->next; | 4471 | it->task_pos = it->mg_tasks_head->next; |
| 4468 | if (it->task_pos == it->mg_tasks_head) | 4472 | it->cur_tasks_head = it->mg_tasks_head; |
| 4473 | } | ||
| 4474 | if (it->task_pos == it->mg_tasks_head) { | ||
| 4469 | it->task_pos = it->dying_tasks_head->next; | 4475 | it->task_pos = it->dying_tasks_head->next; |
| 4476 | it->cur_tasks_head = it->dying_tasks_head; | ||
| 4477 | } | ||
| 4470 | if (it->task_pos == it->dying_tasks_head) | 4478 | if (it->task_pos == it->dying_tasks_head) |
| 4471 | css_task_iter_advance_css_set(it); | 4479 | css_task_iter_advance_css_set(it); |
| 4472 | } else { | 4480 | } else { |
| @@ -4485,11 +4493,12 @@ repeat: · 12 unmodified lines | |||
| 4485 | goto repeat; | 4493 | goto repeat; |
| 4486 | 4494 | ||
| 4487 | /* and dying leaders w/o live member threads */ | 4495 | /* and dying leaders w/o live member threads */ |
| 4488 | if (!atomic_read(&task->signal->live)) | 4496 | if (it->cur_tasks_head == it->dying_tasks_head && |
| 4497 | !atomic_read(&task->signal->live)) | ||
| 4489 | goto repeat; | 4498 | goto repeat; |
| 4490 | } else { | 4499 | } else { |
| 4491 | /* skip all dying ones */ | 4500 | /* skip all dying ones */ |
| 4492 | if (task->flags & PF_EXITING) | 4501 | if (it->cur_tasks_head == it->dying_tasks_head) |
| 4493 | goto repeat; | 4502 | goto repeat; |
| 4494 | } | 4503 | } |
| 4495 | } | 4504 | } |
| @@ -4595,6 +4604,9 @@ static void *cgroup_procs_next(struct seq_file *s, void *v, loff_t *pos) · 99 unmodified lines | |||
| 4595 | struct kernfs_open_file *of = s->private; | 4604 | struct kernfs_open_file *of = s->private; |
| 4596 | struct css_task_iter *it = of->priv; | 4605 | struct css_task_iter *it = of->priv; |
| 4597 | 4606 | ||
| 4607 | if (pos) | ||
| 4608 | (*pos)++; | ||
| 4609 | |||
| 4598 | return css_task_iter_next(it); | 4610 | return css_task_iter_next(it); |
| 4599 | } | 4611 | } |
| 4600 | 4612 | ||
| @@ -4610,7 +4622,7 @@ static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos, · 9 unmodified lines | |||
| 4610 | * from position 0, so we can simply keep iterating on !0 *pos. | 4622 | * from position 0, so we can simply keep iterating on !0 *pos. |
| 4611 | */ | 4623 | */ |
| 4612 | if (!it) { | 4624 | if (!it) { |
| 4613 | if (WARN_ON_ONCE((*pos)++)) | 4625 | if (WARN_ON_ONCE((*pos))) |
| 4614 | return ERR_PTR(-EINVAL); | 4626 | return ERR_PTR(-EINVAL); |
| 4615 | 4627 | ||
| 4616 | it = kzalloc(sizeof(*it), GFP_KERNEL); | 4628 | it = kzalloc(sizeof(*it), GFP_KERNEL); |
| @@ -4618,10 +4630,11 @@ static void *__cgroup_procs_start(struct seq_file *s, loff_t *pos, · 1 unmodified line | |||
| 4618 | return ERR_PTR(-ENOMEM); | 4630 | return ERR_PTR(-ENOMEM); |
| 4619 | of->priv = it; | 4631 | of->priv = it; |
| 4620 | css_task_iter_start(&cgrp->self, iter_flags, it); | 4632 | css_task_iter_start(&cgrp->self, iter_flags, it); |
| 4621 | } else if (!(*pos)++) { | 4633 | } else if (!(*pos)) { |
| 4622 | css_task_iter_end(it); | 4634 | css_task_iter_end(it); |
| 4623 | css_task_iter_start(&cgrp->self, iter_flags, it); | 4635 | css_task_iter_start(&cgrp->self, iter_flags, it); |
| 4624 | } | 4636 | } else |
| 4637 | return it->cur_task; | ||
| 4625 | 4638 | ||
| 4626 | return cgroup_procs_next(s, NULL, NULL); | 4639 | return cgroup_procs_next(s, NULL, NULL); |
| 4627 | } | 4640 | } |
| @@ -6258,6 +6271,10 @@ void cgroup_sk_alloc(struct sock_cgroup_data *skcd) · 1630 unmodified lines | |||
| 6258 | return; | 6271 | return; |
| 6259 | } | 6272 | } |
| 6260 | 6273 | ||
| 6274 | /* Don't associate the sock with unrelated interrupted task's cgroup. */ | ||
| 6275 | if (in_interrupt()) | ||
| 6276 | return; | ||
| 6277 | |||
| 6261 | rcu_read_lock(); | 6278 | rcu_read_lock(); |
| 6262 | 6279 | ||
| 6263 | while (true) { | 6280 | while (true) { |
| @@ -619,8 +619,8 @@ static void forget_original_parent(struct task_struct *father, · 618 unmodified lines | |||
| 619 | reaper = find_new_reaper(father, reaper); | 619 | reaper = find_new_reaper(father, reaper); |
| 620 | list_for_each_entry(p, &father->children, sibling) { | 620 | list_for_each_entry(p, &father->children, sibling) { |
| 621 | for_each_thread(p, t) { | 621 | for_each_thread(p, t) { |
| 622 | t->real_parent = reaper; | 622 | RCU_INIT_POINTER(t->real_parent, reaper); |
| 623 | BUG_ON((!t->ptrace) != (t->parent == father)); | 623 | BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father)); |
| 624 | if (likely(!t->ptrace)) | 624 | if (likely(!t->ptrace)) |
| 625 | t->parent = t->real_parent; | 625 | t->parent = t->real_parent; |
| 626 | if (t->pdeath_signal) | 626 | if (t->pdeath_signal) |
| @@ -1508,7 +1508,7 @@ static int copy_sighand(unsigned long clone_flags, struct task_struct *tsk) · 1507 unmodified lines | |||
| 1508 | return 0; | 1508 | return 0; |
| 1509 | } | 1509 | } |
| 1510 | sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); | 1510 | sig = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); |
| 1511 | rcu_assign_pointer(tsk->sighand, sig); | 1511 | RCU_INIT_POINTER(tsk->sighand, sig); |
| 1512 | if (!sig) | 1512 | if (!sig) |
| 1513 | return -ENOMEM; | 1513 | return -ENOMEM; |
| 1514 | 1514 | ||
| @@ -385,9 +385,9 @@ static inline int hb_waiters_pending(struct futex_hash_bucket *hb) · 384 unmodified lines | |||
| 385 | */ | 385 | */ |
| 386 | static struct futex_hash_bucket *hash_futex(union futex_key *key) | 386 | static struct futex_hash_bucket *hash_futex(union futex_key *key) |
| 387 | { | 387 | { |
| 388 | u32 hash = jhash2((u32*)&key->both.word, | 388 | u32 hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / 4, |
| 389 | (sizeof(key->both.word)+sizeof(key->both.ptr))/4, | ||
| 390 | key->both.offset); | 389 | key->both.offset); |
| 390 | |||
| 391 | return &futex_queues[hash & (futex_hashsize - 1)]; | 391 | return &futex_queues[hash & (futex_hashsize - 1)]; |
| 392 | } | 392 | } |
| 393 | 393 | ||
| @@ -429,7 +429,7 @@ static void get_futex_key_refs(union futex_key *key) · 35 unmodified lines | |||
| 429 | 429 | ||
| 430 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { | 430 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { |
| 431 | case FUT_OFF_INODE: | 431 | case FUT_OFF_INODE: |
| 432 | ihold(key->shared.inode); /* implies smp_mb(); (B) */ | 432 | smp_mb(); /* explicit smp_mb(); (B) */ |
| 433 | break; | 433 | break; |
| 434 | case FUT_OFF_MMSHARED: | 434 | case FUT_OFF_MMSHARED: |
| 435 | futex_get_mm(key); /* implies smp_mb(); (B) */ | 435 | futex_get_mm(key); /* implies smp_mb(); (B) */ |
| @@ -463,7 +463,6 @@ static void drop_futex_key_refs(union futex_key *key) · 27 unmodified lines | |||
| 463 | 463 | ||
| 464 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { | 464 | switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) { |
| 465 | case FUT_OFF_INODE: | 465 | case FUT_OFF_INODE: |
| 466 | iput(key->shared.inode); | ||
| 467 | break; | 466 | break; |
| 468 | case FUT_OFF_MMSHARED: | 467 | case FUT_OFF_MMSHARED: |
| 469 | mmdrop(key->private.mm); | 468 | mmdrop(key->private.mm); |
| @@ -505,6 +504,46 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, · 35 unmodified lines | |||
| 505 | return timeout; | 504 | return timeout; |
| 506 | } | 505 | } |
| 507 | 506 | ||
| 507 | /* | ||
| 508 | * Generate a machine wide unique identifier for this inode. | ||
| 509 | * | ||
| 510 | * This relies on u64 not wrapping in the life-time of the machine; which with | ||
| 511 | * 1ns resolution means almost 585 years. | ||
| 512 | * | ||
| 513 | * This further relies on the fact that a well formed program will not unmap | ||
| 514 | * the file while it has a (shared) futex waiting on it. This mapping will have | ||
| 515 | * a file reference which pins the mount and inode. | ||
| 516 | * | ||
| 517 | * If for some reason an inode gets evicted and read back in again, it will get | ||
| 518 | * a new sequence number and will _NOT_ match, even though it is the exact same | ||
| 519 | * file. | ||
| 520 | * | ||
| 521 | * It is important that match_futex() will never have a false-positive, esp. | ||
| 522 | * for PI futexes that can mess up the state. The above argues that false-negatives | ||
| 523 | * are only possible for malformed programs. | ||
| 524 | */ | ||
| 525 | static u64 get_inode_sequence_number(struct inode *inode) | ||
| 526 | { | ||
| 527 | static atomic64_t i_seq; | ||
| 528 | u64 old; | ||
| 529 | |||
| 530 | /* Does the inode already have a sequence number? */ | ||
| 531 | old = atomic64_read(&inode->i_sequence); | ||
| 532 | if (likely(old)) | ||
| 533 | return old; | ||
| 534 | |||
| 535 | for (;;) { | ||
| 536 | u64 new = atomic64_add_return(1, &i_seq); | ||
| 537 | if (WARN_ON_ONCE(!new)) | ||
| 538 | continue; | ||
| 539 | |||
| 540 | old = atomic64_cmpxchg_relaxed(&inode->i_sequence, 0, new); | ||
| 541 | if (old) | ||
| 542 | return old; | ||
| 543 | return new; | ||
| 544 | } | ||
| 545 | } | ||
| 546 | |||
| 508 | /** | 547 | /** |
| 509 | * get_futex_key() - Get parameters which are the keys for a futex | 548 | * get_futex_key() - Get parameters which are the keys for a futex |
| 510 | * @uaddr: virtual address of the futex | 549 | * @uaddr: virtual address of the futex |
| @@ -517,9 +556,15 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, · 6 unmodified lines | |||
| 517 | * | 556 | * |
| 518 | * The key words are stored in @key on success. | 557 | * The key words are stored in @key on success. |
| 519 | * | 558 | * |
| 520 | * For shared mappings, it's (page->index, file_inode(vma->vm_file), | 559 | * For shared mappings (when @fshared), the key is: |
| 521 | * offset_within_page). For private mappings, it's (uaddr, current->mm). | 560 | * ( inode->i_sequence, page->index, offset_within_page ) |
| 522 | * We can usually work out the index without swapping in the page. | 561 | * [ also see get_inode_sequence_number() ] |
| 562 | * | ||
| 563 | * For private mappings (or when !@fshared), the key is: | ||
| 564 | * ( current->mm, address, 0 ) | ||
| 565 | * | ||
| 566 | * This allows (cross process, where applicable) identification of the futex | ||
| 567 | * without keeping the page pinned for the duration of the FUTEX_WAIT. | ||
| 523 | * | 568 | * |
| 524 | * lock_page() might sleep, the caller should not hold a spinlock. | 569 | * lock_page() might sleep, the caller should not hold a spinlock. |
| 525 | */ | 570 | */ |
| @@ -659,8 +704,6 @@ again: · 133 unmodified lines | |||
| 659 | key->private.mm = mm; | 704 | key->private.mm = mm; |
| 660 | key->private.address = address; | 705 | key->private.address = address; |
| 661 | 706 | ||
| 662 | get_futex_key_refs(key); /* implies smp_mb(); (B) */ | ||
| 663 | |||
| 664 | } else { | 707 | } else { |
| 665 | struct inode *inode; | 708 | struct inode *inode; |
| 666 | 709 | ||
| @@ -692,40 +735,14 @@ again: · 25 unmodified lines | |||
| 692 | goto again; | 735 | goto again; |
| 693 | } | 736 | } |
| 694 | 737 | ||
| 695 | /* | ||
| 696 | * Take a reference unless it is about to be freed. Previously | ||
| 697 | * this reference was taken by ihold under the page lock | ||
| 698 | * pinning the inode in place so i_lock was unnecessary. The | ||
| 699 | * only way for this check to fail is if the inode was | ||
| 700 | * truncated in parallel which is almost certainly an | ||
| 701 | * application bug. In such a case, just retry. | ||
| 702 | * | ||
| 703 | * We are not calling into get_futex_key_refs() in file-backed | ||
| 704 | * cases, therefore a successful atomic_inc return below will | ||
| 705 | * guarantee that get_futex_key() will still imply smp_mb(); (B). | ||
| 706 | */ | ||
| 707 | if (!atomic_inc_not_zero(&inode->i_count)) { | ||
| 708 | rcu_read_unlock(); | ||
| 709 | put_page(page); | ||
| 710 | |||
| 711 | goto again; | ||
| 712 | } | ||
| 713 | |||
| 714 | /* Should be impossible but lets be paranoid for now */ | ||
| 715 | if (WARN_ON_ONCE(inode->i_mapping != mapping)) { | ||
| 716 | err = -EFAULT; | ||
| 717 | rcu_read_unlock(); | ||
| 718 | iput(inode); | ||
| 719 | |||
| 720 | goto out; | ||
| 721 | } | ||
| 722 | |||
| 723 | key->both.offset |= FUT_OFF_INODE; /* inode-based key */ | 738 | key->both.offset |= FUT_OFF_INODE; /* inode-based key */ |
| 724 | key->shared.inode = inode; | 739 | key->shared.i_seq = get_inode_sequence_number(inode); |
| 725 | key->shared.pgoff = basepage_index(tail); | 740 | key->shared.pgoff = basepage_index(tail); |
| 726 | rcu_read_unlock(); | 741 | rcu_read_unlock(); |
| 727 | } | 742 | } |
| 728 | 743 | ||
| 744 | get_futex_key_refs(key); /* implies smp_mb(); (B) */ | ||
| 745 | |||
| 729 | out: | 746 | out: |
| 730 | put_page(page); | 747 | put_page(page); |
| 731 | return err; | 748 | return err; |
| @@ -247,6 +247,16 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, · 246 unmodified lines | |||
| 247 | tmp = tmp->parent; | 247 | tmp = tmp->parent; |
| 248 | } | 248 | } |
| 249 | 249 | ||
| 250 | /* | ||
| 251 | * ENOMEM is not the most obvious choice especially for the case | ||
| 252 | * where the child subreaper has already exited and the pid | ||
| 253 | * namespace denies the creation of any new processes. But ENOMEM | ||
| 254 | * is what we have exposed to userspace for a long time and it is | ||
| 255 | * documented behavior for pid namespaces. So we can't easily | ||
| 256 | * change it even if there were an error code better suited. | ||
| 257 | */ | ||
| 258 | retval = -ENOMEM; | ||
| 259 | |||
| 250 | if (unlikely(is_child_reaper(pid))) { | 260 | if (unlikely(is_child_reaper(pid))) { |
| 251 | if (pid_ns_prepare_proc(ns)) | 261 | if (pid_ns_prepare_proc(ns)) |
| 252 | goto out_free; | 262 | goto out_free; |
| @@ -1681,7 +1681,7 @@ static unsigned long minimum_image_size(unsigned long saveable) · 1680 unmodified lines | |||
| 1681 | * hibernation for allocations made while saving the image and for device | 1681 | * hibernation for allocations made while saving the image and for device |
| 1682 | * drivers, in case they need to allocate memory from their hibernation | 1682 | * drivers, in case they need to allocate memory from their hibernation |
| 1683 | * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough | 1683 | * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough |
| 1684 | * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through | 1684 | * estimate) and reserved_size divided by PAGE_SIZE (which is tunable through |
| 1685 | * /sys/power/reserved_size, respectively). To make this happen, we compute the | 1685 | * /sys/power/reserved_size, respectively). To make this happen, we compute the |
| 1686 | * total number of available page frames and allocate at least | 1686 | * total number of available page frames and allocate at least |
| 1687 | * | 1687 | * |
| @@ -8337,6 +8337,8 @@ static inline void update_sg_wakeup_stats(struct sched_domain *sd, · 8336 unmodified lines | |||
| 8337 | 8337 | ||
| 8338 | sgs->group_capacity = group->sgc->capacity; | 8338 | sgs->group_capacity = group->sgc->capacity; |
| 8339 | 8339 | ||
| 8340 | sgs->group_weight = group->group_weight; | ||
| 8341 | |||
| 8340 | sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); | 8342 | sgs->group_type = group_classify(sd->imbalance_pct, group, sgs); |
| 8341 | 8343 | ||
| 8342 | /* | 8344 | /* |
| @@ -413,27 +413,32 @@ __sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimi · 412 unmodified lines | |||
| 413 | { | 413 | { |
| 414 | struct sigqueue *q = NULL; | 414 | struct sigqueue *q = NULL; |
| 415 | struct user_struct *user; | 415 | struct user_struct *user; |
| 416 | int sigpending; | ||
| 416 | 417 | ||
| 417 | /* | 418 | /* |
| 418 | * Protect access to @t credentials. This can go away when all | 419 | * Protect access to @t credentials. This can go away when all |
| 419 | * callers hold rcu read lock. | 420 | * callers hold rcu read lock. |
| 421 | * | ||
| 422 | * NOTE! A pending signal will hold on to the user refcount, | ||
| 423 | * and we get/put the refcount only when the sigpending count | ||
| 424 | * changes from/to zero. | ||
| 420 | */ | 425 | */ |
| 421 | rcu_read_lock(); | 426 | rcu_read_lock(); |
| 422 | user = get_uid(__task_cred(t)->user); | 427 | user = __task_cred(t)->user; |
| 423 | atomic_inc(&user->sigpending); | 428 | sigpending = atomic_inc_return(&user->sigpending); |
| 429 | if (sigpending == 1) | ||
| 430 | get_uid(user); | ||
| 424 | rcu_read_unlock(); | 431 | rcu_read_unlock(); |
| 425 | 432 | ||
| 426 | if (override_rlimit || | 433 | if (override_rlimit || likely(sigpending <= task_rlimit(t, RLIMIT_SIGPENDING))) { |
| 427 | atomic_read(&user->sigpending) <= | ||
| 428 | task_rlimit(t, RLIMIT_SIGPENDING)) { | ||
| 429 | q = kmem_cache_alloc(sigqueue_cachep, flags); | 434 | q = kmem_cache_alloc(sigqueue_cachep, flags); |
| 430 | } else { | 435 | } else { |
| 431 | print_dropped_signal(sig); | 436 | print_dropped_signal(sig); |
| 432 | } | 437 | } |
| 433 | 438 | ||
| 434 | if (unlikely(q == NULL)) { | 439 | if (unlikely(q == NULL)) { |
| 435 | atomic_dec(&user->sigpending); | 440 | if (atomic_dec_and_test(&user->sigpending)) |
| 436 | free_uid(user); | 441 | free_uid(user); |
| 437 | } else { | 442 | } else { |
| 438 | INIT_LIST_HEAD(&q->list); | 443 | INIT_LIST_HEAD(&q->list); |
| 439 | q->flags = 0; | 444 | q->flags = 0; |
| @@ -447,8 +452,8 @@ static void __sigqueue_free(struct sigqueue *q) · 7 unmodified lines | |||
| 447 | { | 452 | { |
| 448 | if (q->flags & SIGQUEUE_PREALLOC) | 453 | if (q->flags & SIGQUEUE_PREALLOC) |
| 449 | return; | 454 | return; |
| 450 | atomic_dec(&q->user->sigpending); | 455 | if (atomic_dec_and_test(&q->user->sigpending)) |
| 451 | free_uid(q->user); | 456 | free_uid(q->user); |
| 452 | kmem_cache_free(sigqueue_cachep, q); | 457 | kmem_cache_free(sigqueue_cachep, q); |
| 453 | } | 458 | } |
| 454 | 459 | ||
| @@ -47,6 +47,7 @@ · 46 unmodified lines | |||
| 47 | #include <linux/syscalls.h> | 47 | #include <linux/syscalls.h> |
| 48 | #include <linux/kprobes.h> | 48 | #include <linux/kprobes.h> |
| 49 | #include <linux/user_namespace.h> | 49 | #include <linux/user_namespace.h> |
| 50 | #include <linux/time_namespace.h> | ||
| 50 | #include <linux/binfmts.h> | 51 | #include <linux/binfmts.h> |
| 51 | 52 | ||
| 52 | #include <linux/sched.h> | 53 | #include <linux/sched.h> |
| @@ -2546,6 +2547,7 @@ static int do_sysinfo(struct sysinfo *info) · 2493 unmodified lines | |||
| 2546 | memset(info, 0, sizeof(struct sysinfo)); | 2547 | memset(info, 0, sizeof(struct sysinfo)); |
| 2547 | 2548 | ||
| 2548 | ktime_get_boottime_ts64(&tp); | 2549 | ktime_get_boottime_ts64(&tp); |
| 2550 | timens_add_boottime(&tp); | ||
| 2549 | info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0); | 2551 | info->uptime = tp.tv_sec + (tp.tv_nsec ? 1 : 0); |
| 2550 | 2552 | ||
| 2551 | get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT); | 2553 | get_avenrun(info->loads, 0, SI_LOAD_SHIFT - FSHIFT); |
| @@ -143,8 +143,8 @@ if FTRACE · 142 unmodified lines | |||
| 143 | 143 | ||
| 144 | config BOOTTIME_TRACING | 144 | config BOOTTIME_TRACING |
| 145 | bool "Boot-time Tracing support" | 145 | bool "Boot-time Tracing support" |
| 146 | depends on BOOT_CONFIG && TRACING | 146 | depends on TRACING |
| 147 | default y | 147 | select BOOT_CONFIG |
| 148 | help | 148 | help |
| 149 | Enable developer to setup ftrace subsystem via supplemental | 149 | Enable developer to setup ftrace subsystem via supplemental |
| 150 | kernel cmdline at boot time for debugging (tracing) driver | 150 | kernel cmdline at boot time for debugging (tracing) driver |
| @@ -335,6 +335,7 @@ static void put_probe_ref(void) · 334 unmodified lines | |||
| 335 | 335 | ||
| 336 | static void blk_trace_cleanup(struct blk_trace *bt) | 336 | static void blk_trace_cleanup(struct blk_trace *bt) |
| 337 | { | 337 | { |
| 338 | synchronize_rcu(); | ||
| 338 | blk_trace_free(bt); | 339 | blk_trace_free(bt); |
| 339 | put_probe_ref(); | 340 | put_probe_ref(); |
| 340 | } | 341 | } |
| @@ -629,8 +630,10 @@ static int compat_blk_trace_setup(struct request_queue *q, char *name, · 288 unmodified lines | |||
| 629 | static int __blk_trace_startstop(struct request_queue *q, int start) | 630 | static int __blk_trace_startstop(struct request_queue *q, int start) |
| 630 | { | 631 | { |
| 631 | int ret; | 632 | int ret; |
| 632 | struct blk_trace *bt = q->blk_trace; | 633 | struct blk_trace *bt; |
| 633 | 634 | ||
| 635 | bt = rcu_dereference_protected(q->blk_trace, | ||
| 636 | lockdep_is_held(&q->blk_trace_mutex)); | ||
| 634 | if (bt == NULL) | 637 | if (bt == NULL) |
| 635 | return -EINVAL; | 638 | return -EINVAL; |
| 636 | 639 | ||
| @@ -740,8 +743,8 @@ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) · 103 unmodified lines | |||
| 740 | void blk_trace_shutdown(struct request_queue *q) | 743 | void blk_trace_shutdown(struct request_queue *q) |
| 741 | { | 744 | { |
| 742 | mutex_lock(&q->blk_trace_mutex); | 745 | mutex_lock(&q->blk_trace_mutex); |
| 743 | 746 | if (rcu_dereference_protected(q->blk_trace, | |
| 744 | if (q->blk_trace) { | 747 | lockdep_is_held(&q->blk_trace_mutex))) { |
| 745 | __blk_trace_startstop(q, 0); | 748 | __blk_trace_startstop(q, 0); |
| 746 | __blk_trace_remove(q); | 749 | __blk_trace_remove(q); |
| 747 | } | 750 | } |
| @@ -752,8 +755,10 @@ void blk_trace_shutdown(struct request_queue *q) · 4 unmodified lines | |||
| 752 | #ifdef CONFIG_BLK_CGROUP | 755 | #ifdef CONFIG_BLK_CGROUP |
| 753 | static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) | 756 | static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) |
| 754 | { | 757 | { |
| 755 | struct blk_trace *bt = q->blk_trace; | 758 | struct blk_trace *bt; |
| 756 | 759 | ||
| 760 | /* We don't use the 'bt' value here except as an optimization... */ | ||
| 761 | bt = rcu_dereference_protected(q->blk_trace, 1); | ||
| 757 | if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) | 762 | if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) |
| 758 | return 0; | 763 | return 0; |
| 759 | 764 | ||
| @@ -796,10 +801,14 @@ blk_trace_request_get_cgid(struct request_queue *q, struct request *rq) · 36 unmodified lines | |||
| 796 | static void blk_add_trace_rq(struct request *rq, int error, | 801 | static void blk_add_trace_rq(struct request *rq, int error, |
| 797 | unsigned int nr_bytes, u32 what, u64 cgid) | 802 | unsigned int nr_bytes, u32 what, u64 cgid) |
| 798 | { | 803 | { |
| 799 | struct blk_trace *bt = rq->q->blk_trace; | 804 | struct blk_trace *bt; |
| 800 | 805 | ||
| 801 | if (likely(!bt)) | 806 | rcu_read_lock(); |
| 807 | bt = rcu_dereference(rq->q->blk_trace); | ||
| 808 | if (likely(!bt)) { | ||
| 809 | rcu_read_unlock(); | ||
| 802 | return; | 810 | return; |
| 811 | } | ||
| 803 | 812 | ||
| 804 | if (blk_rq_is_passthrough(rq)) | 813 | if (blk_rq_is_passthrough(rq)) |
| 805 | what |= BLK_TC_ACT(BLK_TC_PC); | 814 | what |= BLK_TC_ACT(BLK_TC_PC); |
| @@ -808,6 +817,7 @@ static void blk_add_trace_rq(struct request *rq, int error, · 2 unmodified lines | |||
| 808 | 817 | ||
| 809 | __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), | 818 | __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), |
| 810 | rq->cmd_flags, what, error, 0, NULL, cgid); | 819 | rq->cmd_flags, what, error, 0, NULL, cgid); |
| 820 | rcu_read_unlock(); | ||
| 811 | } | 821 | } |
| 812 | 822 | ||
| 813 | static void blk_add_trace_rq_insert(void *ignore, | 823 | static void blk_add_trace_rq_insert(void *ignore, |
| @@ -853,14 +863,19 @@ static void blk_add_trace_rq_complete(void *ignore, struct request *rq, · 39 unmodified lines | |||
| 853 | static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, | 863 | static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, |
| 854 | u32 what, int error) | 864 | u32 what, int error) |
| 855 | { | 865 | { |
| 856 | struct blk_trace *bt = q->blk_trace; | 866 | struct blk_trace *bt; |
| 857 | 867 | ||
| 858 | if (likely(!bt)) | 868 | rcu_read_lock(); |
| 869 | bt = rcu_dereference(q->blk_trace); | ||
| 870 | if (likely(!bt)) { | ||
| 871 | rcu_read_unlock(); | ||
| 859 | return; | 872 | return; |
| 873 | } | ||
| 860 | 874 | ||
| 861 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, | 875 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, |
| 862 | bio_op(bio), bio->bi_opf, what, error, 0, NULL, | 876 | bio_op(bio), bio->bi_opf, what, error, 0, NULL, |
| 863 | blk_trace_bio_get_cgid(q, bio)); | 877 | blk_trace_bio_get_cgid(q, bio)); |
| 878 | rcu_read_unlock(); | ||
| 864 | } | 879 | } |
| 865 | 880 | ||
| 866 | static void blk_add_trace_bio_bounce(void *ignore, | 881 | static void blk_add_trace_bio_bounce(void *ignore, |
| @@ -905,11 +920,14 @@ static void blk_add_trace_getrq(void *ignore, · 38 unmodified lines | |||
| 905 | if (bio) | 920 | if (bio) |
| 906 | blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0); | 921 | blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0); |
| 907 | else { | 922 | else { |
| 908 | struct blk_trace *bt = q->blk_trace; | 923 | struct blk_trace *bt; |
| 909 | 924 | ||
| 925 | rcu_read_lock(); | ||
| 926 | bt = rcu_dereference(q->blk_trace); | ||
| 910 | if (bt) | 927 | if (bt) |
| 911 | __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_GETRQ, 0, 0, | 928 | __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_GETRQ, 0, 0, |
| 912 | NULL, 0); | 929 | NULL, 0); |
| 930 | rcu_read_unlock(); | ||
| 913 | } | 931 | } |
| 914 | } | 932 | } |
| 915 | 933 | ||
| @@ -921,27 +939,35 @@ static void blk_add_trace_sleeprq(void *ignore, · 5 unmodified lines | |||
| 921 | if (bio) | 939 | if (bio) |
| 922 | blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0); | 940 | blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0); |
| 923 | else { | 941 | else { |
| 924 | struct blk_trace *bt = q->blk_trace; | 942 | struct blk_trace *bt; |
| 925 | 943 | ||
| 944 | rcu_read_lock(); | ||
| 945 | bt = rcu_dereference(q->blk_trace); | ||
| 926 | if (bt) | 946 | if (bt) |
| 927 | __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_SLEEPRQ, | 947 | __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_SLEEPRQ, |
| 928 | 0, 0, NULL, 0); | 948 | 0, 0, NULL, 0); |
| 949 | rcu_read_unlock(); | ||
| 929 | } | 950 | } |
| 930 | } | 951 | } |
| 931 | 952 | ||
| 932 | static void blk_add_trace_plug(void *ignore, struct request_queue *q) | 953 | static void blk_add_trace_plug(void *ignore, struct request_queue *q) |
| 933 | { | 954 | { |
| 934 | struct blk_trace *bt = q->blk_trace; | 955 | struct blk_trace *bt; |
| 935 | 956 | ||
| 957 | rcu_read_lock(); | ||
| 958 | bt = rcu_dereference(q->blk_trace); | ||
| 936 | if (bt) | 959 | if (bt) |
| 937 | __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); | 960 | __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); |
| 961 | rcu_read_unlock(); | ||
| 938 | } | 962 | } |
| 939 | 963 | ||
| 940 | static void blk_add_trace_unplug(void *ignore, struct request_queue *q, | 964 | static void blk_add_trace_unplug(void *ignore, struct request_queue *q, |
| 941 | unsigned int depth, bool explicit) | 965 | unsigned int depth, bool explicit) |
| 942 | { | 966 | { |
| 943 | struct blk_trace *bt = q->blk_trace; | 967 | struct blk_trace *bt; |
| 944 | 968 | ||
| 969 | rcu_read_lock(); | ||
| 970 | bt = rcu_dereference(q->blk_trace); | ||
| 945 | if (bt) { | 971 | if (bt) { |
| 946 | __be64 rpdu = cpu_to_be64(depth); | 972 | __be64 rpdu = cpu_to_be64(depth); |
| 947 | u32 what; | 973 | u32 what; |
| @@ -953,14 +979,17 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q, · 5 unmodified lines | |||
| 953 | 979 | ||
| 954 | __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); | 980 | __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); |
| 955 | } | 981 | } |
| 982 | rcu_read_unlock(); | ||
| 956 | } | 983 | } |
| 957 | 984 | ||
| 958 | static void blk_add_trace_split(void *ignore, | 985 | static void blk_add_trace_split(void *ignore, |
| 959 | struct request_queue *q, struct bio *bio, | 986 | struct request_queue *q, struct bio *bio, |
| 960 | unsigned int pdu) | 987 | unsigned int pdu) |
| 961 | { | 988 | { |
| 962 | struct blk_trace *bt = q->blk_trace; | 989 | struct blk_trace *bt; |
| 963 | 990 | ||
| 991 | rcu_read_lock(); | ||
| 992 | bt = rcu_dereference(q->blk_trace); | ||
| 964 | if (bt) { | 993 | if (bt) { |
| 965 | __be64 rpdu = cpu_to_be64(pdu); | 994 | __be64 rpdu = cpu_to_be64(pdu); |
| 966 | 995 | ||
| @@ -969,6 +998,7 @@ static void blk_add_trace_split(void *ignore, · 2 unmodified lines | |||
| 969 | BLK_TA_SPLIT, bio->bi_status, sizeof(rpdu), | 998 | BLK_TA_SPLIT, bio->bi_status, sizeof(rpdu), |
| 970 | &rpdu, blk_trace_bio_get_cgid(q, bio)); | 999 | &rpdu, blk_trace_bio_get_cgid(q, bio)); |
| 971 | } | 1000 | } |
| 1001 | rcu_read_unlock(); | ||
| 972 | } | 1002 | } |
| 973 | 1003 | ||
| 974 | /** | 1004 | /** |
| @@ -988,11 +1018,15 @@ static void blk_add_trace_bio_remap(void *ignore, · 13 unmodified lines | |||
| 988 | struct request_queue *q, struct bio *bio, | 1018 | struct request_queue *q, struct bio *bio, |
| 989 | dev_t dev, sector_t from) | 1019 | dev_t dev, sector_t from) |
| 990 | { | 1020 | { |
| 991 | struct blk_trace *bt = q->blk_trace; | 1021 | struct blk_trace *bt; |
| 992 | struct blk_io_trace_remap r; | 1022 | struct blk_io_trace_remap r; |
| 993 | 1023 | ||
| 994 | if (likely(!bt)) | 1024 | rcu_read_lock(); |
| 1025 | bt = rcu_dereference(q->blk_trace); | ||
| 1026 | if (likely(!bt)) { | ||
| 1027 | rcu_read_unlock(); | ||
| 995 | return; | 1028 | return; |
| 1029 | } | ||
| 996 | 1030 | ||
| 997 | r.device_from = cpu_to_be32(dev); | 1031 | r.device_from = cpu_to_be32(dev); |
| 998 | r.device_to = cpu_to_be32(bio_dev(bio)); | 1032 | r.device_to = cpu_to_be32(bio_dev(bio)); |
| @@ -1001,6 +1035,7 @@ static void blk_add_trace_bio_remap(void *ignore, · 2 unmodified lines | |||
| 1001 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, | 1035 | __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, |
| 1002 | bio_op(bio), bio->bi_opf, BLK_TA_REMAP, bio->bi_status, | 1036 | bio_op(bio), bio->bi_opf, BLK_TA_REMAP, bio->bi_status, |
| 1003 | sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); | 1037 | sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); |
| 1038 | rcu_read_unlock(); | ||
| 1004 | } | 1039 | } |
| 1005 | 1040 | ||
| 1006 | /** | 1041 | /** |
| @@ -1021,11 +1056,15 @@ static void blk_add_trace_rq_remap(void *ignore, · 14 unmodified lines | |||
| 1021 | struct request *rq, dev_t dev, | 1056 | struct request *rq, dev_t dev, |
| 1022 | sector_t from) | 1057 | sector_t from) |
| 1023 | { | 1058 | { |
| 1024 | struct blk_trace *bt = q->blk_trace; | 1059 | struct blk_trace *bt; |
| 1025 | struct blk_io_trace_remap r; | 1060 | struct blk_io_trace_remap r; |
| 1026 | 1061 | ||
| 1027 | if (likely(!bt)) | 1062 | rcu_read_lock(); |
| 1063 | bt = rcu_dereference(q->blk_trace); | ||
| 1064 | if (likely(!bt)) { | ||
| 1065 | rcu_read_unlock(); | ||
| 1028 | return; | 1066 | return; |
| 1067 | } | ||
| 1029 | 1068 | ||
| 1030 | r.device_from = cpu_to_be32(dev); | 1069 | r.device_from = cpu_to_be32(dev); |
| 1031 | r.device_to = cpu_to_be32(disk_devt(rq->rq_disk)); | 1070 | r.device_to = cpu_to_be32(disk_devt(rq->rq_disk)); |
| @@ -1034,6 +1073,7 @@ static void blk_add_trace_rq_remap(void *ignore, · 2 unmodified lines | |||
| 1034 | __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), | 1073 | __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), |
| 1035 | rq_data_dir(rq), 0, BLK_TA_REMAP, 0, | 1074 | rq_data_dir(rq), 0, BLK_TA_REMAP, 0, |
| 1036 | sizeof(r), &r, blk_trace_request_get_cgid(q, rq)); | 1075 | sizeof(r), &r, blk_trace_request_get_cgid(q, rq)); |
| 1076 | rcu_read_unlock(); | ||
| 1037 | } | 1077 | } |
| 1038 | 1078 | ||
| 1039 | /** | 1079 | /** |
| @@ -1051,14 +1091,19 @@ void blk_add_driver_data(struct request_queue *q, · 11 unmodified lines | |||
| 1051 | struct request *rq, | 1091 | struct request *rq, |
| 1052 | void *data, size_t len) | 1092 | void *data, size_t len) |
| 1053 | { | 1093 | { |
| 1054 | struct blk_trace *bt = q->blk_trace; | 1094 | struct blk_trace *bt; |
| 1055 | 1095 | ||
| 1056 | if (likely(!bt)) | 1096 | rcu_read_lock(); |
| 1097 | bt = rcu_dereference(q->blk_trace); | ||
| 1098 | if (likely(!bt)) { | ||
| 1099 | rcu_read_unlock(); | ||
| 1057 | return; | 1100 | return; |
| 1101 | } | ||
| 1058 | 1102 | ||
| 1059 | __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, | 1103 | __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, |
| 1060 | BLK_TA_DRV_DATA, 0, len, data, | 1104 | BLK_TA_DRV_DATA, 0, len, data, |
| 1061 | blk_trace_request_get_cgid(q, rq)); | 1105 | blk_trace_request_get_cgid(q, rq)); |
| 1106 | rcu_read_unlock(); | ||
| 1062 | } | 1107 | } |
| 1063 | EXPORT_SYMBOL_GPL(blk_add_driver_data); | 1108 | EXPORT_SYMBOL_GPL(blk_add_driver_data); |
| 1064 | 1109 | ||
| @@ -1597,6 +1642,7 @@ static int blk_trace_remove_queue(struct request_queue *q) · 532 unmodified lines | |||
| 1597 | return -EINVAL; | 1642 | return -EINVAL; |
| 1598 | 1643 | ||
| 1599 | put_probe_ref(); | 1644 | put_probe_ref(); |
| 1645 | synchronize_rcu(); | ||
| 1600 | blk_trace_free(bt); | 1646 | blk_trace_free(bt); |
| 1601 | return 0; | 1647 | return 0; |
| 1602 | } | 1648 | } |
| @@ -1758,6 +1804,7 @@ static ssize_t sysfs_blk_trace_attr_show(struct device *dev, · 155 unmodified lines | |||
| 1758 | struct hd_struct *p = dev_to_part(dev); | 1804 | struct hd_struct *p = dev_to_part(dev); |
| 1759 | struct request_queue *q; | 1805 | struct request_queue *q; |
| 1760 | struct block_device *bdev; | 1806 | struct block_device *bdev; |
| 1807 | struct blk_trace *bt; | ||
| 1761 | ssize_t ret = -ENXIO; | 1808 | ssize_t ret = -ENXIO; |
| 1762 | 1809 | ||
| 1763 | bdev = bdget(part_devt(p)); | 1810 | bdev = bdget(part_devt(p)); |
| @@ -1770,21 +1817,23 @@ static ssize_t sysfs_blk_trace_attr_show(struct device *dev, · 6 unmodified lines | |||
| 1770 | 1817 | ||
| 1771 | mutex_lock(&q->blk_trace_mutex); | 1818 | mutex_lock(&q->blk_trace_mutex); |
| 1772 | 1819 | ||
| 1820 | bt = rcu_dereference_protected(q->blk_trace, | ||
| 1821 | lockdep_is_held(&q->blk_trace_mutex)); | ||
| 1773 | if (attr == &dev_attr_enable) { | 1822 | if (attr == &dev_attr_enable) { |
| 1774 | ret = sprintf(buf, "%u\n", !!q->blk_trace); | 1823 | ret = sprintf(buf, "%u\n", !!bt); |
| 1775 | goto out_unlock_bdev; | 1824 | goto out_unlock_bdev; |
| 1776 | } | 1825 | } |
| 1777 | 1826 | ||
| 1778 | if (q->blk_trace == NULL) | 1827 | if (bt == NULL) |
| 1779 | ret = sprintf(buf, "disabled\n"); | 1828 | ret = sprintf(buf, "disabled\n"); |
| 1780 | else if (attr == &dev_attr_act_mask) | 1829 | else if (attr == &dev_attr_act_mask) |
| 1781 | ret = blk_trace_mask2str(buf, q->blk_trace->act_mask); | 1830 | ret = blk_trace_mask2str(buf, bt->act_mask); |
| 1782 | else if (attr == &dev_attr_pid) | 1831 | else if (attr == &dev_attr_pid) |
| 1783 | ret = sprintf(buf, "%u\n", q->blk_trace->pid); | 1832 | ret = sprintf(buf, "%u\n", bt->pid); |
| 1784 | else if (attr == &dev_attr_start_lba) | 1833 | else if (attr == &dev_attr_start_lba) |
| 1785 | ret = sprintf(buf, "%llu\n", q->blk_trace->start_lba); | 1834 | ret = sprintf(buf, "%llu\n", bt->start_lba); |
| 1786 | else if (attr == &dev_attr_end_lba) | 1835 | else if (attr == &dev_attr_end_lba) |
| 1787 | ret = sprintf(buf, "%llu\n", q->blk_trace->end_lba); | 1836 | ret = sprintf(buf, "%llu\n", bt->end_lba); |
| 1788 | 1837 | ||
| 1789 | out_unlock_bdev: | 1838 | out_unlock_bdev: |
| 1790 | mutex_unlock(&q->blk_trace_mutex); | 1839 | mutex_unlock(&q->blk_trace_mutex); |
| @@ -1801,6 +1850,7 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev, · 10 unmodified lines | |||
| 1801 | struct block_device *bdev; | 1850 | struct block_device *bdev; |
| 1802 | struct request_queue *q; | 1851 | struct request_queue *q; |
| 1803 | struct hd_struct *p; | 1852 | struct hd_struct *p; |
| 1853 | struct blk_trace *bt; | ||
| 1804 | u64 value; | 1854 | u64 value; |
| 1805 | ssize_t ret = -EINVAL; | 1855 | ssize_t ret = -EINVAL; |
| 1806 | 1856 | ||
| @@ -1831,8 +1881,10 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev, · 24 unmodified lines | |||
| 1831 | 1881 | ||
| 1832 | mutex_lock(&q->blk_trace_mutex); | 1882 | mutex_lock(&q->blk_trace_mutex); |
| 1833 | 1883 | ||
| 1884 | bt = rcu_dereference_protected(q->blk_trace, | ||
| 1885 | lockdep_is_held(&q->blk_trace_mutex)); | ||
| 1834 | if (attr == &dev_attr_enable) { | 1886 | if (attr == &dev_attr_enable) { |
| 1835 | if (!!value == !!q->blk_trace) { | 1887 | if (!!value == !!bt) { |
| 1836 | ret = 0; | 1888 | ret = 0; |
| 1837 | goto out_unlock_bdev; | 1889 | goto out_unlock_bdev; |
| 1838 | } | 1890 | } |
| @@ -1844,18 +1896,21 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev, · 5 unmodified lines | |||
| 1844 | } | 1896 | } |
| 1845 | 1897 | ||
| 1846 | ret = 0; | 1898 | ret = 0; |
| 1847 | if (q->blk_trace == NULL) | 1899 | if (bt == NULL) { |
| 1848 | ret = blk_trace_setup_queue(q, bdev); | 1900 | ret = blk_trace_setup_queue(q, bdev); |
| 1901 | bt = rcu_dereference_protected(q->blk_trace, | ||
| 1902 | lockdep_is_held(&q->blk_trace_mutex)); | ||
| 1903 | } | ||
| 1849 | 1904 | ||
| 1850 | if (ret == 0) { | 1905 | if (ret == 0) { |
| 1851 | if (attr == &dev_attr_act_mask) | 1906 | if (attr == &dev_attr_act_mask) |
| 1852 | q->blk_trace->act_mask = value; | 1907 | bt->act_mask = value; |
| 1853 | else if (attr == &dev_attr_pid) | 1908 | else if (attr == &dev_attr_pid) |
| 1854 | q->blk_trace->pid = value; | 1909 | bt->pid = value; |
| 1855 | else if (attr == &dev_attr_start_lba) | 1910 | else if (attr == &dev_attr_start_lba) |
| 1856 | q->blk_trace->start_lba = value; | 1911 | bt->start_lba = value; |
| 1857 | else if (attr == &dev_attr_end_lba) | 1912 | else if (attr == &dev_attr_end_lba) |
| 1858 | q->blk_trace->end_lba = value; | 1913 | bt->end_lba = value; |
| 1859 | } | 1914 | } |
| 1860 | 1915 | ||
| 1861 | out_unlock_bdev: | 1916 | out_unlock_bdev: |
| @@ -1547,6 +1547,8 @@ static struct dyn_ftrace *lookup_rec(unsigned long start, unsigned long end) · 1546 unmodified lines | |||
| 1547 | rec = bsearch(&key, pg->records, pg->index, | 1547 | rec = bsearch(&key, pg->records, pg->index, |
| 1548 | sizeof(struct dyn_ftrace), | 1548 | sizeof(struct dyn_ftrace), |
| 1549 | ftrace_cmp_recs); | 1549 | ftrace_cmp_recs); |
| 1550 | if (rec) | ||
| 1551 | break; | ||
| 1550 | } | 1552 | } |
| 1551 | return rec; | 1553 | return rec; |
| 1552 | } | 1554 | } |
| @@ -111,11 +111,11 @@ static int __init test_gen_synth_cmd(void) · 110 unmodified lines | |||
| 111 | /* Create some bogus values just for testing */ | 111 | /* Create some bogus values just for testing */ |
| 112 | 112 | ||
| 113 | vals[0] = 777; /* next_pid_field */ | 113 | vals[0] = 777; /* next_pid_field */ |
| 114 | vals[1] = (u64)"hula hoops"; /* next_comm_field */ | 114 | vals[1] = (u64)(long)"hula hoops"; /* next_comm_field */ |
| 115 | vals[2] = 1000000; /* ts_ns */ | 115 | vals[2] = 1000000; /* ts_ns */ |
| 116 | vals[3] = 1000; /* ts_ms */ | 116 | vals[3] = 1000; /* ts_ms */ |
| 117 | vals[4] = smp_processor_id(); /* cpu */ | 117 | vals[4] = raw_smp_processor_id(); /* cpu */ |
| 118 | vals[5] = (u64)"thneed"; /* my_string_field */ | 118 | vals[5] = (u64)(long)"thneed"; /* my_string_field */ |
| 119 | vals[6] = 598; /* my_int_field */ | 119 | vals[6] = 598; /* my_int_field */ |
| 120 | 120 | ||
| 121 | /* Now generate a gen_synth_test event */ | 121 | /* Now generate a gen_synth_test event */ |
| @@ -218,11 +218,11 @@ static int __init test_empty_synth_event(void) · 96 unmodified lines | |||
| 218 | /* Create some bogus values just for testing */ | 218 | /* Create some bogus values just for testing */ |
| 219 | 219 | ||
| 220 | vals[0] = 777; /* next_pid_field */ | 220 | vals[0] = 777; /* next_pid_field */ |
| 221 | vals[1] = (u64)"tiddlywinks"; /* next_comm_field */ | 221 | vals[1] = (u64)(long)"tiddlywinks"; /* next_comm_field */ |
| 222 | vals[2] = 1000000; /* ts_ns */ | 222 | vals[2] = 1000000; /* ts_ns */ |
| 223 | vals[3] = 1000; /* ts_ms */ | 223 | vals[3] = 1000; /* ts_ms */ |
| 224 | vals[4] = smp_processor_id(); /* cpu */ | 224 | vals[4] = raw_smp_processor_id(); /* cpu */ |
| 225 | vals[5] = (u64)"thneed_2.0"; /* my_string_field */ | 225 | vals[5] = (u64)(long)"thneed_2.0"; /* my_string_field */ |
| 226 | vals[6] = 399; /* my_int_field */ | 226 | vals[6] = 399; /* my_int_field */ |
| 227 | 227 | ||
| 228 | /* Now trace an empty_synth_test event */ | 228 | /* Now trace an empty_synth_test event */ |
| @@ -290,11 +290,11 @@ static int __init test_create_synth_event(void) · 61 unmodified lines | |||
| 290 | /* Create some bogus values just for testing */ | 290 | /* Create some bogus values just for testing */ |
| 291 | 291 | ||
| 292 | vals[0] = 777; /* next_pid_field */ | 292 | vals[0] = 777; /* next_pid_field */ |
| 293 | vals[1] = (u64)"tiddlywinks"; /* next_comm_field */ | 293 | vals[1] = (u64)(long)"tiddlywinks"; /* next_comm_field */ |
| 294 | vals[2] = 1000000; /* ts_ns */ | 294 | vals[2] = 1000000; /* ts_ns */ |
| 295 | vals[3] = 1000; /* ts_ms */ | 295 | vals[3] = 1000; /* ts_ms */ |
| 296 | vals[4] = smp_processor_id(); /* cpu */ | 296 | vals[4] = raw_smp_processor_id(); /* cpu */ |
| 297 | vals[5] = (u64)"thneed"; /* my_string_field */ | 297 | vals[5] = (u64)(long)"thneed"; /* my_string_field */ |
| 298 | vals[6] = 398; /* my_int_field */ | 298 | vals[6] = 398; /* my_int_field */ |
| 299 | 299 | ||
| 300 | /* Now generate a create_synth_test event */ | 300 | /* Now generate a create_synth_test event */ |
| @@ -330,7 +330,7 @@ static int __init test_add_next_synth_val(void) · 29 unmodified lines | |||
| 330 | goto out; | 330 | goto out; |
| 331 | 331 | ||
| 332 | /* next_comm_field */ | 332 | /* next_comm_field */ |
| 333 | ret = synth_event_add_next_val((u64)"slinky", &trace_state); | 333 | ret = synth_event_add_next_val((u64)(long)"slinky", &trace_state); |
| 334 | if (ret) | 334 | if (ret) |
| 335 | goto out; | 335 | goto out; |
| 336 | 336 | ||
| @@ -345,12 +345,12 @@ static int __init test_add_next_synth_val(void) · 8 unmodified lines | |||
| 345 | goto out; | 345 | goto out; |
| 346 | 346 | ||
| 347 | /* cpu */ | 347 | /* cpu */ |
| 348 | ret = synth_event_add_next_val(smp_processor_id(), &trace_state); | 348 | ret = synth_event_add_next_val(raw_smp_processor_id(), &trace_state); |
| 349 | if (ret) | 349 | if (ret) |
| 350 | goto out; | 350 | goto out; |
| 351 | 351 | ||
| 352 | /* my_string_field */ | 352 | /* my_string_field */ |
| 353 | ret = synth_event_add_next_val((u64)"thneed_2.01", &trace_state); | 353 | ret = synth_event_add_next_val((u64)(long)"thneed_2.01", &trace_state); |
| 354 | if (ret) | 354 | if (ret) |
| 355 | goto out; | 355 | goto out; |
| 356 | 356 | ||
| @@ -388,7 +388,7 @@ static int __init test_add_synth_val(void) · 31 unmodified lines | |||
| 388 | if (ret) | 388 | if (ret) |
| 389 | goto out; | 389 | goto out; |
| 390 | 390 | ||
| 391 | ret = synth_event_add_val("cpu", smp_processor_id(), &trace_state); | 391 | ret = synth_event_add_val("cpu", raw_smp_processor_id(), &trace_state); |
| 392 | if (ret) | 392 | if (ret) |
| 393 | goto out; | 393 | goto out; |
| 394 | 394 | ||
| @@ -396,12 +396,12 @@ static int __init test_add_synth_val(void) · 1 unmodified line | |||
| 396 | if (ret) | 396 | if (ret) |
| 397 | goto out; | 397 | goto out; |
| 398 | 398 | ||
| 399 | ret = synth_event_add_val("next_comm_field", (u64)"silly putty", | 399 | ret = synth_event_add_val("next_comm_field", (u64)(long)"silly putty", |
| 400 | &trace_state); | 400 | &trace_state); |
| 401 | if (ret) | 401 | if (ret) |
| 402 | goto out; | 402 | goto out; |
| 403 | 403 | ||
| 404 | ret = synth_event_add_val("my_string_field", (u64)"thneed_9", | 404 | ret = synth_event_add_val("my_string_field", (u64)(long)"thneed_9", |
| 405 | &trace_state); | 405 | &trace_state); |
| 406 | if (ret) | 406 | if (ret) |
| 407 | goto out; | 407 | goto out; |
| @@ -423,13 +423,13 @@ static int __init test_trace_synth_event(void) · 15 unmodified lines | |||
| 423 | 423 | ||
| 424 | /* Trace some bogus values just for testing */ | 424 | /* Trace some bogus values just for testing */ |
| 425 | ret = synth_event_trace(create_synth_test, 7, /* number of values */ | 425 | ret = synth_event_trace(create_synth_test, 7, /* number of values */ |
| 426 | 444, /* next_pid_field */ | 426 | (u64)444, /* next_pid_field */ |
| 427 | (u64)"clackers", /* next_comm_field */ | 427 | (u64)(long)"clackers", /* next_comm_field */ |
| 428 | 1000000, /* ts_ns */ | 428 | (u64)1000000, /* ts_ns */ |
| 429 | 1000, /* ts_ms */ | 429 | (u64)1000, /* ts_ms */ |
| 430 | smp_processor_id(), /* cpu */ | 430 | (u64)raw_smp_processor_id(), /* cpu */ |
| 431 | (u64)"Thneed", /* my_string_field */ | 431 | (u64)(long)"Thneed", /* my_string_field */ |
| 432 | 999); /* my_int_field */ | 432 | (u64)999); /* my_int_field */ |
| 433 | return ret; | 433 | return ret; |
| 434 | } | 434 | } |
| 435 | 435 | ||
| @@ -1837,6 +1837,7 @@ static __init int init_trace_selftests(void) · 1836 unmodified lines | |||
| 1837 | 1837 | ||
| 1838 | pr_info("Running postponed tracer tests:\n"); | 1838 | pr_info("Running postponed tracer tests:\n"); |
| 1839 | 1839 | ||
| 1840 | tracing_selftest_running = true; | ||
| 1840 | list_for_each_entry_safe(p, n, &postponed_selftests, list) { | 1841 | list_for_each_entry_safe(p, n, &postponed_selftests, list) { |
| 1841 | /* This loop can take minutes when sanitizers are enabled, so | 1842 | /* This loop can take minutes when sanitizers are enabled, so |
| 1842 | * lets make sure we allow RCU processing. | 1843 | * lets make sure we allow RCU processing. |
| @@ -1859,6 +1860,7 @@ static __init int init_trace_selftests(void) · 16 unmodified lines | |||
| 1859 | list_del(&p->list); | 1860 | list_del(&p->list); |
| 1860 | kfree(p); | 1861 | kfree(p); |
| 1861 | } | 1862 | } |
| 1863 | tracing_selftest_running = false; | ||
| 1862 | 1864 | ||
| 1863 | out: | 1865 | out: |
| 1864 | mutex_unlock(&trace_types_lock); | 1866 | mutex_unlock(&trace_types_lock); |
| @@ -821,6 +821,29 @@ static const char *synth_field_fmt(char *type) · 820 unmodified lines | |||
| 821 | return fmt; | 821 | return fmt; |
| 822 | } | 822 | } |
| 823 | 823 | ||
| 824 | static void print_synth_event_num_val(struct trace_seq *s, | ||
| 825 | char *print_fmt, char *name, | ||
| 826 | int size, u64 val, char *space) | ||
| 827 | { | ||
| 828 | switch (size) { | ||
| 829 | case 1: | ||
| 830 | trace_seq_printf(s, print_fmt, name, (u8)val, space); | ||
| 831 | break; | ||
| 832 | |||
| 833 | case 2: | ||
| 834 | trace_seq_printf(s, print_fmt, name, (u16)val, space); | ||
| 835 | break; | ||
| 836 | |||
| 837 | case 4: | ||
| 838 | trace_seq_printf(s, print_fmt, name, (u32)val, space); | ||
| 839 | break; | ||
| 840 | |||
| 841 | default: | ||
| 842 | trace_seq_printf(s, print_fmt, name, val, space); | ||
| 843 | break; | ||
| 844 | } | ||
| 845 | } | ||
| 846 | |||
| 824 | static enum print_line_t print_synth_event(struct trace_iterator *iter, | 847 | static enum print_line_t print_synth_event(struct trace_iterator *iter, |
| 825 | int flags, | 848 | int flags, |
| 826 | struct trace_event *event) | 849 | struct trace_event *event) |
| @@ -859,10 +882,13 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter, · 32 unmodified lines | |||
| 859 | } else { | 882 | } else { |
| 860 | struct trace_print_flags __flags[] = { | 883 | struct trace_print_flags __flags[] = { |
| 861 | __def_gfpflag_names, {-1, NULL} }; | 884 | __def_gfpflag_names, {-1, NULL} }; |
| 885 | char *space = (i == se->n_fields - 1 ? "" : " "); | ||
| 862 | 886 | ||
| 863 | trace_seq_printf(s, print_fmt, se->fields[i]->name, | 887 | print_synth_event_num_val(s, print_fmt, |
| 864 | entry->fields[n_u64], | 888 | se->fields[i]->name, |
| 865 | i == se->n_fields - 1 ? "" : " "); | 889 | se->fields[i]->size, |
| 890 | entry->fields[n_u64], | ||
| 891 | space); | ||
| 866 | 892 | ||
| 867 | if (strcmp(se->fields[i]->type, "gfp_t") == 0) { | 893 | if (strcmp(se->fields[i]->type, "gfp_t") == 0) { |
| 868 | trace_seq_puts(s, " ("); | 894 | trace_seq_puts(s, " ("); |
| @@ -1805,6 +1831,8 @@ __synth_event_trace_start(struct trace_event_file *file, · 936 unmodified lines | |||
| 1805 | int entry_size, fields_size = 0; | 1831 | int entry_size, fields_size = 0; |
| 1806 | int ret = 0; | 1832 | int ret = 0; |
| 1807 | 1833 | ||
| 1834 | memset(trace_state, '\0', sizeof(*trace_state)); | ||
| 1835 | |||
| 1808 | /* | 1836 | /* |
| 1809 | * Normal event tracing doesn't get called at all unless the | 1837 | * Normal event tracing doesn't get called at all unless the |
| 1810 | * ENABLED bit is set (which attaches the probe thus allowing | 1838 | * ENABLED bit is set (which attaches the probe thus allowing |
| @@ -1885,6 +1913,11 @@ int synth_event_trace(struct trace_event_file *file, unsigned int n_vals, ...) · 74 unmodified lines | |||
| 1885 | return ret; | 1913 | return ret; |
| 1886 | } | 1914 | } |
| 1887 | 1915 | ||
| 1916 | if (n_vals != state.event->n_fields) { | ||
| 1917 | ret = -EINVAL; | ||
| 1918 | goto out; | ||
| 1919 | } | ||
| 1920 | |||
| 1888 | va_start(args, n_vals); | 1921 | va_start(args, n_vals); |
| 1889 | for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) { | 1922 | for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) { |
| 1890 | u64 val; | 1923 | u64 val; |
| @@ -1898,12 +1931,30 @@ int synth_event_trace(struct trace_event_file *file, unsigned int n_vals, ...) · 7 unmodified lines | |||
| 1898 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); | 1931 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); |
| 1899 | n_u64 += STR_VAR_LEN_MAX / sizeof(u64); | 1932 | n_u64 += STR_VAR_LEN_MAX / sizeof(u64); |
| 1900 | } else { | 1933 | } else { |
| 1901 | state.entry->fields[n_u64] = val; | 1934 | struct synth_field *field = state.event->fields[i]; |
| 1935 | |||
| 1936 | switch (field->size) { | ||
| 1937 | case 1: | ||
| 1938 | *(u8 *)&state.entry->fields[n_u64] = (u8)val; | ||
| 1939 | break; | ||
| 1940 | |||
| 1941 | case 2: | ||
| 1942 | *(u16 *)&state.entry->fields[n_u64] = (u16)val; | ||
| 1943 | break; | ||
| 1944 | |||
| 1945 | case 4: | ||
| 1946 | *(u32 *)&state.entry->fields[n_u64] = (u32)val; | ||
| 1947 | break; | ||
| 1948 | |||
| 1949 | default: | ||
| 1950 | state.entry->fields[n_u64] = val; | ||
| 1951 | break; | ||
| 1952 | } | ||
| 1902 | n_u64++; | 1953 | n_u64++; |
| 1903 | } | 1954 | } |
| 1904 | } | 1955 | } |
| 1905 | va_end(args); | 1956 | va_end(args); |
| 1906 | 1957 | out: | |
| 1907 | __synth_event_trace_end(&state); | 1958 | __synth_event_trace_end(&state); |
| 1908 | 1959 | ||
| 1909 | return ret; | 1960 | return ret; |
| @@ -1942,6 +1993,11 @@ int synth_event_trace_array(struct trace_event_file *file, u64 *vals, · 32 unmodified lines | |||
| 1942 | return ret; | 1993 | return ret; |
| 1943 | } | 1994 | } |
| 1944 | 1995 | ||
| 1996 | if (n_vals != state.event->n_fields) { | ||
| 1997 | ret = -EINVAL; | ||
| 1998 | goto out; | ||
| 1999 | } | ||
| 2000 | |||
| 1945 | for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) { | 2001 | for (i = 0, n_u64 = 0; i < state.event->n_fields; i++) { |
| 1946 | if (state.event->fields[i]->is_string) { | 2002 | if (state.event->fields[i]->is_string) { |
| 1947 | char *str_val = (char *)(long)vals[i]; | 2003 | char *str_val = (char *)(long)vals[i]; |
| @@ -1950,11 +2006,30 @@ int synth_event_trace_array(struct trace_event_file *file, u64 *vals, · 2 unmodified lines | |||
| 1950 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); | 2006 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); |
| 1951 | n_u64 += STR_VAR_LEN_MAX / sizeof(u64); | 2007 | n_u64 += STR_VAR_LEN_MAX / sizeof(u64); |
| 1952 | } else { | 2008 | } else { |
| 1953 | state.entry->fields[n_u64] = vals[i]; | 2009 | struct synth_field *field = state.event->fields[i]; |
| 2010 | u64 val = vals[i]; | ||
| 2011 | |||
| 2012 | switch (field->size) { | ||
| 2013 | case 1: | ||
| 2014 | *(u8 *)&state.entry->fields[n_u64] = (u8)val; | ||
| 2015 | break; | ||
| 2016 | |||
| 2017 | case 2: | ||
| 2018 | *(u16 *)&state.entry->fields[n_u64] = (u16)val; | ||
| 2019 | break; | ||
| 2020 | |||
| 2021 | case 4: | ||
| 2022 | *(u32 *)&state.entry->fields[n_u64] = (u32)val; | ||
| 2023 | break; | ||
| 2024 | |||
| 2025 | default: | ||
| 2026 | state.entry->fields[n_u64] = val; | ||
| 2027 | break; | ||
| 2028 | } | ||
| 1954 | n_u64++; | 2029 | n_u64++; |
| 1955 | } | 2030 | } |
| 1956 | } | 2031 | } |
| 1957 | 2032 | out: | |
| 1958 | __synth_event_trace_end(&state); | 2033 | __synth_event_trace_end(&state); |
| 1959 | 2034 | ||
| 1960 | return ret; | 2035 | return ret; |
| @@ -1997,8 +2072,6 @@ int synth_event_trace_start(struct trace_event_file *file, · 36 unmodified lines | |||
| 1997 | if (!trace_state) | 2072 | if (!trace_state) |
| 1998 | return -EINVAL; | 2073 | return -EINVAL; |
| 1999 | 2074 | ||
| 2000 | memset(trace_state, '\0', sizeof(*trace_state)); | ||
| 2001 | |||
| 2002 | ret = __synth_event_trace_start(file, trace_state); | 2075 | ret = __synth_event_trace_start(file, trace_state); |
| 2003 | if (ret == -ENOENT) | 2076 | if (ret == -ENOENT) |
| 2004 | ret = 0; /* just disabled, not really an error */ | 2077 | ret = 0; /* just disabled, not really an error */ |
| @@ -2069,8 +2142,25 @@ static int __synth_event_add_val(const char *field_name, u64 val, · 64 unmodified lines | |||
| 2069 | 2142 | ||
| 2070 | str_field = (char *)&entry->fields[field->offset]; | 2143 | str_field = (char *)&entry->fields[field->offset]; |
| 2071 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); | 2144 | strscpy(str_field, str_val, STR_VAR_LEN_MAX); |
| 2072 | } else | 2145 | } else { |
| 2073 | entry->fields[field->offset] = val; | 2146 | switch (field->size) { |
| 2147 | case 1: | ||
| 2148 | *(u8 *)&trace_state->entry->fields[field->offset] = (u8)val; | ||
| 2149 | break; | ||
| 2150 | |||
| 2151 | case 2: | ||
| 2152 | *(u16 *)&trace_state->entry->fields[field->offset] = (u16)val; | ||
| 2153 | break; | ||
| 2154 | |||
| 2155 | case 4: | ||
| 2156 | *(u32 *)&trace_state->entry->fields[field->offset] = (u32)val; | ||
| 2157 | break; | ||
| 2158 | |||
| 2159 | default: | ||
| 2160 | trace_state->entry->fields[field->offset] = val; | ||
| 2161 | break; | ||
| 2162 | } | ||
| 2163 | } | ||
| 2074 | out: | 2164 | out: |
| 2075 | return ret; | 2165 | return ret; |
| 2076 | } | 2166 | } |
| @@ -1411,14 +1411,16 @@ static void __queue_work(int cpu, struct workqueue_struct *wq, · 1410 unmodified lines | |||
| 1411 | return; | 1411 | return; |
| 1412 | rcu_read_lock(); | 1412 | rcu_read_lock(); |
| 1413 | retry: | 1413 | retry: |
| 1414 | if (req_cpu == WORK_CPU_UNBOUND) | ||
| 1415 | cpu = wq_select_unbound_cpu(raw_smp_processor_id()); | ||
| 1416 | |||
| 1417 | /* pwq which will be used unless @work is executing elsewhere */ | 1414 | /* pwq which will be used unless @work is executing elsewhere */ |
| 1418 | if (!(wq->flags & WQ_UNBOUND)) | 1415 | if (wq->flags & WQ_UNBOUND) { |
| 1419 | pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); | 1416 | if (req_cpu == WORK_CPU_UNBOUND) |
| 1420 | else | 1417 | cpu = wq_select_unbound_cpu(raw_smp_processor_id()); |
| 1421 | pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); | 1418 | pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); |
| 1419 | } else { | ||
| 1420 | if (req_cpu == WORK_CPU_UNBOUND) | ||
| 1421 | cpu = raw_smp_processor_id(); | ||
| 1422 | pwq = per_cpu_ptr(wq->cpu_pwqs, cpu); | ||
| 1423 | } | ||
| 1422 | 1424 | ||
| 1423 | /* | 1425 | /* |
| 1424 | * If @work was previously on a different pool, it might still be | 1426 | * If @work was previously on a different pool, it might still be |