| @@ -658,6 +658,116 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs, · 657 unmodified lines |
| 658 | 658 | return NULL; |
| 659 | 659 | } |
| 660 | 660 | |
| 661 | +static struct bpf_link *bpf_get_anchor_link(u32 flags, u32 id_or_fd) |
| 662 | +{ |
| 663 | + struct bpf_link *link = ERR_PTR(-EINVAL); |
| 664 | + |
| 665 | + if (flags & BPF_F_ID) |
| 666 | + link = bpf_link_by_id(id_or_fd); |
| 667 | + else if (id_or_fd) |
| 668 | + link = bpf_link_get_from_fd(id_or_fd); |
| 669 | + return link; |
| 670 | +} |
| 671 | + |
| 672 | +static struct bpf_prog *bpf_get_anchor_prog(u32 flags, u32 id_or_fd) |
| 673 | +{ |
| 674 | + struct bpf_prog *prog = ERR_PTR(-EINVAL); |
| 675 | + |
| 676 | + if (flags & BPF_F_ID) |
| 677 | + prog = bpf_prog_by_id(id_or_fd); |
| 678 | + else if (id_or_fd) |
| 679 | + prog = bpf_prog_get(id_or_fd); |
| 680 | + return prog; |
| 681 | +} |
| 682 | + |
| 683 | +static struct bpf_prog_list *get_prog_list(struct hlist_head *progs, struct bpf_prog *prog, |
| 684 | + struct bpf_cgroup_link *link, u32 flags, u32 id_or_fd) |
| 685 | +{ |
| 686 | + bool is_link = flags & BPF_F_LINK, is_id = flags & BPF_F_ID; |
| 687 | + struct bpf_prog_list *pltmp, *pl = ERR_PTR(-EINVAL); |
| 688 | + bool preorder = flags & BPF_F_PREORDER; |
| 689 | + struct bpf_link *anchor_link = NULL; |
| 690 | + struct bpf_prog *anchor_prog = NULL; |
| 691 | + bool is_before, is_after; |
| 692 | + |
| 693 | + is_before = flags & BPF_F_BEFORE; |
| 694 | + is_after = flags & BPF_F_AFTER; |
| 695 | + if (is_link || is_id || id_or_fd) { |
| 696 | + /* flags must have either BPF_F_BEFORE or BPF_F_AFTER */ |
| 697 | + if (is_before == is_after) |
| 698 | + return ERR_PTR(-EINVAL); |
| 699 | + if ((is_link && !link) || (!is_link && !prog)) |
| 700 | + return ERR_PTR(-EINVAL); |
| 701 | + } else if (!hlist_empty(progs)) { |
| 702 | + /* flags cannot have both BPF_F_BEFORE and BPF_F_AFTER */ |
| 703 | + if (is_before && is_after) |
| 704 | + return ERR_PTR(-EINVAL); |
| 705 | + } |
| 706 | + |
| 707 | + if (is_link) { |
| 708 | + anchor_link = bpf_get_anchor_link(flags, id_or_fd); |
| 709 | + if (IS_ERR(anchor_link)) |
| 710 | + return ERR_PTR(PTR_ERR(anchor_link)); |
| 711 | + } else if (is_id || id_or_fd) { |
| 712 | + anchor_prog = bpf_get_anchor_prog(flags, id_or_fd); |
| 713 | + if (IS_ERR(anchor_prog)) |
| 714 | + return ERR_PTR(PTR_ERR(anchor_prog)); |
| 715 | + } |
| 716 | + |
| 717 | + if (!anchor_prog && !anchor_link) { |
| 718 | + /* if there is no anchor_prog/anchor_link, then BPF_F_PREORDER |
| 719 | + * doesn't matter since either prepend or append to a combined |
| 720 | + * list of progs will end up with correct result. |
| 721 | + */ |
| 722 | + hlist_for_each_entry(pltmp, progs, node) { |
| 723 | + if (is_before) |
| 724 | + return pltmp; |
| 725 | + if (pltmp->node.next) |
| 726 | + continue; |
| 727 | + return pltmp; |
| 728 | + } |
| 729 | + return NULL; |
| 730 | + } |
| 731 | + |
| 732 | + hlist_for_each_entry(pltmp, progs, node) { |
| 733 | + if ((anchor_prog && anchor_prog == pltmp->prog) || |
| 734 | + (anchor_link && anchor_link == &pltmp->link->link)) { |
| 735 | + if (!!(pltmp->flags & BPF_F_PREORDER) != preorder) |
| 736 | + goto out; |
| 737 | + pl = pltmp; |
| 738 | + goto out; |
| 739 | + } |
| 740 | + } |
| 741 | + |
| 742 | + pl = ERR_PTR(-ENOENT); |
| 743 | +out: |
| 744 | + if (anchor_link) |
| 745 | + bpf_link_put(anchor_link); |
| 746 | + else |
| 747 | + bpf_prog_put(anchor_prog); |
| 748 | + return pl; |
| 749 | +} |
| 750 | + |
| 751 | +static int insert_pl_to_hlist(struct bpf_prog_list *pl, struct hlist_head *progs, |
| 752 | + struct bpf_prog *prog, struct bpf_cgroup_link *link, |
| 753 | + u32 flags, u32 id_or_fd) |
| 754 | +{ |
| 755 | + struct bpf_prog_list *pltmp; |
| 756 | + |
| 757 | + pltmp = get_prog_list(progs, prog, link, flags, id_or_fd); |
| 758 | + if (IS_ERR(pltmp)) |
| 759 | + return PTR_ERR(pltmp); |
| 760 | + |
| 761 | + if (!pltmp) |
| 762 | + hlist_add_head(&pl->node, progs); |
| 763 | + else if (flags & BPF_F_BEFORE) |
| 764 | + hlist_add_before(&pl->node, &pltmp->node); |
| 765 | + else |
| 766 | + hlist_add_behind(&pl->node, &pltmp->node); |
| 767 | + |
| 768 | + return 0; |
| 769 | +} |
| 770 | + |
| 661 | 771 | /** |
| 662 | 772 | * __cgroup_bpf_attach() - Attach the program or the link to a cgroup, and |
| 663 | 773 | * propagate the change to descendants |
| @@ -667,6 +777,8 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs, · 3 unmodified lines |
| 667 | 777 | * @replace_prog: Previously attached program to replace if BPF_F_REPLACE is set |
| 668 | 778 | * @type: Type of attach operation |
| 669 | 779 | * @flags: Option flags |
| 780 | + * @id_or_fd: Relative prog id or fd |
| 781 | + * @revision: bpf_prog_list revision |
| 670 | 782 | * |
| 671 | 783 | * Exactly one of @prog or @link can be non-null. |
| 672 | 784 | * Must be called with cgroup_mutex held. |
| @@ -674,7 +786,8 @@ static struct bpf_prog_list *find_attach_entry(struct hlist_head *progs, · 1 unmodified line |
| 674 | 786 | static int __cgroup_bpf_attach(struct cgroup *cgrp, |
| 675 | 787 | struct bpf_prog *prog, struct bpf_prog *replace_prog, |
| 676 | 788 | struct bpf_cgroup_link *link, |
| 677 | | − enum bpf_attach_type type, u32 flags) |
| 789 | + enum bpf_attach_type type, u32 flags, u32 id_or_fd, |
| 790 | + u64 revision) |
| 678 | 791 | { |
| 679 | 792 | u32 saved_flags = (flags & (BPF_F_ALLOW_OVERRIDE | BPF_F_ALLOW_MULTI)); |
| 680 | 793 | struct bpf_prog *old_prog = NULL; |
| @@ -690,6 +803,9 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 9 unmodified lines |
| 690 | 803 | ((flags & BPF_F_REPLACE) && !(flags & BPF_F_ALLOW_MULTI))) |
| 691 | 804 | /* invalid combination */ |
| 692 | 805 | return -EINVAL; |
| 806 | + if ((flags & BPF_F_REPLACE) && (flags & (BPF_F_BEFORE | BPF_F_AFTER))) |
| 807 | + /* only either replace or insertion with before/after */ |
| 808 | + return -EINVAL; |
| 693 | 809 | if (link && (prog || replace_prog)) |
| 694 | 810 | /* only either link or prog/replace_prog can be specified */ |
| 695 | 811 | return -EINVAL; |
| @@ -700,6 +816,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 4 unmodified lines |
| 700 | 816 | atype = bpf_cgroup_atype_find(type, new_prog->aux->attach_btf_id); |
| 701 | 817 | if (atype < 0) |
| 702 | 818 | return -EINVAL; |
| 819 | + if (revision && revision != cgrp->bpf.revisions[atype]) |
| 820 | + return -ESTALE; |
| 703 | 821 | |
| 704 | 822 | progs = &cgrp->bpf.progs[atype]; |
| 705 | 823 | |
| @@ -728,22 +846,18 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 22 unmodified lines |
| 728 | 846 | if (pl) { |
| 729 | 847 | old_prog = pl->prog; |
| 730 | 848 | } else { |
| 731 | | − struct hlist_node *last = NULL; |
| 732 | | − |
| 733 | 849 | pl = kmalloc(sizeof(*pl), GFP_KERNEL); |
| 734 | 850 | if (!pl) { |
| 735 | 851 | bpf_cgroup_storages_free(new_storage); |
| 736 | 852 | return -ENOMEM; |
| 737 | 853 | } |
| 738 | | − if (hlist_empty(progs)) |
| 739 | | − hlist_add_head(&pl->node, progs); |
| 740 | | − else |
| 741 | | − hlist_for_each(last, progs) { |
| 742 | | − if (last->next) |
| 743 | | − continue; |
| 744 | | − hlist_add_behind(&pl->node, last); |
| 745 | | − break; |
| 746 | | − } |
| 854 | + |
| 855 | + err = insert_pl_to_hlist(pl, progs, prog, link, flags, id_or_fd); |
| 856 | + if (err) { |
| 857 | + kfree(pl); |
| 858 | + bpf_cgroup_storages_free(new_storage); |
| 859 | + return err; |
| 860 | + } |
| 747 | 861 | } |
| 748 | 862 | |
| 749 | 863 | pl->prog = prog; |
| @@ -762,6 +876,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, · 12 unmodified lines |
| 762 | 876 | if (err) |
| 763 | 877 | goto cleanup_trampoline; |
| 764 | 878 | |
| 879 | + cgrp->bpf.revisions[atype] += 1; |
| 765 | 880 | if (old_prog) { |
| 766 | 881 | if (type == BPF_LSM_CGROUP) |
| 767 | 882 | bpf_trampoline_unlink_cgroup_shim(old_prog); |
| @@ -793,12 +908,13 @@ static int cgroup_bpf_attach(struct cgroup *cgrp, · 25 unmodified lines |
| 793 | 908 | struct bpf_prog *prog, struct bpf_prog *replace_prog, |
| 794 | 909 | struct bpf_cgroup_link *link, |
| 795 | 910 | enum bpf_attach_type type, |
| 796 | | − u32 flags) |
| 911 | + u32 flags, u32 id_or_fd, u64 revision) |
| 797 | 912 | { |
| 798 | 913 | int ret; |
| 799 | 914 | |
| 800 | 915 | cgroup_lock(); |
| 801 | | − ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags); |
| 916 | + ret = __cgroup_bpf_attach(cgrp, prog, replace_prog, link, type, flags, |
| 917 | + id_or_fd, revision); |
| 802 | 918 | cgroup_unlock(); |
| 803 | 919 | return ret; |
| 804 | 920 | } |
| @@ -886,6 +1002,7 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, · 81 unmodified lines |
| 886 | 1002 | if (!found) |
| 887 | 1003 | return -ENOENT; |
| 888 | 1004 | |
| 1005 | + cgrp->bpf.revisions[atype] += 1; |
| 889 | 1006 | old_prog = xchg(&link->link.prog, new_prog); |
| 890 | 1007 | replace_effective_prog(cgrp, atype, link); |
| 891 | 1008 | bpf_prog_put(old_prog); |
| @@ -1011,12 +1128,14 @@ found: · 119 unmodified lines |
| 1011 | 1128 | * @prog: A program to detach or NULL |
| 1012 | 1129 | * @link: A link to detach or NULL |
| 1013 | 1130 | * @type: Type of detach operation |
| 1131 | + * @revision: bpf_prog_list revision |
| 1014 | 1132 | * |
| 1015 | 1133 | * At most one of @prog or @link can be non-NULL. |
| 1016 | 1134 | * Must be called with cgroup_mutex held. |
| 1017 | 1135 | */ |
| 1018 | 1136 | static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, |
| 1019 | | − struct bpf_cgroup_link *link, enum bpf_attach_type type) |
| 1137 | + struct bpf_cgroup_link *link, enum bpf_attach_type type, |
| 1138 | + u64 revision) |
| 1020 | 1139 | { |
| 1021 | 1140 | enum cgroup_bpf_attach_type atype; |
| 1022 | 1141 | struct bpf_prog *old_prog; |
| @@ -1034,6 +1153,9 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 11 unmodified lines |
| 1034 | 1153 | if (atype < 0) |
| 1035 | 1154 | return -EINVAL; |
| 1036 | 1155 | |
| 1156 | + if (revision && revision != cgrp->bpf.revisions[atype]) |
| 1157 | + return -ESTALE; |
| 1158 | + |
| 1037 | 1159 | progs = &cgrp->bpf.progs[atype]; |
| 1038 | 1160 | flags = cgrp->bpf.flags[atype]; |
| 1039 | 1161 | |
| @@ -1059,6 +1181,7 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 19 unmodified lines |
| 1059 | 1181 | |
| 1060 | 1182 | /* now can actually delete it from this cgroup list */ |
| 1061 | 1183 | hlist_del(&pl->node); |
| 1184 | + cgrp->bpf.revisions[atype] += 1; |
| 1062 | 1185 | |
| 1063 | 1186 | kfree(pl); |
| 1064 | 1187 | if (hlist_empty(progs)) |
| @@ -1074,12 +1197,12 @@ static int __cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, · 9 unmodified lines |
| 1074 | 1197 | } |
| 1075 | 1198 | |
| 1076 | 1199 | static int cgroup_bpf_detach(struct cgroup *cgrp, struct bpf_prog *prog, |
| 1077 | | − enum bpf_attach_type type) |
| 1200 | + enum bpf_attach_type type, u64 revision) |
| 1078 | 1201 | { |
| 1079 | 1202 | int ret; |
| 1080 | 1203 | |
| 1081 | 1204 | cgroup_lock(); |
| 1082 | | − ret = __cgroup_bpf_detach(cgrp, prog, NULL, type); |
| 1205 | + ret = __cgroup_bpf_detach(cgrp, prog, NULL, type, revision); |
| 1083 | 1206 | cgroup_unlock(); |
| 1084 | 1207 | return ret; |
| 1085 | 1208 | } |
| @@ -1097,6 +1220,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, · 11 unmodified lines |
| 1097 | 1220 | struct bpf_prog_array *effective; |
| 1098 | 1221 | int cnt, ret = 0, i; |
| 1099 | 1222 | int total_cnt = 0; |
| 1223 | + u64 revision = 0; |
| 1100 | 1224 | u32 flags; |
| 1101 | 1225 | |
| 1102 | 1226 | if (effective_query && prog_attach_flags) |
| @@ -1134,6 +1258,10 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr, · 31 unmodified lines |
| 1134 | 1258 | return -EFAULT; |
| 1135 | 1259 | if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt))) |
| 1136 | 1260 | return -EFAULT; |
| 1261 | + if (!effective_query && from_atype == to_atype) |
| 1262 | + revision = cgrp->bpf.revisions[from_atype]; |
| 1263 | + if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) |
| 1264 | + return -EFAULT; |
| 1137 | 1265 | if (attr->query.prog_cnt == 0 || !prog_ids || !total_cnt) |
| 1138 | 1266 | /* return early if user requested only program count + flags */ |
| 1139 | 1267 | return 0; |
| @@ -1216,7 +1344,8 @@ int cgroup_bpf_prog_attach(const union bpf_attr *attr, · 76 unmodified lines |
| 1216 | 1344 | } |
| 1217 | 1345 | |
| 1218 | 1346 | ret = cgroup_bpf_attach(cgrp, prog, replace_prog, NULL, |
| 1219 | | − attr->attach_type, attr->attach_flags); |
| 1347 | + attr->attach_type, attr->attach_flags, |
| 1348 | + attr->relative_fd, attr->expected_revision); |
| 1220 | 1349 | |
| 1221 | 1350 | if (replace_prog) |
| 1222 | 1351 | bpf_prog_put(replace_prog); |
| @@ -1238,7 +1367,7 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype) · 15 unmodified lines |
| 1238 | 1367 | if (IS_ERR(prog)) |
| 1239 | 1368 | prog = NULL; |
| 1240 | 1369 | |
| 1241 | | − ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type); |
| 1370 | + ret = cgroup_bpf_detach(cgrp, prog, attr->attach_type, attr->expected_revision); |
| 1242 | 1371 | if (prog) |
| 1243 | 1372 | bpf_prog_put(prog); |
| 1244 | 1373 | |
| @@ -1267,7 +1396,7 @@ static void bpf_cgroup_link_release(struct bpf_link *link) · 22 unmodified lines |
| 1267 | 1396 | } |
| 1268 | 1397 | |
| 1269 | 1398 | WARN_ON(__cgroup_bpf_detach(cg_link->cgroup, NULL, cg_link, |
| 1270 | | − cg_link->type)); |
| 1399 | + cg_link->type, 0)); |
| 1271 | 1400 | if (cg_link->type == BPF_LSM_CGROUP) |
| 1272 | 1401 | bpf_trampoline_unlink_cgroup_shim(cg_link->link.prog); |
| 1273 | 1402 | |
| @@ -1339,6 +1468,13 @@ static const struct bpf_link_ops bpf_cgroup_link_lops = { · 65 unmodified lines |
| 1339 | 1468 | .fill_link_info = bpf_cgroup_link_fill_link_info, |
| 1340 | 1469 | }; |
| 1341 | 1470 | |
| 1471 | +#define BPF_F_LINK_ATTACH_MASK \ |
| 1472 | + (BPF_F_ID | \ |
| 1473 | + BPF_F_BEFORE | \ |
| 1474 | + BPF_F_AFTER | \ |
| 1475 | + BPF_F_PREORDER | \ |
| 1476 | + BPF_F_LINK) |
| 1477 | + |
| 1342 | 1478 | int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) |
| 1343 | 1479 | { |
| 1344 | 1480 | struct bpf_link_primer link_primer; |
| @@ -1346,7 +1482,7 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) · 1 unmodified line |
| 1346 | 1482 | struct cgroup *cgrp; |
| 1347 | 1483 | int err; |
| 1348 | 1484 | |
| 1349 | | − if (attr->link_create.flags) |
| 1485 | + if (attr->link_create.flags & (~BPF_F_LINK_ATTACH_MASK)) |
| 1350 | 1486 | return -EINVAL; |
| 1351 | 1487 | |
| 1352 | 1488 | cgrp = cgroup_get_from_fd(attr->link_create.target_fd); |
| @@ -1370,7 +1506,9 @@ int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) · 17 unmodified lines |
| 1370 | 1506 | } |
| 1371 | 1507 | |
| 1372 | 1508 | err = cgroup_bpf_attach(cgrp, NULL, NULL, link, |
| 1373 | | − link->type, BPF_F_ALLOW_MULTI); |
| 1509 | + link->type, BPF_F_ALLOW_MULTI | attr->link_create.flags, |
| 1510 | + attr->link_create.cgroup.relative_fd, |
| 1511 | + attr->link_create.cgroup.expected_revision); |
| 1374 | 1512 | if (err) { |
| 1375 | 1513 | bpf_link_cleanup(&link_primer); |
| 1376 | 1514 | goto out_put_cgroup; |