| @@ -0,0 +1,2016 @@ |
| | 1 | #include <linux/bpf.h> |
| | 2 | #include <linux/errno.h> |
| | 3 | #include <linux/errqueue.h> |
| | 4 | #include <linux/file.h> |
| | 5 | #include <linux/in.h> |
| | 6 | #include <linux/kernel.h> |
| | 7 | #include <linux/module.h> |
| | 8 | #include <linux/net.h> |
| | 9 | #include <linux/netdevice.h> |
| | 10 | #include <linux/poll.h> |
| | 11 | #include <linux/rculist.h> |
| | 12 | #include <linux/skbuff.h> |
| | 13 | #include <linux/socket.h> |
| | 14 | #include <linux/uaccess.h> |
| | 15 | #include <linux/workqueue.h> |
| | 16 | #include <net/kcm.h> |
| | 17 | #include <net/netns/generic.h> |
| | 18 | #include <net/sock.h> |
| | 19 | #include <net/tcp.h> |
| | 20 | #include <uapi/linux/kcm.h> |
| | 21 | |
| | 22 | unsigned int kcm_net_id; |
| | 23 | |
| | 24 | static struct kmem_cache *kcm_psockp __read_mostly; |
| | 25 | static struct kmem_cache *kcm_muxp __read_mostly; |
| | 26 | static struct workqueue_struct *kcm_wq; |
| | 27 | |
| | 28 | static inline struct kcm_sock *kcm_sk(const struct sock *sk) |
| | 29 | { |
| | 30 | return (struct kcm_sock *)sk; |
| | 31 | } |
| | 32 | |
| | 33 | static inline struct kcm_tx_msg *kcm_tx_msg(struct sk_buff *skb) |
| | 34 | { |
| | 35 | return (struct kcm_tx_msg *)skb->cb; |
| | 36 | } |
| | 37 | |
| | 38 | static inline struct kcm_rx_msg *kcm_rx_msg(struct sk_buff *skb) |
| | 39 | { |
| | 40 | return (struct kcm_rx_msg *)((void *)skb->cb + |
| | 41 | offsetof(struct qdisc_skb_cb, data)); |
| | 42 | } |
| | 43 | |
| | 44 | static void report_csk_error(struct sock *csk, int err) |
| | 45 | { |
| | 46 | csk->sk_err = EPIPE; |
| | 47 | csk->sk_error_report(csk); |
| | 48 | } |
| | 49 | |
| | 50 | /* Callback lock held */ |
| | 51 | static void kcm_abort_rx_psock(struct kcm_psock *psock, int err, |
| | 52 | struct sk_buff *skb) |
| | 53 | { |
| | 54 | struct sock *csk = psock->sk; |
| | 55 | |
| | 56 | /* Unrecoverable error in receive */ |
| | 57 | |
| | 58 | if (psock->rx_stopped) |
| | 59 | return; |
| | 60 | |
| | 61 | psock->rx_stopped = 1; |
| | 62 | |
| | 63 | /* Report an error on the lower socket */ |
| | 64 | report_csk_error(csk, err); |
| | 65 | } |
| | 66 | |
| | 67 | static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, |
| | 68 | bool wakeup_kcm) |
| | 69 | { |
| | 70 | struct sock *csk = psock->sk; |
| | 71 | struct kcm_mux *mux = psock->mux; |
| | 72 | |
| | 73 | /* Unrecoverable error in transmit */ |
| | 74 | |
| | 75 | spin_lock_bh(&mux->lock); |
| | 76 | |
| | 77 | if (psock->tx_stopped) { |
| | 78 | spin_unlock_bh(&mux->lock); |
| | 79 | return; |
| | 80 | } |
| | 81 | |
| | 82 | psock->tx_stopped = 1; |
| | 83 | |
| | 84 | if (!psock->tx_kcm) { |
| | 85 | /* Take off psocks_avail list */ |
| | 86 | list_del(&psock->psock_avail_list); |
| | 87 | } else if (wakeup_kcm) { |
| | 88 | /* In this case psock is being aborted while outside of |
| | 89 | * write_msgs and psock is reserved. Schedule tx_work |
| | 90 | * to handle the failure there. Need to commit tx_stopped |
| | 91 | * before queuing work. |
| | 92 | */ |
| | 93 | smp_mb(); |
| | 94 | |
| | 95 | queue_work(kcm_wq, &psock->tx_kcm->tx_work); |
| | 96 | } |
| | 97 | |
| | 98 | spin_unlock_bh(&mux->lock); |
| | 99 | |
| | 100 | /* Report error on lower socket */ |
| | 101 | report_csk_error(csk, err); |
| | 102 | } |
| | 103 | |
| | 104 | static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb); |
| | 105 | |
| | 106 | /* KCM is ready to receive messages on its queue-- either the KCM is new or |
| | 107 | * has become unblocked after being blocked on full socket buffer. Queue any |
| | 108 | * pending ready messages on a psock. RX mux lock held. |
| | 109 | */ |
| | 110 | static void kcm_rcv_ready(struct kcm_sock *kcm) |
| | 111 | { |
| | 112 | struct kcm_mux *mux = kcm->mux; |
| | 113 | struct kcm_psock *psock; |
| | 114 | struct sk_buff *skb; |
| | 115 | |
| | 116 | if (unlikely(kcm->rx_wait || kcm->rx_psock || kcm->rx_disabled)) |
| | 117 | return; |
| | 118 | |
| | 119 | while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) { |
| | 120 | if (kcm_queue_rcv_skb(&kcm->sk, skb)) { |
| | 121 | /* Assuming buffer limit has been reached */ |
| | 122 | skb_queue_head(&mux->rx_hold_queue, skb); |
| | 123 | WARN_ON(!sk_rmem_alloc_get(&kcm->sk)); |
| | 124 | return; |
| | 125 | } |
| | 126 | } |
| | 127 | |
| | 128 | while (!list_empty(&mux->psocks_ready)) { |
| | 129 | psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, |
| | 130 | psock_ready_list); |
| | 131 | |
| | 132 | if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { |
| | 133 | /* Assuming buffer limit has been reached */ |
| | 134 | WARN_ON(!sk_rmem_alloc_get(&kcm->sk)); |
| | 135 | return; |
| | 136 | } |
| | 137 | |
| | 138 | /* Consumed the ready message on the psock. Schedule rx_work to |
| | 139 | * get more messages. |
| | 140 | */ |
| | 141 | list_del(&psock->psock_ready_list); |
| | 142 | psock->ready_rx_msg = NULL; |
| | 143 | |
| | 144 | /* Commit clearing of ready_rx_msg for queuing work */ |
| | 145 | smp_mb(); |
| | 146 | |
| | 147 | queue_work(kcm_wq, &psock->rx_work); |
| | 148 | } |
| | 149 | |
| | 150 | /* Buffer limit is okay now, add to ready list */ |
| | 151 | list_add_tail(&kcm->wait_rx_list, |
| | 152 | &kcm->mux->kcm_rx_waiters); |
| | 153 | kcm->rx_wait = true; |
| | 154 | } |
| | 155 | |
| | 156 | static void kcm_rfree(struct sk_buff *skb) |
| | 157 | { |
| | 158 | struct sock *sk = skb->sk; |
| | 159 | struct kcm_sock *kcm = kcm_sk(sk); |
| | 160 | struct kcm_mux *mux = kcm->mux; |
| | 161 | unsigned int len = skb->truesize; |
| | 162 | |
| | 163 | sk_mem_uncharge(sk, len); |
| | 164 | atomic_sub(len, &sk->sk_rmem_alloc); |
| | 165 | |
| | 166 | /* For reading rx_wait and rx_psock without holding lock */ |
| | 167 | smp_mb__after_atomic(); |
| | 168 | |
| | 169 | if (!kcm->rx_wait && !kcm->rx_psock && |
| | 170 | sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) { |
| | 171 | spin_lock_bh(&mux->rx_lock); |
| | 172 | kcm_rcv_ready(kcm); |
| | 173 | spin_unlock_bh(&mux->rx_lock); |
| | 174 | } |
| | 175 | } |
| | 176 | |
| | 177 | static int kcm_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) |
| | 178 | { |
| | 179 | struct sk_buff_head *list = &sk->sk_receive_queue; |
| | 180 | |
| | 181 | if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) |
| | 182 | return -ENOMEM; |
| | 183 | |
| | 184 | if (!sk_rmem_schedule(sk, skb, skb->truesize)) |
| | 185 | return -ENOBUFS; |
| | 186 | |
| | 187 | skb->dev = NULL; |
| | 188 | |
| | 189 | skb_orphan(skb); |
| | 190 | skb->sk = sk; |
| | 191 | skb->destructor = kcm_rfree; |
| | 192 | atomic_add(skb->truesize, &sk->sk_rmem_alloc); |
| | 193 | sk_mem_charge(sk, skb->truesize); |
| | 194 | |
| | 195 | skb_queue_tail(list, skb); |
| | 196 | |
| | 197 | if (!sock_flag(sk, SOCK_DEAD)) |
| | 198 | sk->sk_data_ready(sk); |
| | 199 | |
| | 200 | return 0; |
| | 201 | } |
| | 202 | |
| | 203 | /* Requeue received messages for a kcm socket to other kcm sockets. This is |
| | 204 | * called with a kcm socket is receive disabled. |
| | 205 | * RX mux lock held. |
| | 206 | */ |
| | 207 | static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head) |
| | 208 | { |
| | 209 | struct sk_buff *skb; |
| | 210 | struct kcm_sock *kcm; |
| | 211 | |
| | 212 | while ((skb = __skb_dequeue(head))) { |
| | 213 | /* Reset destructor to avoid calling kcm_rcv_ready */ |
| | 214 | skb->destructor = sock_rfree; |
| | 215 | skb_orphan(skb); |
| | 216 | try_again: |
| | 217 | if (list_empty(&mux->kcm_rx_waiters)) { |
| | 218 | skb_queue_tail(&mux->rx_hold_queue, skb); |
| | 219 | continue; |
| | 220 | } |
| | 221 | |
| | 222 | kcm = list_first_entry(&mux->kcm_rx_waiters, |
| | 223 | struct kcm_sock, wait_rx_list); |
| | 224 | |
| | 225 | if (kcm_queue_rcv_skb(&kcm->sk, skb)) { |
| | 226 | /* Should mean socket buffer full */ |
| | 227 | list_del(&kcm->wait_rx_list); |
| | 228 | kcm->rx_wait = false; |
| | 229 | |
| | 230 | /* Commit rx_wait to read in kcm_free */ |
| | 231 | smp_wmb(); |
| | 232 | |
| | 233 | goto try_again; |
| | 234 | } |
| | 235 | } |
| | 236 | } |
| | 237 | |
| | 238 | /* Lower sock lock held */ |
| | 239 | static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, |
| | 240 | struct sk_buff *head) |
| | 241 | { |
| | 242 | struct kcm_mux *mux = psock->mux; |
| | 243 | struct kcm_sock *kcm; |
| | 244 | |
| | 245 | WARN_ON(psock->ready_rx_msg); |
| | 246 | |
| | 247 | if (psock->rx_kcm) |
| | 248 | return psock->rx_kcm; |
| | 249 | |
| | 250 | spin_lock_bh(&mux->rx_lock); |
| | 251 | |
| | 252 | if (psock->rx_kcm) { |
| | 253 | spin_unlock_bh(&mux->rx_lock); |
| | 254 | return psock->rx_kcm; |
| | 255 | } |
| | 256 | |
| | 257 | if (list_empty(&mux->kcm_rx_waiters)) { |
| | 258 | psock->ready_rx_msg = head; |
| | 259 | list_add_tail(&psock->psock_ready_list, |
| | 260 | &mux->psocks_ready); |
| | 261 | spin_unlock_bh(&mux->rx_lock); |
| | 262 | return NULL; |
| | 263 | } |
| | 264 | |
| | 265 | kcm = list_first_entry(&mux->kcm_rx_waiters, |
| | 266 | struct kcm_sock, wait_rx_list); |
| | 267 | list_del(&kcm->wait_rx_list); |
| | 268 | kcm->rx_wait = false; |
| | 269 | |
| | 270 | psock->rx_kcm = kcm; |
| | 271 | kcm->rx_psock = psock; |
| | 272 | |
| | 273 | spin_unlock_bh(&mux->rx_lock); |
| | 274 | |
| | 275 | return kcm; |
| | 276 | } |
| | 277 | |
| | 278 | static void kcm_done(struct kcm_sock *kcm); |
| | 279 | |
| | 280 | static void kcm_done_work(struct work_struct *w) |
| | 281 | { |
| | 282 | kcm_done(container_of(w, struct kcm_sock, done_work)); |
| | 283 | } |
| | 284 | |
| | 285 | /* Lower sock held */ |
| | 286 | static void unreserve_rx_kcm(struct kcm_psock *psock, |
| | 287 | bool rcv_ready) |
| | 288 | { |
| | 289 | struct kcm_sock *kcm = psock->rx_kcm; |
| | 290 | struct kcm_mux *mux = psock->mux; |
| | 291 | |
| | 292 | if (!kcm) |
| | 293 | return; |
| | 294 | |
| | 295 | spin_lock_bh(&mux->rx_lock); |
| | 296 | |
| | 297 | psock->rx_kcm = NULL; |
| | 298 | kcm->rx_psock = NULL; |
| | 299 | |
| | 300 | /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with |
| | 301 | * kcm_rfree |
| | 302 | */ |
| | 303 | smp_mb(); |
| | 304 | |
| | 305 | if (unlikely(kcm->done)) { |
| | 306 | spin_unlock_bh(&mux->rx_lock); |
| | 307 | |
| | 308 | /* Need to run kcm_done in a task since we need to qcquire |
| | 309 | * callback locks which may already be held here. |
| | 310 | */ |
| | 311 | INIT_WORK(&kcm->done_work, kcm_done_work); |
| | 312 | schedule_work(&kcm->done_work); |
| | 313 | return; |
| | 314 | } |
| | 315 | |
| | 316 | if (unlikely(kcm->rx_disabled)) { |
| | 317 | requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue); |
| | 318 | } else if (rcv_ready || unlikely(!sk_rmem_alloc_get(&kcm->sk))) { |
| | 319 | /* Check for degenerative race with rx_wait that all |
| | 320 | * data was dequeued (accounted for in kcm_rfree). |
| | 321 | */ |
| | 322 | kcm_rcv_ready(kcm); |
| | 323 | } |
| | 324 | spin_unlock_bh(&mux->rx_lock); |
| | 325 | } |
| | 326 | |
| | 327 | /* Macro to invoke filter function. */ |
| | 328 | #define KCM_RUN_FILTER(prog, ctx) \ |
| | 329 | (*prog->bpf_func)(ctx, prog->insnsi) |
| | 330 | |
| | 331 | /* Lower socket lock held */ |
| | 332 | static int kcm_tcp_recv(read_descriptor_t *desc, struct sk_buff *orig_skb, |
| | 333 | unsigned int orig_offset, size_t orig_len) |
| | 334 | { |
| | 335 | struct kcm_psock *psock = (struct kcm_psock *)desc->arg.data; |
| | 336 | struct kcm_rx_msg *rxm; |
| | 337 | struct kcm_sock *kcm; |
| | 338 | struct sk_buff *head, *skb; |
| | 339 | size_t eaten = 0, cand_len; |
| | 340 | ssize_t extra; |
| | 341 | int err; |
| | 342 | bool cloned_orig = false; |
| | 343 | |
| | 344 | if (psock->ready_rx_msg) |
| | 345 | return 0; |
| | 346 | |
| | 347 | head = psock->rx_skb_head; |
| | 348 | if (head) { |
| | 349 | /* Message already in progress */ |
| | 350 | |
| | 351 | if (unlikely(orig_offset)) { |
| | 352 | /* Getting data with a non-zero offset when a message is |
| | 353 | * in progress is not expected. If it does happen, we |
| | 354 | * need to clone and pull since we can't deal with |
| | 355 | * offsets in the skbs for a message expect in the head. |
| | 356 | */ |
| | 357 | orig_skb = skb_clone(orig_skb, GFP_ATOMIC); |
| | 358 | if (!orig_skb) { |
| | 359 | desc->error = -ENOMEM; |
| | 360 | return 0; |
| | 361 | } |
| | 362 | if (!pskb_pull(orig_skb, orig_offset)) { |
| | 363 | kfree_skb(orig_skb); |
| | 364 | desc->error = -ENOMEM; |
| | 365 | return 0; |
| | 366 | } |
| | 367 | cloned_orig = true; |
| | 368 | orig_offset = 0; |
| | 369 | } |
| | 370 | |
| | 371 | if (!psock->rx_skb_nextp) { |
| | 372 | /* We are going to append to the frags_list of head. |
| | 373 | * Need to unshare the frag_list. |
| | 374 | */ |
| | 375 | err = skb_unclone(head, GFP_ATOMIC); |
| | 376 | if (err) { |
| | 377 | desc->error = err; |
| | 378 | return 0; |
| | 379 | } |
| | 380 | |
| | 381 | if (unlikely(skb_shinfo(head)->frag_list)) { |
| | 382 | /* We can't append to an sk_buff that already |
| | 383 | * has a frag_list. We create a new head, point |
| | 384 | * the frag_list of that to the old head, and |
| | 385 | * then are able to use the old head->next for |
| | 386 | * appending to the message. |
| | 387 | */ |
| | 388 | if (WARN_ON(head->next)) { |
| | 389 | desc->error = -EINVAL; |
| | 390 | return 0; |
| | 391 | } |
| | 392 | |
| | 393 | skb = alloc_skb(0, GFP_ATOMIC); |
| | 394 | if (!skb) { |
| | 395 | desc->error = -ENOMEM; |
| | 396 | return 0; |
| | 397 | } |
| | 398 | skb->len = head->len; |
| | 399 | skb->data_len = head->len; |
| | 400 | skb->truesize = head->truesize; |
| | 401 | *kcm_rx_msg(skb) = *kcm_rx_msg(head); |
| | 402 | psock->rx_skb_nextp = &head->next; |
| | 403 | skb_shinfo(skb)->frag_list = head; |
| | 404 | psock->rx_skb_head = skb; |
| | 405 | head = skb; |
| | 406 | } else { |
| | 407 | psock->rx_skb_nextp = |
| | 408 | &skb_shinfo(head)->frag_list; |
| | 409 | } |
| | 410 | } |
| | 411 | } |
| | 412 | |
| | 413 | while (eaten < orig_len) { |
| | 414 | /* Always clone since we will consume something */ |
| | 415 | skb = skb_clone(orig_skb, GFP_ATOMIC); |
| | 416 | if (!skb) { |
| | 417 | desc->error = -ENOMEM; |
| | 418 | break; |
| | 419 | } |
| | 420 | |
| | 421 | cand_len = orig_len - eaten; |
| | 422 | |
| | 423 | head = psock->rx_skb_head; |
| | 424 | if (!head) { |
| | 425 | head = skb; |
| | 426 | psock->rx_skb_head = head; |
| | 427 | /* Will set rx_skb_nextp on next packet if needed */ |
| | 428 | psock->rx_skb_nextp = NULL; |
| | 429 | rxm = kcm_rx_msg(head); |
| | 430 | memset(rxm, 0, sizeof(*rxm)); |
| | 431 | rxm->offset = orig_offset + eaten; |
| | 432 | } else { |
| | 433 | /* Unclone since we may be appending to an skb that we |
| | 434 | * already share a frag_list with. |
| | 435 | */ |
| | 436 | err = skb_unclone(skb, GFP_ATOMIC); |
| | 437 | if (err) { |
| | 438 | desc->error = err; |
| | 439 | break; |
| | 440 | } |
| | 441 | |
| | 442 | rxm = kcm_rx_msg(head); |
| | 443 | *psock->rx_skb_nextp = skb; |
| | 444 | psock->rx_skb_nextp = &skb->next; |
| | 445 | head->data_len += skb->len; |
| | 446 | head->len += skb->len; |
| | 447 | head->truesize += skb->truesize; |
| | 448 | } |
| | 449 | |
| | 450 | if (!rxm->full_len) { |
| | 451 | ssize_t len; |
| | 452 | |
| | 453 | len = KCM_RUN_FILTER(psock->bpf_prog, head); |
| | 454 | |
| | 455 | if (!len) { |
| | 456 | /* Need more header to determine length */ |
| | 457 | rxm->accum_len += cand_len; |
| | 458 | eaten += cand_len; |
| | 459 | WARN_ON(eaten != orig_len); |
| | 460 | break; |
| | 461 | } else if (len <= (ssize_t)head->len - |
| | 462 | skb->len - rxm->offset) { |
| | 463 | /* Length must be into new skb (and also |
| | 464 | * greater than zero) |
| | 465 | */ |
| | 466 | desc->error = -EPROTO; |
| | 467 | psock->rx_skb_head = NULL; |
| | 468 | kcm_abort_rx_psock(psock, EPROTO, head); |
| | 469 | break; |
| | 470 | } |
| | 471 | |
| | 472 | rxm->full_len = len; |
| | 473 | } |
| | 474 | |
| | 475 | extra = (ssize_t)(rxm->accum_len + cand_len) - rxm->full_len; |
| | 476 | |
| | 477 | if (extra < 0) { |
| | 478 | /* Message not complete yet. */ |
| | 479 | rxm->accum_len += cand_len; |
| | 480 | eaten += cand_len; |
| | 481 | WARN_ON(eaten != orig_len); |
| | 482 | break; |
| | 483 | } |
| | 484 | |
| | 485 | /* Positive extra indicates ore bytes than needed for the |
| | 486 | * message |
| | 487 | */ |
| | 488 | |
| | 489 | WARN_ON(extra > cand_len); |
| | 490 | |
| | 491 | eaten += (cand_len - extra); |
| | 492 | |
| | 493 | /* Hurray, we have a new message! */ |
| | 494 | psock->rx_skb_head = NULL; |
| | 495 | |
| | 496 | try_queue: |
| | 497 | kcm = reserve_rx_kcm(psock, head); |
| | 498 | if (!kcm) { |
| | 499 | /* Unable to reserve a KCM, message is held in psock. */ |
| | 500 | break; |
| | 501 | } |
| | 502 | |
| | 503 | if (kcm_queue_rcv_skb(&kcm->sk, head)) { |
| | 504 | /* Should mean socket buffer full */ |
| | 505 | unreserve_rx_kcm(psock, false); |
| | 506 | goto try_queue; |
| | 507 | } |
| | 508 | } |
| | 509 | |
| | 510 | if (cloned_orig) |
| | 511 | kfree_skb(orig_skb); |
| | 512 | |
| | 513 | return eaten; |
| | 514 | } |
| | 515 | |
| | 516 | /* Called with lock held on lower socket */ |
| | 517 | static int psock_tcp_read_sock(struct kcm_psock *psock) |
| | 518 | { |
| | 519 | read_descriptor_t desc; |
| | 520 | |
| | 521 | desc.arg.data = psock; |
| | 522 | desc.error = 0; |
| | 523 | desc.count = 1; /* give more than one skb per call */ |
| | 524 | |
| | 525 | /* sk should be locked here, so okay to do tcp_read_sock */ |
| | 526 | tcp_read_sock(psock->sk, &desc, kcm_tcp_recv); |
| | 527 | |
| | 528 | unreserve_rx_kcm(psock, true); |
| | 529 | |
| | 530 | return desc.error; |
| | 531 | } |
| | 532 | |
| | 533 | /* Lower sock lock held */ |
| | 534 | static void psock_tcp_data_ready(struct sock *sk) |
| | 535 | { |
| | 536 | struct kcm_psock *psock; |
| | 537 | |
| | 538 | read_lock_bh(&sk->sk_callback_lock); |
| | 539 | |
| | 540 | psock = (struct kcm_psock *)sk->sk_user_data; |
| | 541 | if (unlikely(!psock || psock->rx_stopped)) |
| | 542 | goto out; |
| | 543 | |
| | 544 | if (psock->ready_rx_msg) |
| | 545 | goto out; |
| | 546 | |
| | 547 | if (psock_tcp_read_sock(psock) == -ENOMEM) |
| | 548 | queue_delayed_work(kcm_wq, &psock->rx_delayed_work, 0); |
| | 549 | |
| | 550 | out: |
| | 551 | read_unlock_bh(&sk->sk_callback_lock); |
| | 552 | } |
| | 553 | |
| | 554 | static void do_psock_rx_work(struct kcm_psock *psock) |
| | 555 | { |
| | 556 | read_descriptor_t rd_desc; |
| | 557 | struct sock *csk = psock->sk; |
| | 558 | |
| | 559 | /* We need the read lock to synchronize with psock_tcp_data_ready. We |
| | 560 | * need the socket lock for calling tcp_read_sock. |
| | 561 | */ |
| | 562 | lock_sock(csk); |
| | 563 | read_lock_bh(&csk->sk_callback_lock); |
| | 564 | |
| | 565 | if (unlikely(csk->sk_user_data != psock)) |
| | 566 | goto out; |
| | 567 | |
| | 568 | if (unlikely(psock->rx_stopped)) |
| | 569 | goto out; |
| | 570 | |
| | 571 | if (psock->ready_rx_msg) |
| | 572 | goto out; |
| | 573 | |
| | 574 | rd_desc.arg.data = psock; |
| | 575 | |
| | 576 | if (psock_tcp_read_sock(psock) == -ENOMEM) |
| | 577 | queue_delayed_work(kcm_wq, &psock->rx_delayed_work, 0); |
| | 578 | |
| | 579 | out: |
| | 580 | read_unlock_bh(&csk->sk_callback_lock); |
| | 581 | release_sock(csk); |
| | 582 | } |
| | 583 | |
| | 584 | static void psock_rx_work(struct work_struct *w) |
| | 585 | { |
| | 586 | do_psock_rx_work(container_of(w, struct kcm_psock, rx_work)); |
| | 587 | } |
| | 588 | |
| | 589 | static void psock_rx_delayed_work(struct work_struct *w) |
| | 590 | { |
| | 591 | do_psock_rx_work(container_of(w, struct kcm_psock, |
| | 592 | rx_delayed_work.work)); |
| | 593 | } |
| | 594 | |
| | 595 | static void psock_tcp_state_change(struct sock *sk) |
| | 596 | { |
| | 597 | /* TCP only does a POLLIN for a half close. Do a POLLHUP here |
| | 598 | * since application will normally not poll with POLLIN |
| | 599 | * on the TCP sockets. |
| | 600 | */ |
| | 601 | |
| | 602 | report_csk_error(sk, EPIPE); |
| | 603 | } |
| | 604 | |
| | 605 | static void psock_tcp_write_space(struct sock *sk) |
| | 606 | { |
| | 607 | struct kcm_psock *psock; |
| | 608 | struct kcm_mux *mux; |
| | 609 | struct kcm_sock *kcm; |
| | 610 | |
| | 611 | read_lock_bh(&sk->sk_callback_lock); |
| | 612 | |
| | 613 | psock = (struct kcm_psock *)sk->sk_user_data; |
| | 614 | if (unlikely(!psock)) |
| | 615 | goto out; |
| | 616 | |
| | 617 | mux = psock->mux; |
| | 618 | |
| | 619 | spin_lock_bh(&mux->lock); |
| | 620 | |
| | 621 | /* Check if the socket is reserved so someone is waiting for sending. */ |
| | 622 | kcm = psock->tx_kcm; |
| | 623 | if (kcm) |
| | 624 | queue_work(kcm_wq, &kcm->tx_work); |
| | 625 | |
| | 626 | spin_unlock_bh(&mux->lock); |
| | 627 | out: |
| | 628 | read_unlock_bh(&sk->sk_callback_lock); |
| | 629 | } |
| | 630 | |
| | 631 | static void unreserve_psock(struct kcm_sock *kcm); |
| | 632 | |
| | 633 | /* kcm sock is locked. */ |
| | 634 | static struct kcm_psock *reserve_psock(struct kcm_sock *kcm) |
| | 635 | { |
| | 636 | struct kcm_mux *mux = kcm->mux; |
| | 637 | struct kcm_psock *psock; |
| | 638 | |
| | 639 | psock = kcm->tx_psock; |
| | 640 | |
| | 641 | smp_rmb(); /* Must read tx_psock before tx_wait */ |
| | 642 | |
| | 643 | if (psock) { |
| | 644 | WARN_ON(kcm->tx_wait); |
| | 645 | if (unlikely(psock->tx_stopped)) |
| | 646 | unreserve_psock(kcm); |
| | 647 | else |
| | 648 | return kcm->tx_psock; |
| | 649 | } |
| | 650 | |
| | 651 | spin_lock_bh(&mux->lock); |
| | 652 | |
| | 653 | /* Check again under lock to see if psock was reserved for this |
| | 654 | * psock via psock_unreserve. |
| | 655 | */ |
| | 656 | psock = kcm->tx_psock; |
| | 657 | if (unlikely(psock)) { |
| | 658 | WARN_ON(kcm->tx_wait); |
| | 659 | spin_unlock_bh(&mux->lock); |
| | 660 | return kcm->tx_psock; |
| | 661 | } |
| | 662 | |
| | 663 | if (!list_empty(&mux->psocks_avail)) { |
| | 664 | psock = list_first_entry(&mux->psocks_avail, |
| | 665 | struct kcm_psock, |
| | 666 | psock_avail_list); |
| | 667 | list_del(&psock->psock_avail_list); |
| | 668 | if (kcm->tx_wait) { |
| | 669 | list_del(&kcm->wait_psock_list); |
| | 670 | kcm->tx_wait = false; |
| | 671 | } |
| | 672 | kcm->tx_psock = psock; |
| | 673 | psock->tx_kcm = kcm; |
| | 674 | } else if (!kcm->tx_wait) { |
| | 675 | list_add_tail(&kcm->wait_psock_list, |
| | 676 | &mux->kcm_tx_waiters); |
| | 677 | kcm->tx_wait = true; |
| | 678 | } |
| | 679 | |
| | 680 | spin_unlock_bh(&mux->lock); |
| | 681 | |
| | 682 | return psock; |
| | 683 | } |
| | 684 | |
| | 685 | /* mux lock held */ |
| | 686 | static void psock_now_avail(struct kcm_psock *psock) |
| | 687 | { |
| | 688 | struct kcm_mux *mux = psock->mux; |
| | 689 | struct kcm_sock *kcm; |
| | 690 | |
| | 691 | if (list_empty(&mux->kcm_tx_waiters)) { |
| | 692 | list_add_tail(&psock->psock_avail_list, |
| | 693 | &mux->psocks_avail); |
| | 694 | } else { |
| | 695 | kcm = list_first_entry(&mux->kcm_tx_waiters, |
| | 696 | struct kcm_sock, |
| | 697 | wait_psock_list); |
| | 698 | list_del(&kcm->wait_psock_list); |
| | 699 | kcm->tx_wait = false; |
| | 700 | psock->tx_kcm = kcm; |
| | 701 | |
| | 702 | /* Commit before changing tx_psock since that is read in |
| | 703 | * reserve_psock before queuing work. |
| | 704 | */ |
| | 705 | smp_mb(); |
| | 706 | |
| | 707 | kcm->tx_psock = psock; |
| | 708 | queue_work(kcm_wq, &kcm->tx_work); |
| | 709 | } |
| | 710 | } |
| | 711 | |
| | 712 | /* kcm sock is locked. */ |
| | 713 | static void unreserve_psock(struct kcm_sock *kcm) |
| | 714 | { |
| | 715 | struct kcm_psock *psock; |
| | 716 | struct kcm_mux *mux = kcm->mux; |
| | 717 | |
| | 718 | spin_lock_bh(&mux->lock); |
| | 719 | |
| | 720 | psock = kcm->tx_psock; |
| | 721 | |
| | 722 | if (WARN_ON(!psock)) { |
| | 723 | spin_unlock_bh(&mux->lock); |
| | 724 | return; |
| | 725 | } |
| | 726 | |
| | 727 | smp_rmb(); /* Read tx_psock before tx_wait */ |
| | 728 | |
| | 729 | WARN_ON(kcm->tx_wait); |
| | 730 | |
| | 731 | kcm->tx_psock = NULL; |
| | 732 | psock->tx_kcm = NULL; |
| | 733 | |
| | 734 | if (unlikely(psock->tx_stopped)) { |
| | 735 | if (psock->done) { |
| | 736 | /* Deferred free */ |
| | 737 | list_del(&psock->psock_list); |
| | 738 | mux->psocks_cnt--; |
| | 739 | sock_put(psock->sk); |
| | 740 | fput(psock->sk->sk_socket->file); |
| | 741 | kmem_cache_free(kcm_psockp, psock); |
| | 742 | } |
| | 743 | |
| | 744 | /* Don't put back on available list */ |
| | 745 | |
| | 746 | spin_unlock_bh(&mux->lock); |
| | 747 | |
| | 748 | return; |
| | 749 | } |
| | 750 | |
| | 751 | psock_now_avail(psock); |
| | 752 | |
| | 753 | spin_unlock_bh(&mux->lock); |
| | 754 | } |
| | 755 | |
| | 756 | /* Write any messages ready on the kcm socket. Called with kcm sock lock |
| | 757 | * held. Return bytes actually sent or error. |
| | 758 | */ |
| | 759 | static int kcm_write_msgs(struct kcm_sock *kcm) |
| | 760 | { |
| | 761 | struct sock *sk = &kcm->sk; |
| | 762 | struct kcm_psock *psock; |
| | 763 | struct sk_buff *skb, *head; |
| | 764 | struct kcm_tx_msg *txm; |
| | 765 | unsigned short fragidx, frag_offset; |
| | 766 | unsigned int sent, total_sent = 0; |
| | 767 | int ret = 0; |
| | 768 | |
| | 769 | kcm->tx_wait_more = false; |
| | 770 | psock = kcm->tx_psock; |
| | 771 | if (unlikely(psock && psock->tx_stopped)) { |
| | 772 | /* A reserved psock was aborted asynchronously. Unreserve |
| | 773 | * it and we'll retry the message. |
| | 774 | */ |
| | 775 | unreserve_psock(kcm); |
| | 776 | if (skb_queue_empty(&sk->sk_write_queue)) |
| | 777 | return 0; |
| | 778 | |
| | 779 | kcm_tx_msg(skb_peek(&sk->sk_write_queue))->sent = 0; |
| | 780 | |
| | 781 | } else if (skb_queue_empty(&sk->sk_write_queue)) { |
| | 782 | return 0; |
| | 783 | } |
| | 784 | |
| | 785 | head = skb_peek(&sk->sk_write_queue); |
| | 786 | txm = kcm_tx_msg(head); |
| | 787 | |
| | 788 | if (txm->sent) { |
| | 789 | /* Send of first skbuff in queue already in progress */ |
| | 790 | if (WARN_ON(!psock)) { |
| | 791 | ret = -EINVAL; |
| | 792 | goto out; |
| | 793 | } |
| | 794 | sent = txm->sent; |
| | 795 | frag_offset = txm->frag_offset; |
| | 796 | fragidx = txm->fragidx; |
| | 797 | skb = txm->frag_skb; |
| | 798 | |
| | 799 | goto do_frag; |
| | 800 | } |
| | 801 | |
| | 802 | try_again: |
| | 803 | psock = reserve_psock(kcm); |
| | 804 | if (!psock) |
| | 805 | goto out; |
| | 806 | |
| | 807 | do { |
| | 808 | skb = head; |
| | 809 | txm = kcm_tx_msg(head); |
| | 810 | sent = 0; |
| | 811 | |
| | 812 | do_frag_list: |
| | 813 | if (WARN_ON(!skb_shinfo(skb)->nr_frags)) { |
| | 814 | ret = -EINVAL; |
| | 815 | goto out; |
| | 816 | } |
| | 817 | |
| | 818 | for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; |
| | 819 | fragidx++) { |
| | 820 | skb_frag_t *frag; |
| | 821 | |
| | 822 | frag_offset = 0; |
| | 823 | do_frag: |
| | 824 | frag = &skb_shinfo(skb)->frags[fragidx]; |
| | 825 | if (WARN_ON(!frag->size)) { |
| | 826 | ret = -EINVAL; |
| | 827 | goto out; |
| | 828 | } |
| | 829 | |
| | 830 | ret = kernel_sendpage(psock->sk->sk_socket, |
| | 831 | frag->page.p, |
| | 832 | frag->page_offset + frag_offset, |
| | 833 | frag->size - frag_offset, |
| | 834 | MSG_DONTWAIT); |
| | 835 | if (ret <= 0) { |
| | 836 | if (ret == -EAGAIN) { |
| | 837 | /* Save state to try again when there's |
| | 838 | * write space on the socket |
| | 839 | */ |
| | 840 | txm->sent = sent; |
| | 841 | txm->frag_offset = frag_offset; |
| | 842 | txm->fragidx = fragidx; |
| | 843 | txm->frag_skb = skb; |
| | 844 | |
| | 845 | ret = 0; |
| | 846 | goto out; |
| | 847 | } |
| | 848 | |
| | 849 | /* Hard failure in sending message, abort this |
| | 850 | * psock since it has lost framing |
| | 851 | * synchonization and retry sending the |
| | 852 | * message from the beginning. |
| | 853 | */ |
| | 854 | kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, |
| | 855 | true); |
| | 856 | unreserve_psock(kcm); |
| | 857 | |
| | 858 | txm->sent = 0; |
| | 859 | ret = 0; |
| | 860 | |
| | 861 | goto try_again; |
| | 862 | } |
| | 863 | |
| | 864 | sent += ret; |
| | 865 | frag_offset += ret; |
| | 866 | if (frag_offset < frag->size) { |
| | 867 | /* Not finished with this frag */ |
| | 868 | goto do_frag; |
| | 869 | } |
| | 870 | } |
| | 871 | |
| | 872 | if (skb == head) { |
| | 873 | if (skb_has_frag_list(skb)) { |
| | 874 | skb = skb_shinfo(skb)->frag_list; |
| | 875 | goto do_frag_list; |
| | 876 | } |
| | 877 | } else if (skb->next) { |
| | 878 | skb = skb->next; |
| | 879 | goto do_frag_list; |
| | 880 | } |
| | 881 | |
| | 882 | /* Successfully sent the whole packet, account for it. */ |
| | 883 | skb_dequeue(&sk->sk_write_queue); |
| | 884 | kfree_skb(head); |
| | 885 | sk->sk_wmem_queued -= sent; |
| | 886 | total_sent += sent; |
| | 887 | } while ((head = skb_peek(&sk->sk_write_queue))); |
| | 888 | out: |
| | 889 | if (!head) { |
| | 890 | /* Done with all queued messages. */ |
| | 891 | WARN_ON(!skb_queue_empty(&sk->sk_write_queue)); |
| | 892 | unreserve_psock(kcm); |
| | 893 | } |
| | 894 | |
| | 895 | /* Check if write space is available */ |
| | 896 | sk->sk_write_space(sk); |
| | 897 | |
| | 898 | return total_sent ? : ret; |
| | 899 | } |
| | 900 | |
| | 901 | static void kcm_tx_work(struct work_struct *w) |
| | 902 | { |
| | 903 | struct kcm_sock *kcm = container_of(w, struct kcm_sock, tx_work); |
| | 904 | struct sock *sk = &kcm->sk; |
| | 905 | int err; |
| | 906 | |
| | 907 | lock_sock(sk); |
| | 908 | |
| | 909 | /* Primarily for SOCK_DGRAM sockets, also handle asynchronous tx |
| | 910 | * aborts |
| | 911 | */ |
| | 912 | err = kcm_write_msgs(kcm); |
| | 913 | if (err < 0) { |
| | 914 | /* Hard failure in write, report error on KCM socket */ |
| | 915 | pr_warn("KCM: Hard failure on kcm_write_msgs %d\n", err); |
| | 916 | report_csk_error(&kcm->sk, -err); |
| | 917 | goto out; |
| | 918 | } |
| | 919 | |
| | 920 | /* Primarily for SOCK_SEQPACKET sockets */ |
| | 921 | if (likely(sk->sk_socket) && |
| | 922 | test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { |
| | 923 | clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); |
| | 924 | sk->sk_write_space(sk); |
| | 925 | } |
| | 926 | |
| | 927 | out: |
| | 928 | release_sock(sk); |
| | 929 | } |
| | 930 | |
| | 931 | static void kcm_push(struct kcm_sock *kcm) |
| | 932 | { |
| | 933 | if (kcm->tx_wait_more) |
| | 934 | kcm_write_msgs(kcm); |
| | 935 | } |
| | 936 | |
| | 937 | static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) |
| | 938 | { |
| | 939 | struct sock *sk = sock->sk; |
| | 940 | struct kcm_sock *kcm = kcm_sk(sk); |
| | 941 | struct sk_buff *skb = NULL, *head = NULL; |
| | 942 | size_t copy, copied = 0; |
| | 943 | long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); |
| | 944 | int eor = (sock->type == SOCK_DGRAM) ? |
| | 945 | !(msg->msg_flags & MSG_MORE) : !!(msg->msg_flags & MSG_EOR); |
| | 946 | int err = -EPIPE; |
| | 947 | |
| | 948 | lock_sock(sk); |
| | 949 | |
| | 950 | /* Per tcp_sendmsg this should be in poll */ |
| | 951 | sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk); |
| | 952 | |
| | 953 | if (sk->sk_err) |
| | 954 | goto out_error; |
| | 955 | |
| | 956 | if (kcm->seq_skb) { |
| | 957 | /* Previously opened message */ |
| | 958 | head = kcm->seq_skb; |
| | 959 | skb = kcm_tx_msg(head)->last_skb; |
| | 960 | goto start; |
| | 961 | } |
| | 962 | |
| | 963 | /* Call the sk_stream functions to manage the sndbuf mem. */ |
| | 964 | if (!sk_stream_memory_free(sk)) { |
| | 965 | kcm_push(kcm); |
| | 966 | set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); |
| | 967 | err = sk_stream_wait_memory(sk, &timeo); |
| | 968 | if (err) |
| | 969 | goto out_error; |
| | 970 | } |
| | 971 | |
| | 972 | /* New message, alloc head skb */ |
| | 973 | head = alloc_skb(0, sk->sk_allocation); |
| | 974 | while (!head) { |
| | 975 | kcm_push(kcm); |
| | 976 | err = sk_stream_wait_memory(sk, &timeo); |
| | 977 | if (err) |
| | 978 | goto out_error; |
| | 979 | |
| | 980 | head = alloc_skb(0, sk->sk_allocation); |
| | 981 | } |
| | 982 | |
| | 983 | skb = head; |
| | 984 | |
| | 985 | /* Set ip_summed to CHECKSUM_UNNECESSARY to avoid calling |
| | 986 | * csum_and_copy_from_iter from skb_do_copy_data_nocache. |
| | 987 | */ |
| | 988 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
| | 989 | |
| | 990 | start: |
| | 991 | while (msg_data_left(msg)) { |
| | 992 | bool merge = true; |
| | 993 | int i = skb_shinfo(skb)->nr_frags; |
| | 994 | struct page_frag *pfrag = sk_page_frag(sk); |
| | 995 | |
| | 996 | if (!sk_page_frag_refill(sk, pfrag)) |
| | 997 | goto wait_for_memory; |
| | 998 | |
| | 999 | if (!skb_can_coalesce(skb, i, pfrag->page, |
| | 1000 | pfrag->offset)) { |
| | 1001 | if (i == MAX_SKB_FRAGS) { |
| | 1002 | struct sk_buff *tskb; |
| | 1003 | |
| | 1004 | tskb = alloc_skb(0, sk->sk_allocation); |
| | 1005 | if (!tskb) |
| | 1006 | goto wait_for_memory; |
| | 1007 | |
| | 1008 | if (head == skb) |
| | 1009 | skb_shinfo(head)->frag_list = tskb; |
| | 1010 | else |
| | 1011 | skb->next = tskb; |
| | 1012 | |
| | 1013 | skb = tskb; |
| | 1014 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
| | 1015 | continue; |
| | 1016 | } |
| | 1017 | merge = false; |
| | 1018 | } |
| | 1019 | |
| | 1020 | copy = min_t(int, msg_data_left(msg), |
| | 1021 | pfrag->size - pfrag->offset); |
| | 1022 | |
| | 1023 | if (!sk_wmem_schedule(sk, copy)) |
| | 1024 | goto wait_for_memory; |
| | 1025 | |
| | 1026 | err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, |
| | 1027 | pfrag->page, |
| | 1028 | pfrag->offset, |
| | 1029 | copy); |
| | 1030 | if (err) |
| | 1031 | goto out_error; |
| | 1032 | |
| | 1033 | /* Update the skb. */ |
| | 1034 | if (merge) { |
| | 1035 | skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); |
| | 1036 | } else { |
| | 1037 | skb_fill_page_desc(skb, i, pfrag->page, |
| | 1038 | pfrag->offset, copy); |
| | 1039 | get_page(pfrag->page); |
| | 1040 | } |
| | 1041 | |
| | 1042 | pfrag->offset += copy; |
| | 1043 | copied += copy; |
| | 1044 | if (head != skb) { |
| | 1045 | head->len += copy; |
| | 1046 | head->data_len += copy; |
| | 1047 | } |
| | 1048 | |
| | 1049 | continue; |
| | 1050 | |
| | 1051 | wait_for_memory: |
| | 1052 | kcm_push(kcm); |
| | 1053 | err = sk_stream_wait_memory(sk, &timeo); |
| | 1054 | if (err) |
| | 1055 | goto out_error; |
| | 1056 | } |
| | 1057 | |
| | 1058 | if (eor) { |
| | 1059 | bool not_busy = skb_queue_empty(&sk->sk_write_queue); |
| | 1060 | |
| | 1061 | /* Message complete, queue it on send buffer */ |
| | 1062 | __skb_queue_tail(&sk->sk_write_queue, head); |
| | 1063 | kcm->seq_skb = NULL; |
| | 1064 | |
| | 1065 | if (msg->msg_flags & MSG_BATCH) { |
| | 1066 | kcm->tx_wait_more = true; |
| | 1067 | } else if (kcm->tx_wait_more || not_busy) { |
| | 1068 | err = kcm_write_msgs(kcm); |
| | 1069 | if (err < 0) { |
| | 1070 | /* We got a hard error in write_msgs but have |
| | 1071 | * already queued this message. Report an error |
| | 1072 | * in the socket, but don't affect return value |
| | 1073 | * from sendmsg |
| | 1074 | */ |
| | 1075 | pr_warn("KCM: Hard failure on kcm_write_msgs\n"); |
| | 1076 | report_csk_error(&kcm->sk, -err); |
| | 1077 | } |
| | 1078 | } |
| | 1079 | } else { |
| | 1080 | /* Message not complete, save state */ |
| | 1081 | partial_message: |
| | 1082 | kcm->seq_skb = head; |
| | 1083 | kcm_tx_msg(head)->last_skb = skb; |
| | 1084 | } |
| | 1085 | |
| | 1086 | release_sock(sk); |
| | 1087 | return copied; |
| | 1088 | |
| | 1089 | out_error: |
| | 1090 | kcm_push(kcm); |
| | 1091 | |
| | 1092 | if (copied && sock->type == SOCK_SEQPACKET) { |
| | 1093 | /* Wrote some bytes before encountering an |
| | 1094 | * error, return partial success. |
| | 1095 | */ |
| | 1096 | goto partial_message; |
| | 1097 | } |
| | 1098 | |
| | 1099 | if (head != kcm->seq_skb) |
| | 1100 | kfree_skb(head); |
| | 1101 | |
| | 1102 | err = sk_stream_error(sk, msg->msg_flags, err); |
| | 1103 | |
| | 1104 | /* make sure we wake any epoll edge trigger waiter */ |
| | 1105 | if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 && err == -EAGAIN)) |
| | 1106 | sk->sk_write_space(sk); |
| | 1107 | |
| | 1108 | release_sock(sk); |
| | 1109 | return err; |
| | 1110 | } |
| | 1111 | |
| | 1112 | static struct sk_buff *kcm_wait_data(struct sock *sk, int flags, |
| | 1113 | long timeo, int *err) |
| | 1114 | { |
| | 1115 | struct sk_buff *skb; |
| | 1116 | |
| | 1117 | while (!(skb = skb_peek(&sk->sk_receive_queue))) { |
| | 1118 | if (sk->sk_err) { |
| | 1119 | *err = sock_error(sk); |
| | 1120 | return NULL; |
| | 1121 | } |
| | 1122 | |
| | 1123 | if (sock_flag(sk, SOCK_DONE)) |
| | 1124 | return NULL; |
| | 1125 | |
| | 1126 | if ((flags & MSG_DONTWAIT) || !timeo) { |
| | 1127 | *err = -EAGAIN; |
| | 1128 | return NULL; |
| | 1129 | } |
| | 1130 | |
| | 1131 | sk_wait_data(sk, &timeo, NULL); |
| | 1132 | |
| | 1133 | /* Handle signals */ |
| | 1134 | if (signal_pending(current)) { |
| | 1135 | *err = sock_intr_errno(timeo); |
| | 1136 | return NULL; |
| | 1137 | } |
| | 1138 | } |
| | 1139 | |
| | 1140 | return skb; |
| | 1141 | } |
| | 1142 | |
| | 1143 | static int kcm_recvmsg(struct socket *sock, struct msghdr *msg, |
| | 1144 | size_t len, int flags) |
| | 1145 | { |
| | 1146 | struct sock *sk = sock->sk; |
| | 1147 | int err = 0; |
| | 1148 | long timeo; |
| | 1149 | struct kcm_rx_msg *rxm; |
| | 1150 | int copied = 0; |
| | 1151 | struct sk_buff *skb; |
| | 1152 | |
| | 1153 | timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); |
| | 1154 | |
| | 1155 | lock_sock(sk); |
| | 1156 | |
| | 1157 | skb = kcm_wait_data(sk, flags, timeo, &err); |
| | 1158 | if (!skb) |
| | 1159 | goto out; |
| | 1160 | |
| | 1161 | /* Okay, have a message on the receive queue */ |
| | 1162 | |
| | 1163 | rxm = kcm_rx_msg(skb); |
| | 1164 | |
| | 1165 | if (len > rxm->full_len) |
| | 1166 | len = rxm->full_len; |
| | 1167 | |
| | 1168 | err = skb_copy_datagram_msg(skb, rxm->offset, msg, len); |
| | 1169 | if (err < 0) |
| | 1170 | goto out; |
| | 1171 | |
| | 1172 | copied = len; |
| | 1173 | if (likely(!(flags & MSG_PEEK))) { |
| | 1174 | if (copied < rxm->full_len) { |
| | 1175 | if (sock->type == SOCK_DGRAM) { |
| | 1176 | /* Truncated message */ |
| | 1177 | msg->msg_flags |= MSG_TRUNC; |
| | 1178 | goto msg_finished; |
| | 1179 | } |
| | 1180 | rxm->offset += copied; |
| | 1181 | rxm->full_len -= copied; |
| | 1182 | } else { |
| | 1183 | msg_finished: |
| | 1184 | /* Finished with message */ |
| | 1185 | msg->msg_flags |= MSG_EOR; |
| | 1186 | skb_unlink(skb, &sk->sk_receive_queue); |
| | 1187 | kfree_skb(skb); |
| | 1188 | } |
| | 1189 | } |
| | 1190 | |
| | 1191 | out: |
| | 1192 | release_sock(sk); |
| | 1193 | |
| | 1194 | return copied ? : err; |
| | 1195 | } |
| | 1196 | |
| | 1197 | /* kcm sock lock held */ |
| | 1198 | static void kcm_recv_disable(struct kcm_sock *kcm) |
| | 1199 | { |
| | 1200 | struct kcm_mux *mux = kcm->mux; |
| | 1201 | |
| | 1202 | if (kcm->rx_disabled) |
| | 1203 | return; |
| | 1204 | |
| | 1205 | spin_lock_bh(&mux->rx_lock); |
| | 1206 | |
| | 1207 | kcm->rx_disabled = 1; |
| | 1208 | |
| | 1209 | /* If a psock is reserved we'll do cleanup in unreserve */ |
| | 1210 | if (!kcm->rx_psock) { |
| | 1211 | if (kcm->rx_wait) { |
| | 1212 | list_del(&kcm->wait_rx_list); |
| | 1213 | kcm->rx_wait = false; |
| | 1214 | } |
| | 1215 | |
| | 1216 | requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue); |
| | 1217 | } |
| | 1218 | |
| | 1219 | spin_unlock_bh(&mux->rx_lock); |
| | 1220 | } |
| | 1221 | |
| | 1222 | /* kcm sock lock held */ |
| | 1223 | static void kcm_recv_enable(struct kcm_sock *kcm) |
| | 1224 | { |
| | 1225 | struct kcm_mux *mux = kcm->mux; |
| | 1226 | |
| | 1227 | if (!kcm->rx_disabled) |
| | 1228 | return; |
| | 1229 | |
| | 1230 | spin_lock_bh(&mux->rx_lock); |
| | 1231 | |
| | 1232 | kcm->rx_disabled = 0; |
| | 1233 | kcm_rcv_ready(kcm); |
| | 1234 | |
| | 1235 | spin_unlock_bh(&mux->rx_lock); |
| | 1236 | } |
| | 1237 | |
| | 1238 | static int kcm_setsockopt(struct socket *sock, int level, int optname, |
| | 1239 | char __user *optval, unsigned int optlen) |
| | 1240 | { |
| | 1241 | struct kcm_sock *kcm = kcm_sk(sock->sk); |
| | 1242 | int val, valbool; |
| | 1243 | int err = 0; |
| | 1244 | |
| | 1245 | if (level != SOL_KCM) |
| | 1246 | return -ENOPROTOOPT; |
| | 1247 | |
| | 1248 | if (optlen < sizeof(int)) |
| | 1249 | return -EINVAL; |
| | 1250 | |
| | 1251 | if (get_user(val, (int __user *)optval)) |
| | 1252 | return -EINVAL; |
| | 1253 | |
| | 1254 | valbool = val ? 1 : 0; |
| | 1255 | |
| | 1256 | switch (optname) { |
| | 1257 | case KCM_RECV_DISABLE: |
| | 1258 | lock_sock(&kcm->sk); |
| | 1259 | if (valbool) |
| | 1260 | kcm_recv_disable(kcm); |
| | 1261 | else |
| | 1262 | kcm_recv_enable(kcm); |
| | 1263 | release_sock(&kcm->sk); |
| | 1264 | break; |
| | 1265 | default: |
| | 1266 | err = -ENOPROTOOPT; |
| | 1267 | } |
| | 1268 | |
| | 1269 | return err; |
| | 1270 | } |
| | 1271 | |
| | 1272 | static int kcm_getsockopt(struct socket *sock, int level, int optname, |
| | 1273 | char __user *optval, int __user *optlen) |
| | 1274 | { |
| | 1275 | struct kcm_sock *kcm = kcm_sk(sock->sk); |
| | 1276 | int val, len; |
| | 1277 | |
| | 1278 | if (level != SOL_KCM) |
| | 1279 | return -ENOPROTOOPT; |
| | 1280 | |
| | 1281 | if (get_user(len, optlen)) |
| | 1282 | return -EFAULT; |
| | 1283 | |
| | 1284 | len = min_t(unsigned int, len, sizeof(int)); |
| | 1285 | if (len < 0) |
| | 1286 | return -EINVAL; |
| | 1287 | |
| | 1288 | switch (optname) { |
| | 1289 | case KCM_RECV_DISABLE: |
| | 1290 | val = kcm->rx_disabled; |
| | 1291 | break; |
| | 1292 | default: |
| | 1293 | return -ENOPROTOOPT; |
| | 1294 | } |
| | 1295 | |
| | 1296 | if (put_user(len, optlen)) |
| | 1297 | return -EFAULT; |
| | 1298 | if (copy_to_user(optval, &val, len)) |
| | 1299 | return -EFAULT; |
| | 1300 | return 0; |
| | 1301 | } |
| | 1302 | |
| | 1303 | static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux) |
| | 1304 | { |
| | 1305 | struct kcm_sock *tkcm; |
| | 1306 | struct list_head *head; |
| | 1307 | int index = 0; |
| | 1308 | |
| | 1309 | /* For SOCK_SEQPACKET sock type, datagram_poll checks the sk_state, so |
| | 1310 | * we set sk_state, otherwise epoll_wait always returns right away with |
| | 1311 | * POLLHUP |
| | 1312 | */ |
| | 1313 | kcm->sk.sk_state = TCP_ESTABLISHED; |
| | 1314 | |
| | 1315 | /* Add to mux's kcm sockets list */ |
| | 1316 | kcm->mux = mux; |
| | 1317 | spin_lock_bh(&mux->lock); |
| | 1318 | |
| | 1319 | head = &mux->kcm_socks; |
| | 1320 | list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) { |
| | 1321 | if (tkcm->index != index) |
| | 1322 | break; |
| | 1323 | head = &tkcm->kcm_sock_list; |
| | 1324 | index++; |
| | 1325 | } |
| | 1326 | |
| | 1327 | list_add(&kcm->kcm_sock_list, head); |
| | 1328 | kcm->index = index; |
| | 1329 | |
| | 1330 | mux->kcm_socks_cnt++; |
| | 1331 | spin_unlock_bh(&mux->lock); |
| | 1332 | |
| | 1333 | INIT_WORK(&kcm->tx_work, kcm_tx_work); |
| | 1334 | |
| | 1335 | spin_lock_bh(&mux->rx_lock); |
| | 1336 | kcm_rcv_ready(kcm); |
| | 1337 | spin_unlock_bh(&mux->rx_lock); |
| | 1338 | } |
| | 1339 | |
| | 1340 | static int kcm_attach(struct socket *sock, struct socket *csock, |
| | 1341 | struct bpf_prog *prog) |
| | 1342 | { |
| | 1343 | struct kcm_sock *kcm = kcm_sk(sock->sk); |
| | 1344 | struct kcm_mux *mux = kcm->mux; |
| | 1345 | struct sock *csk; |
| | 1346 | struct kcm_psock *psock = NULL, *tpsock; |
| | 1347 | struct list_head *head; |
| | 1348 | int index = 0; |
| | 1349 | |
| | 1350 | if (csock->ops->family != PF_INET && |
| | 1351 | csock->ops->family != PF_INET6) |
| | 1352 | return -EINVAL; |
| | 1353 | |
| | 1354 | csk = csock->sk; |
| | 1355 | if (!csk) |
| | 1356 | return -EINVAL; |
| | 1357 | |
| | 1358 | /* Only support TCP for now */ |
| | 1359 | if (csk->sk_protocol != IPPROTO_TCP) |
| | 1360 | return -EINVAL; |
| | 1361 | |
| | 1362 | psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); |
| | 1363 | if (!psock) |
| | 1364 | return -ENOMEM; |
| | 1365 | |
| | 1366 | psock->mux = mux; |
| | 1367 | psock->sk = csk; |
| | 1368 | psock->bpf_prog = prog; |
| | 1369 | INIT_WORK(&psock->rx_work, psock_rx_work); |
| | 1370 | INIT_DELAYED_WORK(&psock->rx_delayed_work, psock_rx_delayed_work); |
| | 1371 | |
| | 1372 | sock_hold(csk); |
| | 1373 | |
| | 1374 | write_lock_bh(&csk->sk_callback_lock); |
| | 1375 | psock->save_data_ready = csk->sk_data_ready; |
| | 1376 | psock->save_write_space = csk->sk_write_space; |
| | 1377 | psock->save_state_change = csk->sk_state_change; |
| | 1378 | csk->sk_user_data = psock; |
| | 1379 | csk->sk_data_ready = psock_tcp_data_ready; |
| | 1380 | csk->sk_write_space = psock_tcp_write_space; |
| | 1381 | csk->sk_state_change = psock_tcp_state_change; |
| | 1382 | write_unlock_bh(&csk->sk_callback_lock); |
| | 1383 | |
| | 1384 | /* Finished initialization, now add the psock to the MUX. */ |
| | 1385 | spin_lock_bh(&mux->lock); |
| | 1386 | head = &mux->psocks; |
| | 1387 | list_for_each_entry(tpsock, &mux->psocks, psock_list) { |
| | 1388 | if (tpsock->index != index) |
| | 1389 | break; |
| | 1390 | head = &tpsock->psock_list; |
| | 1391 | index++; |
| | 1392 | } |
| | 1393 | |
| | 1394 | list_add(&psock->psock_list, head); |
| | 1395 | psock->index = index; |
| | 1396 | |
| | 1397 | mux->psocks_cnt++; |
| | 1398 | psock_now_avail(psock); |
| | 1399 | spin_unlock_bh(&mux->lock); |
| | 1400 | |
| | 1401 | /* Schedule RX work in case there are already bytes queued */ |
| | 1402 | queue_work(kcm_wq, &psock->rx_work); |
| | 1403 | |
| | 1404 | return 0; |
| | 1405 | } |
| | 1406 | |
| | 1407 | static int kcm_attach_ioctl(struct socket *sock, struct kcm_attach *info) |
| | 1408 | { |
| | 1409 | struct socket *csock; |
| | 1410 | struct bpf_prog *prog; |
| | 1411 | int err; |
| | 1412 | |
| | 1413 | csock = sockfd_lookup(info->fd, &err); |
| | 1414 | if (!csock) |
| | 1415 | return -ENOENT; |
| | 1416 | |
| | 1417 | prog = bpf_prog_get(info->bpf_fd); |
| | 1418 | if (IS_ERR(prog)) { |
| | 1419 | err = PTR_ERR(prog); |
| | 1420 | goto out; |
| | 1421 | } |
| | 1422 | |
| | 1423 | if (prog->type != BPF_PROG_TYPE_SOCKET_FILTER) { |
| | 1424 | bpf_prog_put(prog); |
| | 1425 | err = -EINVAL; |
| | 1426 | goto out; |
| | 1427 | } |
| | 1428 | |
| | 1429 | err = kcm_attach(sock, csock, prog); |
| | 1430 | if (err) { |
| | 1431 | bpf_prog_put(prog); |
| | 1432 | goto out; |
| | 1433 | } |
| | 1434 | |
| | 1435 | /* Keep reference on file also */ |
| | 1436 | |
| | 1437 | return 0; |
| | 1438 | out: |
| | 1439 | fput(csock->file); |
| | 1440 | return err; |
| | 1441 | } |
| | 1442 | |
| | 1443 | static void kcm_unattach(struct kcm_psock *psock) |
| | 1444 | { |
| | 1445 | struct sock *csk = psock->sk; |
| | 1446 | struct kcm_mux *mux = psock->mux; |
| | 1447 | |
| | 1448 | /* Stop getting callbacks from TCP socket. After this there should |
| | 1449 | * be no way to reserve a kcm for this psock. |
| | 1450 | */ |
| | 1451 | write_lock_bh(&csk->sk_callback_lock); |
| | 1452 | csk->sk_user_data = NULL; |
| | 1453 | csk->sk_data_ready = psock->save_data_ready; |
| | 1454 | csk->sk_write_space = psock->save_write_space; |
| | 1455 | csk->sk_state_change = psock->save_state_change; |
| | 1456 | psock->rx_stopped = 1; |
| | 1457 | |
| | 1458 | if (WARN_ON(psock->rx_kcm)) { |
| | 1459 | write_unlock_bh(&csk->sk_callback_lock); |
| | 1460 | return; |
| | 1461 | } |
| | 1462 | |
| | 1463 | spin_lock_bh(&mux->rx_lock); |
| | 1464 | |
| | 1465 | /* Stop receiver activities. After this point psock should not be |
| | 1466 | * able to get onto ready list either through callbacks or work. |
| | 1467 | */ |
| | 1468 | if (psock->ready_rx_msg) { |
| | 1469 | list_del(&psock->psock_ready_list); |
| | 1470 | kfree_skb(psock->ready_rx_msg); |
| | 1471 | psock->ready_rx_msg = NULL; |
| | 1472 | } |
| | 1473 | |
| | 1474 | spin_unlock_bh(&mux->rx_lock); |
| | 1475 | |
| | 1476 | write_unlock_bh(&csk->sk_callback_lock); |
| | 1477 | |
| | 1478 | cancel_work_sync(&psock->rx_work); |
| | 1479 | cancel_delayed_work_sync(&psock->rx_delayed_work); |
| | 1480 | |
| | 1481 | bpf_prog_put(psock->bpf_prog); |
| | 1482 | |
| | 1483 | kfree_skb(psock->rx_skb_head); |
| | 1484 | psock->rx_skb_head = NULL; |
| | 1485 | |
| | 1486 | spin_lock_bh(&mux->lock); |
| | 1487 | |
| | 1488 | if (psock->tx_kcm) { |
| | 1489 | /* psock was reserved. Just mark it finished and we will clean |
| | 1490 | * up in the kcm paths, we need kcm lock which can not be |
| | 1491 | * acquired here. |
| | 1492 | */ |
| | 1493 | spin_unlock_bh(&mux->lock); |
| | 1494 | |
| | 1495 | /* We are unattaching a socket that is reserved. Abort the |
| | 1496 | * socket since we may be out of sync in sending on it. We need |
| | 1497 | * to do this without the mux lock. |
| | 1498 | */ |
| | 1499 | kcm_abort_tx_psock(psock, EPIPE, false); |
| | 1500 | |
| | 1501 | spin_lock_bh(&mux->lock); |
| | 1502 | if (!psock->tx_kcm) { |
| | 1503 | /* psock now unreserved in window mux was unlocked */ |
| | 1504 | goto no_reserved; |
| | 1505 | } |
| | 1506 | psock->done = 1; |
| | 1507 | |
| | 1508 | /* Commit done before queuing work to process it */ |
| | 1509 | smp_mb(); |
| | 1510 | |
| | 1511 | /* Queue tx work to make sure psock->done is handled */ |
| | 1512 | queue_work(kcm_wq, &psock->tx_kcm->tx_work); |
| | 1513 | spin_unlock_bh(&mux->lock); |
| | 1514 | } else { |
| | 1515 | no_reserved: |
| | 1516 | if (!psock->tx_stopped) |
| | 1517 | list_del(&psock->psock_avail_list); |
| | 1518 | list_del(&psock->psock_list); |
| | 1519 | mux->psocks_cnt--; |
| | 1520 | spin_unlock_bh(&mux->lock); |
| | 1521 | |
| | 1522 | sock_put(csk); |
| | 1523 | fput(csk->sk_socket->file); |
| | 1524 | kmem_cache_free(kcm_psockp, psock); |
| | 1525 | } |
| | 1526 | } |
| | 1527 | |
| | 1528 | static int kcm_unattach_ioctl(struct socket *sock, struct kcm_unattach *info) |
| | 1529 | { |
| | 1530 | struct kcm_sock *kcm = kcm_sk(sock->sk); |
| | 1531 | struct kcm_mux *mux = kcm->mux; |
| | 1532 | struct kcm_psock *psock; |
| | 1533 | struct socket *csock; |
| | 1534 | struct sock *csk; |
| | 1535 | int err; |
| | 1536 | |
| | 1537 | csock = sockfd_lookup(info->fd, &err); |
| | 1538 | if (!csock) |
| | 1539 | return -ENOENT; |
| | 1540 | |
| | 1541 | csk = csock->sk; |
| | 1542 | if (!csk) { |
| | 1543 | err = -EINVAL; |
| | 1544 | goto out; |
| | 1545 | } |
| | 1546 | |
| | 1547 | err = -ENOENT; |
| | 1548 | |
| | 1549 | spin_lock_bh(&mux->lock); |
| | 1550 | |
| | 1551 | list_for_each_entry(psock, &mux->psocks, psock_list) { |
| | 1552 | if (psock->sk != csk) |
| | 1553 | continue; |
| | 1554 | |
| | 1555 | /* Found the matching psock */ |
| | 1556 | |
| | 1557 | if (psock->unattaching || WARN_ON(psock->done)) { |
| | 1558 | err = -EALREADY; |
| | 1559 | break; |
| | 1560 | } |
| | 1561 | |
| | 1562 | psock->unattaching = 1; |
| | 1563 | |
| | 1564 | spin_unlock_bh(&mux->lock); |
| | 1565 | |
| | 1566 | kcm_unattach(psock); |
| | 1567 | |
| | 1568 | err = 0; |
| | 1569 | goto out; |
| | 1570 | } |
| | 1571 | |
| | 1572 | spin_unlock_bh(&mux->lock); |
| | 1573 | |
| | 1574 | out: |
| | 1575 | fput(csock->file); |
| | 1576 | return err; |
| | 1577 | } |
| | 1578 | |
| | 1579 | static struct proto kcm_proto = { |
| | 1580 | .name = "KCM", |
| | 1581 | .owner = THIS_MODULE, |
| | 1582 | .obj_size = sizeof(struct kcm_sock), |
| | 1583 | }; |
| | 1584 | |
| | 1585 | /* Clone a kcm socket. */ |
| | 1586 | static int kcm_clone(struct socket *osock, struct kcm_clone *info, |
| | 1587 | struct socket **newsockp) |
| | 1588 | { |
| | 1589 | struct socket *newsock; |
| | 1590 | struct sock *newsk; |
| | 1591 | struct file *newfile; |
| | 1592 | int err, newfd; |
| | 1593 | |
| | 1594 | err = -ENFILE; |
| | 1595 | newsock = sock_alloc(); |
| | 1596 | if (!newsock) |
| | 1597 | goto out; |
| | 1598 | |
| | 1599 | newsock->type = osock->type; |
| | 1600 | newsock->ops = osock->ops; |
| | 1601 | |
| | 1602 | __module_get(newsock->ops->owner); |
| | 1603 | |
| | 1604 | newfd = get_unused_fd_flags(0); |
| | 1605 | if (unlikely(newfd < 0)) { |
| | 1606 | err = newfd; |
| | 1607 | goto out_fd_fail; |
| | 1608 | } |
| | 1609 | |
| | 1610 | newfile = sock_alloc_file(newsock, 0, osock->sk->sk_prot_creator->name); |
| | 1611 | if (unlikely(IS_ERR(newfile))) { |
| | 1612 | err = PTR_ERR(newfile); |
| | 1613 | goto out_sock_alloc_fail; |
| | 1614 | } |
| | 1615 | |
| | 1616 | newsk = sk_alloc(sock_net(osock->sk), PF_KCM, GFP_KERNEL, |
| | 1617 | &kcm_proto, true); |
| | 1618 | if (!newsk) { |
| | 1619 | err = -ENOMEM; |
| | 1620 | goto out_sk_alloc_fail; |
| | 1621 | } |
| | 1622 | |
| | 1623 | sock_init_data(newsock, newsk); |
| | 1624 | init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux); |
| | 1625 | |
| | 1626 | fd_install(newfd, newfile); |
| | 1627 | *newsockp = newsock; |
| | 1628 | info->fd = newfd; |
| | 1629 | |
| | 1630 | return 0; |
| | 1631 | |
| | 1632 | out_sk_alloc_fail: |
| | 1633 | fput(newfile); |
| | 1634 | out_sock_alloc_fail: |
| | 1635 | put_unused_fd(newfd); |
| | 1636 | out_fd_fail: |
| | 1637 | sock_release(newsock); |
| | 1638 | out: |
| | 1639 | return err; |
| | 1640 | } |
| | 1641 | |
| | 1642 | static int kcm_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) |
| | 1643 | { |
| | 1644 | int err; |
| | 1645 | |
| | 1646 | switch (cmd) { |
| | 1647 | case SIOCKCMATTACH: { |
| | 1648 | struct kcm_attach info; |
| | 1649 | |
| | 1650 | if (copy_from_user(&info, (void __user *)arg, sizeof(info))) |
| | 1651 | err = -EFAULT; |
| | 1652 | |
| | 1653 | err = kcm_attach_ioctl(sock, &info); |
| | 1654 | |
| | 1655 | break; |
| | 1656 | } |
| | 1657 | case SIOCKCMUNATTACH: { |
| | 1658 | struct kcm_unattach info; |
| | 1659 | |
| | 1660 | if (copy_from_user(&info, (void __user *)arg, sizeof(info))) |
| | 1661 | err = -EFAULT; |
| | 1662 | |
| | 1663 | err = kcm_unattach_ioctl(sock, &info); |
| | 1664 | |
| | 1665 | break; |
| | 1666 | } |
| | 1667 | case SIOCKCMCLONE: { |
| | 1668 | struct kcm_clone info; |
| | 1669 | struct socket *newsock = NULL; |
| | 1670 | |
| | 1671 | if (copy_from_user(&info, (void __user *)arg, sizeof(info))) |
| | 1672 | err = -EFAULT; |
| | 1673 | |
| | 1674 | err = kcm_clone(sock, &info, &newsock); |
| | 1675 | |
| | 1676 | if (!err) { |
| | 1677 | if (copy_to_user((void __user *)arg, &info, |
| | 1678 | sizeof(info))) { |
| | 1679 | err = -EFAULT; |
| | 1680 | sock_release(newsock); |
| | 1681 | } |
| | 1682 | } |
| | 1683 | |
| | 1684 | break; |
| | 1685 | } |
| | 1686 | default: |
| | 1687 | err = -ENOIOCTLCMD; |
| | 1688 | break; |
| | 1689 | } |
| | 1690 | |
| | 1691 | return err; |
| | 1692 | } |
| | 1693 | |
| | 1694 | static void free_mux(struct rcu_head *rcu) |
| | 1695 | { |
| | 1696 | struct kcm_mux *mux = container_of(rcu, |
| | 1697 | struct kcm_mux, rcu); |
| | 1698 | |
| | 1699 | kmem_cache_free(kcm_muxp, mux); |
| | 1700 | } |
| | 1701 | |
| | 1702 | static void release_mux(struct kcm_mux *mux) |
| | 1703 | { |
| | 1704 | struct kcm_net *knet = mux->knet; |
| | 1705 | struct kcm_psock *psock, *tmp_psock; |
| | 1706 | |
| | 1707 | /* Release psocks */ |
| | 1708 | list_for_each_entry_safe(psock, tmp_psock, |
| | 1709 | &mux->psocks, psock_list) { |
| | 1710 | if (!WARN_ON(psock->unattaching)) |
| | 1711 | kcm_unattach(psock); |
| | 1712 | } |
| | 1713 | |
| | 1714 | if (WARN_ON(mux->psocks_cnt)) |
| | 1715 | return; |
| | 1716 | |
| | 1717 | __skb_queue_purge(&mux->rx_hold_queue); |
| | 1718 | |
| | 1719 | mutex_lock(&knet->mutex); |
| | 1720 | list_del_rcu(&mux->kcm_mux_list); |
| | 1721 | knet->count--; |
| | 1722 | mutex_unlock(&knet->mutex); |
| | 1723 | |
| | 1724 | call_rcu(&mux->rcu, free_mux); |
| | 1725 | } |
| | 1726 | |
| | 1727 | static void kcm_done(struct kcm_sock *kcm) |
| | 1728 | { |
| | 1729 | struct kcm_mux *mux = kcm->mux; |
| | 1730 | struct sock *sk = &kcm->sk; |
| | 1731 | int socks_cnt; |
| | 1732 | |
| | 1733 | spin_lock_bh(&mux->rx_lock); |
| | 1734 | if (kcm->rx_psock) { |
| | 1735 | /* Cleanup in unreserve_rx_kcm */ |
| | 1736 | WARN_ON(kcm->done); |
| | 1737 | kcm->rx_disabled = 1; |
| | 1738 | kcm->done = 1; |
| | 1739 | spin_unlock_bh(&mux->rx_lock); |
| | 1740 | return; |
| | 1741 | } |
| | 1742 | |
| | 1743 | if (kcm->rx_wait) { |
| | 1744 | list_del(&kcm->wait_rx_list); |
| | 1745 | kcm->rx_wait = false; |
| | 1746 | } |
| | 1747 | /* Move any pending receive messages to other kcm sockets */ |
| | 1748 | requeue_rx_msgs(mux, &sk->sk_receive_queue); |
| | 1749 | |
| | 1750 | spin_unlock_bh(&mux->rx_lock); |
| | 1751 | |
| | 1752 | if (WARN_ON(sk_rmem_alloc_get(sk))) |
| | 1753 | return; |
| | 1754 | |
| | 1755 | /* Detach from MUX */ |
| | 1756 | spin_lock_bh(&mux->lock); |
| | 1757 | |
| | 1758 | list_del(&kcm->kcm_sock_list); |
| | 1759 | mux->kcm_socks_cnt--; |
| | 1760 | socks_cnt = mux->kcm_socks_cnt; |
| | 1761 | |
| | 1762 | spin_unlock_bh(&mux->lock); |
| | 1763 | |
| | 1764 | if (!socks_cnt) { |
| | 1765 | /* We are done with the mux now. */ |
| | 1766 | release_mux(mux); |
| | 1767 | } |
| | 1768 | |
| | 1769 | WARN_ON(kcm->rx_wait); |
| | 1770 | |
| | 1771 | sock_put(&kcm->sk); |
| | 1772 | } |
| | 1773 | |
| | 1774 | /* Called by kcm_release to close a KCM socket. |
| | 1775 | * If this is the last KCM socket on the MUX, destroy the MUX. |
| | 1776 | */ |
| | 1777 | static int kcm_release(struct socket *sock) |
| | 1778 | { |
| | 1779 | struct sock *sk = sock->sk; |
| | 1780 | struct kcm_sock *kcm; |
| | 1781 | struct kcm_mux *mux; |
| | 1782 | struct kcm_psock *psock; |
| | 1783 | |
| | 1784 | if (!sk) |
| | 1785 | return 0; |
| | 1786 | |
| | 1787 | kcm = kcm_sk(sk); |
| | 1788 | mux = kcm->mux; |
| | 1789 | |
| | 1790 | sock_orphan(sk); |
| | 1791 | kfree_skb(kcm->seq_skb); |
| | 1792 | |
| | 1793 | lock_sock(sk); |
| | 1794 | /* Purge queue under lock to avoid race condition with tx_work trying |
| | 1795 | * to act when queue is nonempty. If tx_work runs after this point |
| | 1796 | * it will just return. |
| | 1797 | */ |
| | 1798 | __skb_queue_purge(&sk->sk_write_queue); |
| | 1799 | release_sock(sk); |
| | 1800 | |
| | 1801 | spin_lock_bh(&mux->lock); |
| | 1802 | if (kcm->tx_wait) { |
| | 1803 | /* Take of tx_wait list, after this point there should be no way |
| | 1804 | * that a psock will be assigned to this kcm. |
| | 1805 | */ |
| | 1806 | list_del(&kcm->wait_psock_list); |
| | 1807 | kcm->tx_wait = false; |
| | 1808 | } |
| | 1809 | spin_unlock_bh(&mux->lock); |
| | 1810 | |
| | 1811 | /* Cancel work. After this point there should be no outside references |
| | 1812 | * to the kcm socket. |
| | 1813 | */ |
| | 1814 | cancel_work_sync(&kcm->tx_work); |
| | 1815 | |
| | 1816 | lock_sock(sk); |
| | 1817 | psock = kcm->tx_psock; |
| | 1818 | if (psock) { |
| | 1819 | /* A psock was reserved, so we need to kill it since it |
| | 1820 | * may already have some bytes queued from a message. We |
| | 1821 | * need to do this after removing kcm from tx_wait list. |
| | 1822 | */ |
| | 1823 | kcm_abort_tx_psock(psock, EPIPE, false); |
| | 1824 | unreserve_psock(kcm); |
| | 1825 | } |
| | 1826 | release_sock(sk); |
| | 1827 | |
| | 1828 | WARN_ON(kcm->tx_wait); |
| | 1829 | WARN_ON(kcm->tx_psock); |
| | 1830 | |
| | 1831 | sock->sk = NULL; |
| | 1832 | |
| | 1833 | kcm_done(kcm); |
| | 1834 | |
| | 1835 | return 0; |
| | 1836 | } |
| | 1837 | |
| | 1838 | static const struct proto_ops kcm_ops = { |
| | 1839 | .family = PF_KCM, |
| | 1840 | .owner = THIS_MODULE, |
| | 1841 | .release = kcm_release, |
| | 1842 | .bind = sock_no_bind, |
| | 1843 | .connect = sock_no_connect, |
| | 1844 | .socketpair = sock_no_socketpair, |
| | 1845 | .accept = sock_no_accept, |
| | 1846 | .getname = sock_no_getname, |
| | 1847 | .poll = datagram_poll, |
| | 1848 | .ioctl = kcm_ioctl, |
| | 1849 | .listen = sock_no_listen, |
| | 1850 | .shutdown = sock_no_shutdown, |
| | 1851 | .setsockopt = kcm_setsockopt, |
| | 1852 | .getsockopt = kcm_getsockopt, |
| | 1853 | .sendmsg = kcm_sendmsg, |
| | 1854 | .recvmsg = kcm_recvmsg, |
| | 1855 | .mmap = sock_no_mmap, |
| | 1856 | .sendpage = sock_no_sendpage, |
| | 1857 | }; |
| | 1858 | |
| | 1859 | /* Create proto operation for kcm sockets */ |
| | 1860 | static int kcm_create(struct net *net, struct socket *sock, |
| | 1861 | int protocol, int kern) |
| | 1862 | { |
| | 1863 | struct kcm_net *knet = net_generic(net, kcm_net_id); |
| | 1864 | struct sock *sk; |
| | 1865 | struct kcm_mux *mux; |
| | 1866 | |
| | 1867 | switch (sock->type) { |
| | 1868 | case SOCK_DGRAM: |
| | 1869 | case SOCK_SEQPACKET: |
| | 1870 | sock->ops = &kcm_ops; |
| | 1871 | break; |
| | 1872 | default: |
| | 1873 | return -ESOCKTNOSUPPORT; |
| | 1874 | } |
| | 1875 | |
| | 1876 | if (protocol != KCMPROTO_CONNECTED) |
| | 1877 | return -EPROTONOSUPPORT; |
| | 1878 | |
| | 1879 | sk = sk_alloc(net, PF_KCM, GFP_KERNEL, &kcm_proto, kern); |
| | 1880 | if (!sk) |
| | 1881 | return -ENOMEM; |
| | 1882 | |
| | 1883 | /* Allocate a kcm mux, shared between KCM sockets */ |
| | 1884 | mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL); |
| | 1885 | if (!mux) { |
| | 1886 | sk_free(sk); |
| | 1887 | return -ENOMEM; |
| | 1888 | } |
| | 1889 | |
| | 1890 | spin_lock_init(&mux->lock); |
| | 1891 | spin_lock_init(&mux->rx_lock); |
| | 1892 | INIT_LIST_HEAD(&mux->kcm_socks); |
| | 1893 | INIT_LIST_HEAD(&mux->kcm_rx_waiters); |
| | 1894 | INIT_LIST_HEAD(&mux->kcm_tx_waiters); |
| | 1895 | |
| | 1896 | INIT_LIST_HEAD(&mux->psocks); |
| | 1897 | INIT_LIST_HEAD(&mux->psocks_ready); |
| | 1898 | INIT_LIST_HEAD(&mux->psocks_avail); |
| | 1899 | |
| | 1900 | mux->knet = knet; |
| | 1901 | |
| | 1902 | /* Add new MUX to list */ |
| | 1903 | mutex_lock(&knet->mutex); |
| | 1904 | list_add_rcu(&mux->kcm_mux_list, &knet->mux_list); |
| | 1905 | knet->count++; |
| | 1906 | mutex_unlock(&knet->mutex); |
| | 1907 | |
| | 1908 | skb_queue_head_init(&mux->rx_hold_queue); |
| | 1909 | |
| | 1910 | /* Init KCM socket */ |
| | 1911 | sock_init_data(sock, sk); |
| | 1912 | init_kcm_sock(kcm_sk(sk), mux); |
| | 1913 | |
| | 1914 | return 0; |
| | 1915 | } |
| | 1916 | |
| | 1917 | static struct net_proto_family kcm_family_ops = { |
| | 1918 | .family = PF_KCM, |
| | 1919 | .create = kcm_create, |
| | 1920 | .owner = THIS_MODULE, |
| | 1921 | }; |
| | 1922 | |
| | 1923 | static __net_init int kcm_init_net(struct net *net) |
| | 1924 | { |
| | 1925 | struct kcm_net *knet = net_generic(net, kcm_net_id); |
| | 1926 | |
| | 1927 | INIT_LIST_HEAD_RCU(&knet->mux_list); |
| | 1928 | mutex_init(&knet->mutex); |
| | 1929 | |
| | 1930 | return 0; |
| | 1931 | } |
| | 1932 | |
| | 1933 | static __net_exit void kcm_exit_net(struct net *net) |
| | 1934 | { |
| | 1935 | struct kcm_net *knet = net_generic(net, kcm_net_id); |
| | 1936 | |
| | 1937 | /* All KCM sockets should be closed at this point, which should mean |
| | 1938 | * that all multiplexors and psocks have been destroyed. |
| | 1939 | */ |
| | 1940 | WARN_ON(!list_empty(&knet->mux_list)); |
| | 1941 | } |
| | 1942 | |
| | 1943 | static struct pernet_operations kcm_net_ops = { |
| | 1944 | .init = kcm_init_net, |
| | 1945 | .exit = kcm_exit_net, |
| | 1946 | .id = &kcm_net_id, |
| | 1947 | .size = sizeof(struct kcm_net), |
| | 1948 | }; |
| | 1949 | |
| | 1950 | static int __init kcm_init(void) |
| | 1951 | { |
| | 1952 | int err = -ENOMEM; |
| | 1953 | |
| | 1954 | kcm_muxp = kmem_cache_create("kcm_mux_cache", |
| | 1955 | sizeof(struct kcm_mux), 0, |
| | 1956 | SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); |
| | 1957 | if (!kcm_muxp) |
| | 1958 | goto fail; |
| | 1959 | |
| | 1960 | kcm_psockp = kmem_cache_create("kcm_psock_cache", |
| | 1961 | sizeof(struct kcm_psock), 0, |
| | 1962 | SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); |
| | 1963 | if (!kcm_psockp) |
| | 1964 | goto fail; |
| | 1965 | |
| | 1966 | kcm_wq = create_singlethread_workqueue("kkcmd"); |
| | 1967 | if (!kcm_wq) |
| | 1968 | goto fail; |
| | 1969 | |
| | 1970 | err = proto_register(&kcm_proto, 1); |
| | 1971 | if (err) |
| | 1972 | goto fail; |
| | 1973 | |
| | 1974 | err = sock_register(&kcm_family_ops); |
| | 1975 | if (err) |
| | 1976 | goto sock_register_fail; |
| | 1977 | |
| | 1978 | err = register_pernet_device(&kcm_net_ops); |
| | 1979 | if (err) |
| | 1980 | goto net_ops_fail; |
| | 1981 | |
| | 1982 | return 0; |
| | 1983 | |
| | 1984 | net_ops_fail: |
| | 1985 | sock_unregister(PF_KCM); |
| | 1986 | |
| | 1987 | sock_register_fail: |
| | 1988 | proto_unregister(&kcm_proto); |
| | 1989 | |
| | 1990 | fail: |
| | 1991 | kmem_cache_destroy(kcm_muxp); |
| | 1992 | kmem_cache_destroy(kcm_psockp); |
| | 1993 | |
| | 1994 | if (kcm_wq) |
| | 1995 | destroy_workqueue(kcm_wq); |
| | 1996 | |
| | 1997 | return err; |
| | 1998 | } |
| | 1999 | |
| | 2000 | static void __exit kcm_exit(void) |
| | 2001 | { |
| | 2002 | unregister_pernet_device(&kcm_net_ops); |
| | 2003 | sock_unregister(PF_KCM); |
| | 2004 | proto_unregister(&kcm_proto); |
| | 2005 | destroy_workqueue(kcm_wq); |
| | 2006 | |
| | 2007 | kmem_cache_destroy(kcm_muxp); |
| | 2008 | kmem_cache_destroy(kcm_psockp); |
| | 2009 | } |
| | 2010 | |
| | 2011 | module_init(kcm_init); |
| | 2012 | module_exit(kcm_exit); |
| | 2013 | |
| | 2014 | MODULE_LICENSE("GPL"); |
| | 2015 | MODULE_ALIAS_NETPROTO(PF_KCM); |
| | 2016 | |