| @@ -15,7 +15,8 @@ · 14 unmodified lines |
| 15 | | 15 | |
| 16 | struct kmem_cache *fscache_cookie_jar; | 16 | struct kmem_cache *fscache_cookie_jar; |
| 17 | | 17 | |
| 18 | static void fscache_drop_cookie(struct fscache_cookie *cookie); | 18 | static void fscache_cookie_worker(struct work_struct *work); |
| | 19 | static void fscache_unhash_cookie(struct fscache_cookie *cookie); |
| 19 | | 20 | |
| 20 | #define fscache_cookie_hash_shift 15 | 21 | #define fscache_cookie_hash_shift 15 |
| 21 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; | 22 | static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; |
| @@ -62,6 +63,19 @@ static void fscache_free_cookie(struct fscache_cookie *cookie) · 40 unmodified lines |
| 62 | kmem_cache_free(fscache_cookie_jar, cookie); | 63 | kmem_cache_free(fscache_cookie_jar, cookie); |
| 63 | } | 64 | } |
| 64 | | 65 | |
| | 66 | static void __fscache_queue_cookie(struct fscache_cookie *cookie) |
| | 67 | { |
| | 68 | if (!queue_work(fscache_wq, &cookie->work)) |
| | 69 | fscache_put_cookie(cookie, fscache_cookie_put_over_queued); |
| | 70 | } |
| | 71 | |
| | 72 | static void fscache_queue_cookie(struct fscache_cookie *cookie, |
| | 73 | enum fscache_cookie_trace where) |
| | 74 | { |
| | 75 | fscache_get_cookie(cookie, where); |
| | 76 | __fscache_queue_cookie(cookie); |
| | 77 | } |
| | 78 | |
| 65 | /* | 79 | /* |
| 66 | * Initialise the access gate on a cookie by setting a flag to prevent the | 80 | * Initialise the access gate on a cookie by setting a flag to prevent the |
| 67 | * state machine from being queued when the access counter transitions to 0. | 81 | * state machine from being queued when the access counter transitions to 0. |
| @@ -98,9 +112,8 @@ void fscache_end_cookie_access(struct fscache_cookie *cookie, · 30 unmodified lines |
| 98 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), | 112 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), |
| 99 | n_accesses, why); | 113 | n_accesses, why); |
| 100 | if (n_accesses == 0 && | 114 | if (n_accesses == 0 && |
| 101 | !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) { | 115 | !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) |
| 102 | // PLACEHOLDER: Need to poke the state machine | 116 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); |
| 103 | } | | |
| 104 | } | 117 | } |
| 105 | EXPORT_SYMBOL(fscache_end_cookie_access); | 118 | EXPORT_SYMBOL(fscache_end_cookie_access); |
| 106 | | 119 | |
| @@ -171,35 +184,58 @@ static inline void wake_up_cookie_state(struct fscache_cookie *cookie) · 64 unmodified lines |
| 171 | wake_up_var(&cookie->state); | 184 | wake_up_var(&cookie->state); |
| 172 | } | 185 | } |
| 173 | | 186 | |
| | 187 | /* |
| | 188 | * Change the state a cookie is at and wake up anyone waiting for that. Impose |
| | 189 | * an ordering between the stuff stored in the cookie and the state member. |
| | 190 | * Paired with fscache_cookie_state(). |
| | 191 | */ |
| 174 | static void __fscache_set_cookie_state(struct fscache_cookie *cookie, | 192 | static void __fscache_set_cookie_state(struct fscache_cookie *cookie, |
| 175 | enum fscache_cookie_state state) | 193 | enum fscache_cookie_state state) |
| 176 | { | 194 | { |
| 177 | cookie->state = state; | 195 | smp_store_release(&cookie->state, state); |
| 178 | } | 196 | } |
| 179 | | 197 | |
| 180 | /* | 198 | static void fscache_set_cookie_state(struct fscache_cookie *cookie, |
| 181 | * Change the state a cookie is at and wake up anyone waiting for that - but | 199 | enum fscache_cookie_state state) |
| 182 | * only if the cookie isn't already marked as being in a cleanup state. | | |
| 183 | */ | | |
| 184 | void fscache_set_cookie_state(struct fscache_cookie *cookie, | | |
| 185 | enum fscache_cookie_state state) | | |
| 186 | { | 200 | { |
| 187 | bool changed = false; | | |
| 188 | | | |
| 189 | spin_lock(&cookie->lock); | 201 | spin_lock(&cookie->lock); |
| 190 | switch (cookie->state) { | 202 | __fscache_set_cookie_state(cookie, state); |
| 191 | case FSCACHE_COOKIE_STATE_RELINQUISHING: | | |
| 192 | break; | | |
| 193 | default: | | |
| 194 | __fscache_set_cookie_state(cookie, state); | | |
| 195 | changed = true; | | |
| 196 | break; | | |
| 197 | } | | |
| 198 | spin_unlock(&cookie->lock); | 203 | spin_unlock(&cookie->lock); |
| 199 | if (changed) | 204 | wake_up_cookie_state(cookie); |
| 200 | wake_up_cookie_state(cookie); | 205 | } |
| | 206 | |
| | 207 | /** |
| | 208 | * fscache_cookie_lookup_negative - Note negative lookup |
| | 209 | * @cookie: The cookie that was being looked up |
| | 210 | * |
| | 211 | * Note that some part of the metadata path in the cache doesn't exist and so |
| | 212 | * we can release any waiting readers in the certain knowledge that there's |
| | 213 | * nothing for them to actually read. |
| | 214 | * |
| | 215 | * This function uses no locking and must only be called from the state machine. |
| | 216 | */ |
| | 217 | void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) |
| | 218 | { |
| | 219 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); |
| | 220 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); |
| | 221 | } |
| | 222 | EXPORT_SYMBOL(fscache_cookie_lookup_negative); |
| | 223 | |
| | 224 | /** |
| | 225 | * fscache_caching_failed - Report that a failure stopped caching on a cookie |
| | 226 | * @cookie: The cookie that was affected |
| | 227 | * |
| | 228 | * Tell fscache that caching on a cookie needs to be stopped due to some sort |
| | 229 | * of failure. |
| | 230 | * |
| | 231 | * This function uses no locking and must only be called from the state machine. |
| | 232 | */ |
| | 233 | void fscache_caching_failed(struct fscache_cookie *cookie) |
| | 234 | { |
| | 235 | clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); |
| | 236 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); |
| 201 | } | 237 | } |
| 202 | EXPORT_SYMBOL(fscache_set_cookie_state); | 238 | EXPORT_SYMBOL(fscache_caching_failed); |
| 203 | | 239 | |
| 204 | /* | 240 | /* |
| 205 | * Set the index key in a cookie. The cookie struct has space for a 16-byte | 241 | * Set the index key in a cookie. The cookie struct has space for a 16-byte |
| @@ -291,10 +327,10 @@ static struct fscache_cookie *fscache_alloc_cookie( · 85 unmodified lines |
| 291 | | 327 | |
| 292 | refcount_set(&cookie->ref, 1); | 328 | refcount_set(&cookie->ref, 1); |
| 293 | cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); | 329 | cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); |
| 294 | cookie->state = FSCACHE_COOKIE_STATE_QUIESCENT; | | |
| 295 | spin_lock_init(&cookie->lock); | 330 | spin_lock_init(&cookie->lock); |
| 296 | INIT_LIST_HEAD(&cookie->commit_link); | 331 | INIT_LIST_HEAD(&cookie->commit_link); |
| 297 | INIT_WORK(&cookie->work, NULL /* PLACEHOLDER */); | 332 | INIT_WORK(&cookie->work, fscache_cookie_worker); |
| | 333 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); |
| 298 | | 334 | |
| 299 | write_lock(&fscache_cookies_lock); | 335 | write_lock(&fscache_cookies_lock); |
| 300 | list_add_tail(&cookie->proc_link, &fscache_cookies); | 336 | list_add_tail(&cookie->proc_link, &fscache_cookies); |
| @@ -417,6 +453,192 @@ struct fscache_cookie *__fscache_acquire_cookie( · 116 unmodified lines |
| 417 | } | 453 | } |
| 418 | EXPORT_SYMBOL(__fscache_acquire_cookie); | 454 | EXPORT_SYMBOL(__fscache_acquire_cookie); |
| 419 | | 455 | |
| | 456 | /* |
| | 457 | * Prepare a cache object to be written to. |
| | 458 | */ |
| | 459 | static void fscache_prepare_to_write(struct fscache_cookie *cookie) |
| | 460 | { |
| | 461 | cookie->volume->cache->ops->prepare_to_write(cookie); |
| | 462 | } |
| | 463 | |
| | 464 | /* |
| | 465 | * Look up a cookie in the cache. |
| | 466 | */ |
| | 467 | static void fscache_perform_lookup(struct fscache_cookie *cookie) |
| | 468 | { |
| | 469 | enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; |
| | 470 | bool need_withdraw = false; |
| | 471 | |
| | 472 | _enter(""); |
| | 473 | |
| | 474 | if (!cookie->volume->cache_priv) { |
| | 475 | fscache_create_volume(cookie->volume, true); |
| | 476 | if (!cookie->volume->cache_priv) { |
| | 477 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); |
| | 478 | goto out; |
| | 479 | } |
| | 480 | } |
| | 481 | |
| | 482 | if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { |
| | 483 | if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) |
| | 484 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); |
| | 485 | need_withdraw = true; |
| | 486 | _leave(" [fail]"); |
| | 487 | goto out; |
| | 488 | } |
| | 489 | |
| | 490 | fscache_see_cookie(cookie, fscache_cookie_see_active); |
| | 491 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); |
| | 492 | trace = fscache_access_lookup_cookie_end; |
| | 493 | |
| | 494 | out: |
| | 495 | fscache_end_cookie_access(cookie, trace); |
| | 496 | if (need_withdraw) |
| | 497 | fscache_withdraw_cookie(cookie); |
| | 498 | fscache_end_volume_access(cookie->volume, cookie, trace); |
| | 499 | } |
| | 500 | |
| | 501 | /* |
| | 502 | * Perform work upon the cookie, such as committing its cache state, |
| | 503 | * relinquishing it or withdrawing the backing cache. We're protected from the |
| | 504 | * cache going away under us as object withdrawal must come through this |
| | 505 | * non-reentrant work item. |
| | 506 | */ |
| | 507 | static void fscache_cookie_state_machine(struct fscache_cookie *cookie) |
| | 508 | { |
| | 509 | enum fscache_cookie_state state; |
| | 510 | bool wake = false; |
| | 511 | |
| | 512 | _enter("c=%x", cookie->debug_id); |
| | 513 | |
| | 514 | again: |
| | 515 | spin_lock(&cookie->lock); |
| | 516 | again_locked: |
| | 517 | state = cookie->state; |
| | 518 | switch (state) { |
| | 519 | case FSCACHE_COOKIE_STATE_QUIESCENT: |
| | 520 | /* The QUIESCENT state is jumped to the LOOKING_UP state by |
| | 521 | * fscache_use_cookie(). |
| | 522 | */ |
| | 523 | |
| | 524 | if (atomic_read(&cookie->n_accesses) == 0 && |
| | 525 | test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { |
| | 526 | __fscache_set_cookie_state(cookie, |
| | 527 | FSCACHE_COOKIE_STATE_RELINQUISHING); |
| | 528 | wake = true; |
| | 529 | goto again_locked; |
| | 530 | } |
| | 531 | break; |
| | 532 | |
| | 533 | case FSCACHE_COOKIE_STATE_LOOKING_UP: |
| | 534 | spin_unlock(&cookie->lock); |
| | 535 | fscache_init_access_gate(cookie); |
| | 536 | fscache_perform_lookup(cookie); |
| | 537 | goto again; |
| | 538 | |
| | 539 | case FSCACHE_COOKIE_STATE_ACTIVE: |
| | 540 | if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { |
| | 541 | spin_unlock(&cookie->lock); |
| | 542 | fscache_prepare_to_write(cookie); |
| | 543 | spin_lock(&cookie->lock); |
| | 544 | } |
| | 545 | fallthrough; |
| | 546 | |
| | 547 | case FSCACHE_COOKIE_STATE_FAILED: |
| | 548 | if (atomic_read(&cookie->n_accesses) != 0) |
| | 549 | break; |
| | 550 | if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { |
| | 551 | __fscache_set_cookie_state(cookie, |
| | 552 | FSCACHE_COOKIE_STATE_RELINQUISHING); |
| | 553 | wake = true; |
| | 554 | goto again_locked; |
| | 555 | } |
| | 556 | if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { |
| | 557 | __fscache_set_cookie_state(cookie, |
| | 558 | FSCACHE_COOKIE_STATE_WITHDRAWING); |
| | 559 | wake = true; |
| | 560 | goto again_locked; |
| | 561 | } |
| | 562 | break; |
| | 563 | |
| | 564 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
| | 565 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
| | 566 | if (cookie->cache_priv) { |
| | 567 | spin_unlock(&cookie->lock); |
| | 568 | cookie->volume->cache->ops->withdraw_cookie(cookie); |
| | 569 | spin_lock(&cookie->lock); |
| | 570 | } |
| | 571 | |
| | 572 | switch (state) { |
| | 573 | case FSCACHE_COOKIE_STATE_RELINQUISHING: |
| | 574 | fscache_see_cookie(cookie, fscache_cookie_see_relinquish); |
| | 575 | fscache_unhash_cookie(cookie); |
| | 576 | __fscache_set_cookie_state(cookie, |
| | 577 | FSCACHE_COOKIE_STATE_DROPPED); |
| | 578 | wake = true; |
| | 579 | goto out; |
| | 580 | case FSCACHE_COOKIE_STATE_WITHDRAWING: |
| | 581 | fscache_see_cookie(cookie, fscache_cookie_see_withdraw); |
| | 582 | break; |
| | 583 | default: |
| | 584 | BUG(); |
| | 585 | } |
| | 586 | |
| | 587 | clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); |
| | 588 | clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); |
| | 589 | clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); |
| | 590 | clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); |
| | 591 | set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); |
| | 592 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); |
| | 593 | wake = true; |
| | 594 | goto again_locked; |
| | 595 | |
| | 596 | case FSCACHE_COOKIE_STATE_DROPPED: |
| | 597 | break; |
| | 598 | |
| | 599 | default: |
| | 600 | WARN_ONCE(1, "Cookie %x in unexpected state %u\n", |
| | 601 | cookie->debug_id, state); |
| | 602 | break; |
| | 603 | } |
| | 604 | |
| | 605 | out: |
| | 606 | spin_unlock(&cookie->lock); |
| | 607 | if (wake) |
| | 608 | wake_up_cookie_state(cookie); |
| | 609 | _leave(""); |
| | 610 | } |
| | 611 | |
| | 612 | static void fscache_cookie_worker(struct work_struct *work) |
| | 613 | { |
| | 614 | struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); |
| | 615 | |
| | 616 | fscache_see_cookie(cookie, fscache_cookie_see_work); |
| | 617 | fscache_cookie_state_machine(cookie); |
| | 618 | fscache_put_cookie(cookie, fscache_cookie_put_work); |
| | 619 | } |
| | 620 | |
| | 621 | /* |
| | 622 | * Wait for the object to become inactive. The cookie's work item will be |
| | 623 | * scheduled when someone transitions n_accesses to 0 - but if someone's |
| | 624 | * already done that, schedule it anyway. |
| | 625 | */ |
| | 626 | static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) |
| | 627 | { |
| | 628 | int n_accesses; |
| | 629 | bool unpinned; |
| | 630 | |
| | 631 | unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); |
| | 632 | |
| | 633 | /* Need to read the access count after unpinning */ |
| | 634 | n_accesses = atomic_read(&cookie->n_accesses); |
| | 635 | if (unpinned) |
| | 636 | trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), |
| | 637 | n_accesses, fscache_access_cache_unpin); |
| | 638 | if (n_accesses == 0) |
| | 639 | fscache_queue_cookie(cookie, fscache_cookie_get_end_access); |
| | 640 | } |
| | 641 | |
| 420 | /* | 642 | /* |
| 421 | * Remove a cookie from the hash table. | 643 | * Remove a cookie from the hash table. |
| 422 | */ | 644 | */ |
| @@ -432,21 +654,27 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie) · 9 unmodified lines |
| 432 | hlist_bl_del(&cookie->hash_link); | 654 | hlist_bl_del(&cookie->hash_link); |
| 433 | clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); | 655 | clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); |
| 434 | hlist_bl_unlock(h); | 656 | hlist_bl_unlock(h); |
| | 657 | fscache_stat(&fscache_n_relinquishes_dropped); |
| 435 | } | 658 | } |
| 436 | | 659 | |
| 437 | /* | 660 | static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) |
| 438 | * Finalise a cookie after all its resources have been disposed of. | | |
| 439 | */ | | |
| 440 | static void fscache_drop_cookie(struct fscache_cookie *cookie) | | |
| 441 | { | 661 | { |
| 442 | spin_lock(&cookie->lock); | 662 | __fscache_withdraw_cookie(cookie); |
| 443 | __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); | 663 | } |
| 444 | spin_unlock(&cookie->lock); | | |
| 445 | wake_up_cookie_state(cookie); | | |
| 446 | | 664 | |
| 447 | fscache_unhash_cookie(cookie); | 665 | /** |
| 448 | fscache_stat(&fscache_n_relinquishes_dropped); | 666 | * fscache_withdraw_cookie - Mark a cookie for withdrawal |
| | 667 | * @cookie: The cookie to be withdrawn. |
| | 668 | * |
| | 669 | * Allow the cache backend to withdraw the backing for a cookie for its own |
| | 670 | * reasons, even if that cookie is in active use. |
| | 671 | */ |
| | 672 | void fscache_withdraw_cookie(struct fscache_cookie *cookie) |
| | 673 | { |
| | 674 | set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); |
| | 675 | fscache_drop_withdraw_cookie(cookie); |
| 449 | } | 676 | } |
| | 677 | EXPORT_SYMBOL(fscache_withdraw_cookie); |
| 450 | | 678 | |
| 451 | /* | 679 | /* |
| 452 | * Allow the netfs to release a cookie back to the cache. | 680 | * Allow the netfs to release a cookie back to the cache. |
| @@ -473,12 +701,13 @@ void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) · 20 unmodified lines |
| 473 | ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); | 701 | ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); |
| 474 | atomic_dec(&cookie->volume->n_cookies); | 702 | atomic_dec(&cookie->volume->n_cookies); |
| 475 | | 703 | |
| 476 | set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); | 704 | if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { |
| 477 | | 705 | set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); |
| 478 | if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) | 706 | fscache_drop_withdraw_cookie(cookie); |
| 479 | ; // PLACEHOLDER: Do something here if the cookie was cached | 707 | } else { |
| 480 | else | 708 | fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); |
| 481 | fscache_drop_cookie(cookie); | 709 | fscache_unhash_cookie(cookie); |
| | 710 | } |
| 482 | fscache_put_cookie(cookie, fscache_cookie_put_relinquish); | 711 | fscache_put_cookie(cookie, fscache_cookie_put_relinquish); |
| 483 | } | 712 | } |
| 484 | EXPORT_SYMBOL(__fscache_relinquish_cookie); | 713 | EXPORT_SYMBOL(__fscache_relinquish_cookie); |