Before suspend, Linux must make sure all the hv_sock channels have been properly cleaned up, because a hv_sock connection can not persist across hibernation, and the user-space app must be properly notified of the state change of the connection. Before suspend, Linux also must make sure all the sub-channels have been destroyed, i.e. the related channel structs of the sub-channels must be properly removed, otherwise they would cause a conflict when the sub-channels are recreated upon resume. Add a counter to track such channels, and vmbus_bus_suspend() should wait for the counter to drop to zero.
Unified · Split
| @@ -545,6 +545,10 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) · 544 unmodified lines | ||
| 545 | 545 | |
| 546 | 546 | mutex_lock(&vmbus_connection.channel_mutex); |
| 547 | 547 | |
| 548 | + /* Remember the channels that should be cleaned up upon suspend. */ | |
| 549 | + if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) | |
| 550 | + atomic_inc(&vmbus_connection.nr_chan_close_on_suspend); | |
| 551 | + | |
| 548 | 552 | /* |
| 549 | 553 | * Now that we have acquired the channel_mutex, |
| 550 | 554 | * we can release the potentially racing rescind thread. |
| @@ -944,6 +948,16 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) · 393 unmodified lines | ||
| 944 | 948 | vmbus_process_offer(newchannel); |
| 945 | 949 | } |
| 946 | 950 | |
| 951 | +static void check_ready_for_suspend_event(void) | |
| 952 | +{ | |
| 953 | + /* | |
| 954 | + * If all the sub-channels or hv_sock channels have been cleaned up, | |
| 955 | + * then it's safe to suspend. | |
| 956 | + */ | |
| 957 | + if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend)) | |
| 958 | + complete(&vmbus_connection.ready_for_suspend_event); | |
| 959 | +} | |
| 960 | + | |
| 947 | 961 | /* |
| 948 | 962 | * vmbus_onoffer_rescind - Rescind offer handler. |
| 949 | 963 | * |
| @@ -954,6 +968,7 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) · 4 unmodified lines | ||
| 954 | 968 | struct vmbus_channel_rescind_offer *rescind; |
| 955 | 969 | struct vmbus_channel *channel; |
| 956 | 970 | struct device *dev; |
| 971 | + bool clean_up_chan_for_suspend; | |
| 957 | 972 | |
| 958 | 973 | rescind = (struct vmbus_channel_rescind_offer *)hdr; |
| 959 | 974 | |
| @@ -993,6 +1008,8 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) · 33 unmodified lines | ||
| 993 | 1008 | return; |
| 994 | 1009 | } |
| 995 | 1010 | |
| 1011 | + clean_up_chan_for_suspend = is_hvsock_channel(channel) || | |
| 1012 | + is_sub_channel(channel); | |
| 996 | 1013 | /* |
| 997 | 1014 | * Before setting channel->rescind in vmbus_rescind_cleanup(), we |
| 998 | 1015 | * should make sure the channel callback is not running any more. |
| @@ -1018,6 +1035,10 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) · 19 unmodified lines | ||
| 1018 | 1035 | if (channel->device_obj) { |
| 1019 | 1036 | if (channel->chn_rescind_callback) { |
| 1020 | 1037 | channel->chn_rescind_callback(channel); |
| 1038 | + | |
| 1039 | + if (clean_up_chan_for_suspend) | |
| 1040 | + check_ready_for_suspend_event(); | |
| 1041 | + | |
| 1021 | 1042 | return; |
| 1022 | 1043 | } |
| 1023 | 1044 | /* |
| @@ -1050,6 +1071,11 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) · 26 unmodified lines | ||
| 1050 | 1071 | } |
| 1051 | 1072 | mutex_unlock(&vmbus_connection.channel_mutex); |
| 1052 | 1073 | } |
| 1074 | + | |
| 1075 | + /* The "channel" may have been freed. Do not access it any longer. */ | |
| 1076 | + | |
| 1077 | + if (clean_up_chan_for_suspend) | |
| 1078 | + check_ready_for_suspend_event(); | |
| 1053 | 1079 | } |
| 1054 | 1080 | |
| 1055 | 1081 | void vmbus_hvsock_device_unregister(struct vmbus_channel *channel) |
| @@ -26,6 +26,9 @@ · 25 unmodified lines | ||
| 26 | 26 | struct vmbus_connection vmbus_connection = { |
| 27 | 27 | .conn_state = DISCONNECTED, |
| 28 | 28 | .next_gpadl_handle = ATOMIC_INIT(0xE1E10), |
| 29 | + | |
| 30 | + .ready_for_suspend_event= COMPLETION_INITIALIZER( | |
| 31 | + vmbus_connection.ready_for_suspend_event), | |
| 29 | 32 | }; |
| 30 | 33 | EXPORT_SYMBOL_GPL(vmbus_connection); |
| 31 | 34 | |
| @@ -260,6 +260,18 @@ struct vmbus_connection { · 259 unmodified lines | ||
| 260 | 260 | struct workqueue_struct *work_queue; |
| 261 | 261 | struct workqueue_struct *handle_primary_chan_wq; |
| 262 | 262 | struct workqueue_struct *handle_sub_chan_wq; |
| 263 | + | |
| 264 | + /* | |
| 265 | + * The number of sub-channels and hv_sock channels that should be | |
| 266 | + * cleaned up upon suspend: sub-channels will be re-created upon | |
| 267 | + * resume, and hv_sock channels should not survive suspend. | |
| 268 | + */ | |
| 269 | + atomic_t nr_chan_close_on_suspend; | |
| 270 | + /* | |
| 271 | + * vmbus_bus_suspend() waits for "nr_chan_close_on_suspend" to | |
| 272 | + * drop to zero. | |
| 273 | + */ | |
| 274 | + struct completion ready_for_suspend_event; | |
| 263 | 275 | }; |
| 264 | 276 | |
| 265 | 277 | |
| @@ -2127,7 +2127,8 @@ acpi_walk_err: · 2126 unmodified lines | ||
| 2127 | 2127 | |
| 2128 | 2128 | static int vmbus_bus_suspend(struct device *dev) |
| 2129 | 2129 | { |
| 2130 | − struct vmbus_channel *channel; | |
| 2130 | + struct vmbus_channel *channel, *sc; | |
| 2131 | + unsigned long flags; | |
| 2131 | 2132 | |
| 2132 | 2133 | while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { |
| 2133 | 2134 | /* |
| @@ -2146,6 +2147,44 @@ static int vmbus_bus_suspend(struct device *dev) · 12 unmodified lines | ||
| 2146 | 2147 | } |
| 2147 | 2148 | mutex_unlock(&vmbus_connection.channel_mutex); |
| 2148 | 2149 | |
| 2150 | + /* | |
| 2151 | + * Wait until all the sub-channels and hv_sock channels have been | |
| 2152 | + * cleaned up. Sub-channels should be destroyed upon suspend, otherwise | |
| 2153 | + * they would conflict with the new sub-channels that will be created | |
| 2154 | + * in the resume path. hv_sock channels should also be destroyed, but | |
| 2155 | + * a hv_sock channel of an established hv_sock connection can not be | |
| 2156 | + * really destroyed since it may still be referenced by the userspace | |
| 2157 | + * application, so we just force the hv_sock channel to be rescinded | |
| 2158 | + * by vmbus_force_channel_rescinded(), and the userspace application | |
| 2159 | + * will thoroughly destroy the channel after hibernation. | |
| 2160 | + * | |
| 2161 | + * Note: the counter nr_chan_close_on_suspend may never go above 0 if | |
| 2162 | + * the VM has no sub-channel and hv_sock channel, e.g. a 1-vCPU VM. | |
| 2163 | + */ | |
| 2164 | + if (atomic_read(&vmbus_connection.nr_chan_close_on_suspend) > 0) | |
| 2165 | + wait_for_completion(&vmbus_connection.ready_for_suspend_event); | |
| 2166 | + | |
| 2167 | + mutex_lock(&vmbus_connection.channel_mutex); | |
| 2168 | + | |
| 2169 | + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { | |
| 2170 | + if (is_hvsock_channel(channel)) { | |
| 2171 | + if (!channel->rescind) { | |
| 2172 | + pr_err("hv_sock channel not rescinded!\n"); | |
| 2173 | + WARN_ON_ONCE(1); | |
| 2174 | + } | |
| 2175 | + continue; | |
| 2176 | + } | |
| 2177 | + | |
| 2178 | + spin_lock_irqsave(&channel->lock, flags); | |
| 2179 | + list_for_each_entry(sc, &channel->sc_list, sc_list) { | |
| 2180 | + pr_err("Sub-channel not deleted!\n"); | |
| 2181 | + WARN_ON_ONCE(1); | |
| 2182 | + } | |
| 2183 | + spin_unlock_irqrestore(&channel->lock, flags); | |
| 2184 | + } | |
| 2185 | + | |
| 2186 | + mutex_unlock(&vmbus_connection.channel_mutex); | |
| 2187 | + | |
| 2149 | 2188 | vmbus_initiate_unload(false); |
| 2150 | 2189 | |
| 2151 | 2190 | vmbus_connection.conn_state = DISCONNECTED; |
| @@ -2186,6 +2225,9 @@ static int vmbus_bus_resume(struct device *dev) · 34 unmodified lines | ||
| 2186 | 2225 | |
| 2187 | 2226 | vmbus_request_offers(); |
| 2188 | 2227 | |
| 2228 | + /* Reset the event for the next suspend. */ | |
| 2229 | + reinit_completion(&vmbus_connection.ready_for_suspend_event); | |
| 2230 | + | |
| 2189 | 2231 | return 0; |
| 2190 | 2232 | } |
| 2191 | 2233 | |