History

scsi: storvsc: Re-init stor_chns when a channel interrupt is re-assigned

For each storvsc_device, storvsc keeps track of the channel target CPUs associated to the device (alloced_cpus) and it uses this information to fill a "cache" (stor_chns) mapping CPU->channel according to a certain heuristic. Update the alloced_cpus mask and the stor_chns array when a channel of the storvsc device is re-assigned to a different CPU.

Andrea Parri (Microsoft) committed 6 years ago

Commit 7769e181 parent 75278103 files · +95 −8Browse files

Unified · Split

3 files
  1. drivers/hv/vmbus_drv.c +4 −0
  2. drivers/scsi/storvsc_drv.c +88 −8
  3. include/linux/hyperv.h +3 −0
drivers/hv/vmbus_drv.c+4−0
@@ -1777,6 +1777,10 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel, · 1776 unmodified lines
17771777 * in on a CPU that is different from the channel target_cpu value.
17781778 */
17791779
1780+ if (channel->change_target_cpu_callback)
1781+ (*channel->change_target_cpu_callback)(channel,
1782+ channel->target_cpu, target_cpu);
1783+
17801784 channel->target_cpu = target_cpu;
17811785 channel->target_vp = hv_cpu_number_to_vp_number(target_cpu);
17821786 channel->numa_node = cpu_to_node(target_cpu);
drivers/scsi/storvsc_drv.c+88−8
@@ -621,6 +621,64 @@ get_in_err: · 620 unmodified lines
621621
622622 }
623623
624+static void storvsc_change_target_cpu(struct vmbus_channel *channel, u32 old,
625+ u32 new)
626+{
627+ struct storvsc_device *stor_device;
628+ struct vmbus_channel *cur_chn;
629+ bool old_is_alloced = false;
630+ struct hv_device *device;
631+ unsigned long flags;
632+ int cpu;
633+
634+ device = channel->primary_channel ?
635+ channel->primary_channel->device_obj
636+ : channel->device_obj;
637+ stor_device = get_out_stor_device(device);
638+ if (!stor_device)
639+ return;
640+
641+ /* See storvsc_do_io() -> get_og_chn(). */
642+ spin_lock_irqsave(&device->channel->lock, flags);
643+
644+ /*
645+ * Determines if the storvsc device has other channels assigned to
646+ * the "old" CPU to update the alloced_cpus mask and the stor_chns
647+ * array.
648+ */
649+ if (device->channel != channel && device->channel->target_cpu == old) {
650+ cur_chn = device->channel;
651+ old_is_alloced = true;
652+ goto old_is_alloced;
653+ }
654+ list_for_each_entry(cur_chn, &device->channel->sc_list, sc_list) {
655+ if (cur_chn == channel)
656+ continue;
657+ if (cur_chn->target_cpu == old) {
658+ old_is_alloced = true;
659+ goto old_is_alloced;
660+ }
661+ }
662+
663+old_is_alloced:
664+ if (old_is_alloced)
665+ WRITE_ONCE(stor_device->stor_chns[old], cur_chn);
666+ else
667+ cpumask_clear_cpu(old, &stor_device->alloced_cpus);
668+
669+ /* "Flush" the stor_chns array. */
670+ for_each_possible_cpu(cpu) {
671+ if (stor_device->stor_chns[cpu] && !cpumask_test_cpu(
672+ cpu, &stor_device->alloced_cpus))
673+ WRITE_ONCE(stor_device->stor_chns[cpu], NULL);
674+ }
675+
676+ WRITE_ONCE(stor_device->stor_chns[new], channel);
677+ cpumask_set_cpu(new, &stor_device->alloced_cpus);
678+
679+ spin_unlock_irqrestore(&device->channel->lock, flags);
680+}
681+
624682 static void handle_sc_creation(struct vmbus_channel *new_sc)
625683 {
626684 struct hv_device *device = new_sc->primary_channel->device_obj;
@@ -648,6 +706,8 @@ static void handle_sc_creation(struct vmbus_channel *new_sc) · 21 unmodified lines
648706 return;
649707 }
650708
709+ new_sc->change_target_cpu_callback = storvsc_change_target_cpu;
710+
651711 /* Add the sub-channel to the array of available channels. */
652712 stor_device->stor_chns[new_sc->target_cpu] = new_sc;
653713 cpumask_set_cpu(new_sc->target_cpu, &stor_device->alloced_cpus);
@@ -876,6 +936,8 @@ static int storvsc_channel_init(struct hv_device *device, bool is_fc) · 222 unmodified lines
876936 if (stor_device->stor_chns == NULL)
877937 return -ENOMEM;
878938
939+ device->channel->change_target_cpu_callback = storvsc_change_target_cpu;
940+
879941 stor_device->stor_chns[device->channel->target_cpu] = device->channel;
880942 cpumask_set_cpu(device->channel->target_cpu,
881943 &stor_device->alloced_cpus);
@@ -1248,8 +1310,10 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, · 366 unmodified lines
12481310 const struct cpumask *node_mask;
12491311 int num_channels, tgt_cpu;
12501312
1251− if (stor_device->num_sc == 0)
1313+ if (stor_device->num_sc == 0) {
1314+ stor_device->stor_chns[q_num] = stor_device->device->channel;
12521315 return stor_device->device->channel;
1316+ }
12531317
12541318 /*
12551319 * Our channel array is sparsley populated and we
@@ -1258,7 +1322,6 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, · 2 unmodified lines
12581322 * The strategy is simple:
12591323 * I. Ensure NUMA locality
12601324 * II. Distribute evenly (best effort)
1261− * III. Mapping is persistent.
12621325 */
12631326
12641327 node_mask = cpumask_of_node(cpu_to_node(q_num));
@@ -1268,8 +1331,10 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, · 3 unmodified lines
12681331 if (cpumask_test_cpu(tgt_cpu, node_mask))
12691332 num_channels++;
12701333 }
1271− if (num_channels == 0)
1334+ if (num_channels == 0) {
1335+ stor_device->stor_chns[q_num] = stor_device->device->channel;
12721336 return stor_device->device->channel;
1337+ }
12731338
12741339 hash_qnum = q_num;
12751340 while (hash_qnum >= num_channels)
@@ -1295,6 +1360,7 @@ static int storvsc_do_io(struct hv_device *device, · 19 unmodified lines
12951360 struct storvsc_device *stor_device;
12961361 struct vstor_packet *vstor_packet;
12971362 struct vmbus_channel *outgoing_channel, *channel;
1363+ unsigned long flags;
12981364 int ret = 0;
12991365 const struct cpumask *node_mask;
13001366 int tgt_cpu;
@@ -1308,10 +1374,11 @@ static int storvsc_do_io(struct hv_device *device, · 7 unmodified lines
13081374
13091375 request->device = device;
13101376 /*
1311− * Select an an appropriate channel to send the request out.
1377+ * Select an appropriate channel to send the request out.
13121378 */
1313− if (stor_device->stor_chns[q_num] != NULL) {
1314− outgoing_channel = stor_device->stor_chns[q_num];
1379+ /* See storvsc_change_target_cpu(). */
1380+ outgoing_channel = READ_ONCE(stor_device->stor_chns[q_num]);
1381+ if (outgoing_channel != NULL) {
13151382 if (outgoing_channel->target_cpu == q_num) {
13161383 /*
13171384 * Ideally, we want to pick a different channel if
@@ -1324,7 +1391,10 @@ static int storvsc_do_io(struct hv_device *device, · 6 unmodified lines
13241391 continue;
13251392 if (tgt_cpu == q_num)
13261393 continue;
1327− channel = stor_device->stor_chns[tgt_cpu];
1394+ channel = READ_ONCE(
1395+ stor_device->stor_chns[tgt_cpu]);
1396+ if (channel == NULL)
1397+ continue;
13281398 if (hv_get_avail_to_write_percent(
13291399 &channel->outbound)
13301400 > ring_avail_percent_lowater) {
@@ -1350,7 +1420,10 @@ static int storvsc_do_io(struct hv_device *device, · 19 unmodified lines
13501420 for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) {
13511421 if (cpumask_test_cpu(tgt_cpu, node_mask))
13521422 continue;
1353− channel = stor_device->stor_chns[tgt_cpu];
1423+ channel = READ_ONCE(
1424+ stor_device->stor_chns[tgt_cpu]);
1425+ if (channel == NULL)
1426+ continue;
13541427 if (hv_get_avail_to_write_percent(
13551428 &channel->outbound)
13561429 > ring_avail_percent_lowater) {
@@ -1360,7 +1433,14 @@ static int storvsc_do_io(struct hv_device *device, · 3 unmodified lines
13601433 }
13611434 }
13621435 } else {
1436+ spin_lock_irqsave(&device->channel->lock, flags);
1437+ outgoing_channel = stor_device->stor_chns[q_num];
1438+ if (outgoing_channel != NULL) {
1439+ spin_unlock_irqrestore(&device->channel->lock, flags);
1440+ goto found_channel;
1441+ }
13631442 outgoing_channel = get_og_chn(stor_device, q_num);
1443+ spin_unlock_irqrestore(&device->channel->lock, flags);
13641444 }
13651445
13661446 found_channel:
include/linux/hyperv.h+3−0
@@ -773,6 +773,9 @@ struct vmbus_channel { · 772 unmodified lines
773773 void (*onchannel_callback)(void *context);
774774 void *channel_callback_context;
775775
776+ void (*change_target_cpu_callback)(struct vmbus_channel *channel,
777+ u32 old, u32 new);
778+
776779 /*
777780 * Synchronize channel scheduling and channel removal; see the inline
778781 * comments in vmbus_chan_sched() and vmbus_reset_channel_cb().