Pull VFIO updates from Alex Williamson: - Replace deprecated git://github.com link in MAINTAINERS (Palmer Dabbelt) - Simplify vfio/mlx5 with module_pci_driver() helper (Shang XiaoJing) - Drop unnecessary buffer from ACPI call (Rafael Mendonca) - Correct latent missing include issue in iova-bitmap and fix support for unaligned bitmaps. Follow-up with better fix through refactor (Joao Martins) - Rework ccw mdev driver to split private data from parent structure, better aligning with the mdev lifecycle and allowing us to remove a temporary workaround (Eric Farman) - Add an interface to get an estimated migration data size for a device, allowing userspace to make informed decisions, ex. more accurately predicting VM downtime (Yishai Hadas) - Fix minor typo in vfio/mlx5 array declaration (Yishai Hadas) - Simplify module and Kconfig through consolidating SPAPR/EEH code and config options and folding virqfd module into main vfio module (Jason Gunthorpe) - Fix error path from device_register() across all vfio mdev and sample drivers (Alex Williamson) - Define migration pre-copy interface and implement for vfio/mlx5 devices, allowing portions of the device state to be saved while the device continues operation, towards reducing the stop-copy state size (Jason Gunthorpe, Yishai Hadas, Shay Drory) - Implement pre-copy for hisi_acc devices (Shameer Kolothum) - Fixes to mdpy mdev driver remove path and error path on probe (Shang XiaoJing) - vfio/mlx5 fixes for incorrect return after copy_to_user() fault and incorrect buffer freeing (Dan Carpenter) * tag 'vfio-v6.2-rc1' of https://github.com/awilliam/linux-vfio: (42 commits) vfio/mlx5: error pointer dereference in error handling vfio/mlx5: fix error code in mlx5vf_precopy_ioctl() samples: vfio-mdev: Fix missing pci_disable_device() in mdpy_fb_probe() hisi_acc_vfio_pci: Enable PRE_COPY flag hisi_acc_vfio_pci: Move the dev compatibility tests for early check hisi_acc_vfio_pci: Introduce support for PRE_COPY state transitions hisi_acc_vfio_pci: Add support for precopy IOCTL vfio/mlx5: Enable MIGRATION_PRE_COPY flag vfio/mlx5: Fallback to STOP_COPY upon specific PRE_COPY error vfio/mlx5: Introduce multiple loads vfio/mlx5: Consider temporary end of stream as part of PRE_COPY vfio/mlx5: Introduce vfio precopy ioctl implementation vfio/mlx5: Introduce SW headers for migration states vfio/mlx5: Introduce device transitions of PRE_COPY vfio/mlx5: Refactor to use queue based data chunks vfio/mlx5: Refactor migration file state vfio/mlx5: Refactor MKEY usage vfio/mlx5: Refactor PD usage vfio/mlx5: Enforce a single SAVE command at a time vfio: Extend the device migration protocol with PRE_COPY ...
Showing the changes to include/uapi (1 of 34 files). Show all
Unified · Split
| @@ -819,12 +819,20 @@ struct vfio_device_feature { · 818 unmodified lines | |||
| 819 | * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P means that RUNNING_P2P | 819 | * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P means that RUNNING_P2P |
| 820 | * is supported in addition to the STOP_COPY states. | 820 | * is supported in addition to the STOP_COPY states. |
| 821 | * | 821 | * |
| 822 | * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY means that | ||
| 823 | * PRE_COPY is supported in addition to the STOP_COPY states. | ||
| 824 | * | ||
| 825 | * VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P | VFIO_MIGRATION_PRE_COPY | ||
| 826 | * means that RUNNING_P2P, PRE_COPY and PRE_COPY_P2P are supported | ||
| 827 | * in addition to the STOP_COPY states. | ||
| 828 | * | ||
| 822 | * Other combinations of flags have behavior to be defined in the future. | 829 | * Other combinations of flags have behavior to be defined in the future. |
| 823 | */ | 830 | */ |
| 824 | struct vfio_device_feature_migration { | 831 | struct vfio_device_feature_migration { |
| 825 | __aligned_u64 flags; | 832 | __aligned_u64 flags; |
| 826 | #define VFIO_MIGRATION_STOP_COPY (1 << 0) | 833 | #define VFIO_MIGRATION_STOP_COPY (1 << 0) |
| 827 | #define VFIO_MIGRATION_P2P (1 << 1) | 834 | #define VFIO_MIGRATION_P2P (1 << 1) |
| 835 | #define VFIO_MIGRATION_PRE_COPY (1 << 2) | ||
| 828 | }; | 836 | }; |
| 829 | #define VFIO_DEVICE_FEATURE_MIGRATION 1 | 837 | #define VFIO_DEVICE_FEATURE_MIGRATION 1 |
| 830 | 838 | ||
| @@ -875,8 +883,13 @@ struct vfio_device_feature_mig_state { · 44 unmodified lines | |||
| 875 | * RESUMING - The device is stopped and is loading a new internal state | 883 | * RESUMING - The device is stopped and is loading a new internal state |
| 876 | * ERROR - The device has failed and must be reset | 884 | * ERROR - The device has failed and must be reset |
| 877 | * | 885 | * |
| 878 | * And 1 optional state to support VFIO_MIGRATION_P2P: | 886 | * And optional states to support VFIO_MIGRATION_P2P: |
| 879 | * RUNNING_P2P - RUNNING, except the device cannot do peer to peer DMA | 887 | * RUNNING_P2P - RUNNING, except the device cannot do peer to peer DMA |
| 888 | * And VFIO_MIGRATION_PRE_COPY: | ||
| 889 | * PRE_COPY - The device is running normally but tracking internal state | ||
| 890 | * changes | ||
| 891 | * And VFIO_MIGRATION_P2P | VFIO_MIGRATION_PRE_COPY: | ||
| 892 | * PRE_COPY_P2P - PRE_COPY, except the device cannot do peer to peer DMA | ||
| 880 | * | 893 | * |
| 881 | * The FSM takes actions on the arcs between FSM states. The driver implements | 894 | * The FSM takes actions on the arcs between FSM states. The driver implements |
| 882 | * the following behavior for the FSM arcs: | 895 | * the following behavior for the FSM arcs: |
| @@ -908,20 +921,48 @@ struct vfio_device_feature_mig_state { · 25 unmodified lines | |||
| 908 | * | 921 | * |
| 909 | * To abort a RESUMING session the device must be reset. | 922 | * To abort a RESUMING session the device must be reset. |
| 910 | * | 923 | * |
| 924 | * PRE_COPY -> RUNNING | ||
| 911 | * RUNNING_P2P -> RUNNING | 925 | * RUNNING_P2P -> RUNNING |
| 912 | * While in RUNNING the device is fully operational, the device may generate | 926 | * While in RUNNING the device is fully operational, the device may generate |
| 913 | * interrupts, DMA, respond to MMIO, all vfio device regions are functional, | 927 | * interrupts, DMA, respond to MMIO, all vfio device regions are functional, |
| 914 | * and the device may advance its internal state. | 928 | * and the device may advance its internal state. |
| 915 | * | 929 | * |
| 930 | * The PRE_COPY arc will terminate a data transfer session. | ||
| 931 | * | ||
| 932 | * PRE_COPY_P2P -> RUNNING_P2P | ||
| 916 | * RUNNING -> RUNNING_P2P | 933 | * RUNNING -> RUNNING_P2P |
| 917 | * STOP -> RUNNING_P2P | 934 | * STOP -> RUNNING_P2P |
| 918 | * While in RUNNING_P2P the device is partially running in the P2P quiescent | 935 | * While in RUNNING_P2P the device is partially running in the P2P quiescent |
| 919 | * state defined below. | 936 | * state defined below. |
| 920 | * | 937 | * |
| 938 | * The PRE_COPY_P2P arc will terminate a data transfer session. | ||
| 939 | * | ||
| 940 | * RUNNING -> PRE_COPY | ||
| 941 | * RUNNING_P2P -> PRE_COPY_P2P | ||
| 921 | * STOP -> STOP_COPY | 942 | * STOP -> STOP_COPY |
| 922 | * This arc begin the process of saving the device state and will return a | 943 | * PRE_COPY, PRE_COPY_P2P and STOP_COPY form the "saving group" of states |
| 923 | * new data_fd. | 944 | * which share a data transfer session. Moving between these states alters |
| 945 | * what is streamed in session, but does not terminate or otherwise affect | ||
| 946 | * the associated fd. | ||
| 947 | * | ||
| 948 | * These arcs begin the process of saving the device state and will return a | ||
| 949 | * new data_fd. The migration driver may perform actions such as enabling | ||
| 950 | * dirty logging of device state when entering PRE_COPY or PER_COPY_P2P. | ||
| 951 | * | ||
| 952 | * Each arc does not change the device operation, the device remains | ||
| 953 | * RUNNING, P2P quiesced or in STOP. The STOP_COPY state is described below | ||
| 954 | * in PRE_COPY_P2P -> STOP_COPY. | ||
| 924 | * | 955 | * |
| 956 | * PRE_COPY -> PRE_COPY_P2P | ||
| 957 | * Entering PRE_COPY_P2P continues all the behaviors of PRE_COPY above. | ||
| 958 | * However, while in the PRE_COPY_P2P state, the device is partially running | ||
| 959 | * in the P2P quiescent state defined below, like RUNNING_P2P. | ||
| 960 | * | ||
| 961 | * PRE_COPY_P2P -> PRE_COPY | ||
| 962 | * This arc allows returning the device to a full RUNNING behavior while | ||
| 963 | * continuing all the behaviors of PRE_COPY. | ||
| 964 | * | ||
| 965 | * PRE_COPY_P2P -> STOP_COPY | ||
| 925 | * While in the STOP_COPY state the device has the same behavior as STOP | 966 | * While in the STOP_COPY state the device has the same behavior as STOP |
| 926 | * with the addition that the data transfers session continues to stream the | 967 | * with the addition that the data transfers session continues to stream the |
| 927 | * migration state. End of stream on the FD indicates the entire device | 968 | * migration state. End of stream on the FD indicates the entire device |
| @@ -939,6 +980,13 @@ struct vfio_device_feature_mig_state { · 11 unmodified lines | |||
| 939 | * device state for this arc if required to prepare the device to receive the | 980 | * device state for this arc if required to prepare the device to receive the |
| 940 | * migration data. | 981 | * migration data. |
| 941 | * | 982 | * |
| 983 | * STOP_COPY -> PRE_COPY | ||
| 984 | * STOP_COPY -> PRE_COPY_P2P | ||
| 985 | * These arcs are not permitted and return error if requested. Future | ||
| 986 | * revisions of this API may define behaviors for these arcs, in this case | ||
| 987 | * support will be discoverable by a new flag in | ||
| 988 | * VFIO_DEVICE_FEATURE_MIGRATION. | ||
| 989 | * | ||
| 942 | * any -> ERROR | 990 | * any -> ERROR |
| 943 | * ERROR cannot be specified as a device state, however any transition request | 991 | * ERROR cannot be specified as a device state, however any transition request |
| 944 | * can be failed with an errno return and may then move the device_state into | 992 | * can be failed with an errno return and may then move the device_state into |
| @@ -950,7 +998,7 @@ struct vfio_device_feature_mig_state { · 5 unmodified lines | |||
| 950 | * The optional peer to peer (P2P) quiescent state is intended to be a quiescent | 998 | * The optional peer to peer (P2P) quiescent state is intended to be a quiescent |
| 951 | * state for the device for the purposes of managing multiple devices within a | 999 | * state for the device for the purposes of managing multiple devices within a |
| 952 | * user context where peer-to-peer DMA between devices may be active. The | 1000 | * user context where peer-to-peer DMA between devices may be active. The |
| 953 | * RUNNING_P2P states must prevent the device from initiating | 1001 | * RUNNING_P2P and PRE_COPY_P2P states must prevent the device from initiating |
| 954 | * any new P2P DMA transactions. If the device can identify P2P transactions | 1002 | * any new P2P DMA transactions. If the device can identify P2P transactions |
| 955 | * then it can stop only P2P DMA, otherwise it must stop all DMA. The migration | 1003 | * then it can stop only P2P DMA, otherwise it must stop all DMA. The migration |
| 956 | * driver must complete any such outstanding operations prior to completing the | 1004 | * driver must complete any such outstanding operations prior to completing the |
| @@ -963,6 +1011,8 @@ struct vfio_device_feature_mig_state { · 6 unmodified lines | |||
| 963 | * above FSM arcs. As there are multiple paths through the FSM arcs the path | 1011 | * above FSM arcs. As there are multiple paths through the FSM arcs the path |
| 964 | * should be selected based on the following rules: | 1012 | * should be selected based on the following rules: |
| 965 | * - Select the shortest path. | 1013 | * - Select the shortest path. |
| 1014 | * - The path cannot have saving group states as interior arcs, only | ||
| 1015 | * starting/end states. | ||
| 966 | * Refer to vfio_mig_get_next_state() for the result of the algorithm. | 1016 | * Refer to vfio_mig_get_next_state() for the result of the algorithm. |
| 967 | * | 1017 | * |
| 968 | * The automatic transit through the FSM arcs that make up the combination | 1018 | * The automatic transit through the FSM arcs that make up the combination |
| @@ -976,6 +1026,9 @@ struct vfio_device_feature_mig_state { · 7 unmodified lines | |||
| 976 | * support them. The user can discover if these states are supported by using | 1026 | * support them. The user can discover if these states are supported by using |
| 977 | * VFIO_DEVICE_FEATURE_MIGRATION. By using combination transitions the user can | 1027 | * VFIO_DEVICE_FEATURE_MIGRATION. By using combination transitions the user can |
| 978 | * avoid knowing about these optional states if the kernel driver supports them. | 1028 | * avoid knowing about these optional states if the kernel driver supports them. |
| 1029 | * | ||
| 1030 | * Arcs touching PRE_COPY and PRE_COPY_P2P are removed if support for PRE_COPY | ||
| 1031 | * is not present. | ||
| 979 | */ | 1032 | */ |
| 980 | enum vfio_device_mig_state { | 1033 | enum vfio_device_mig_state { |
| 981 | VFIO_DEVICE_STATE_ERROR = 0, | 1034 | VFIO_DEVICE_STATE_ERROR = 0, |
| @@ -984,8 +1037,70 @@ enum vfio_device_mig_state { · 2 unmodified lines | |||
| 984 | VFIO_DEVICE_STATE_STOP_COPY = 3, | 1037 | VFIO_DEVICE_STATE_STOP_COPY = 3, |
| 985 | VFIO_DEVICE_STATE_RESUMING = 4, | 1038 | VFIO_DEVICE_STATE_RESUMING = 4, |
| 986 | VFIO_DEVICE_STATE_RUNNING_P2P = 5, | 1039 | VFIO_DEVICE_STATE_RUNNING_P2P = 5, |
| 1040 | VFIO_DEVICE_STATE_PRE_COPY = 6, | ||
| 1041 | VFIO_DEVICE_STATE_PRE_COPY_P2P = 7, | ||
| 1042 | }; | ||
| 1043 | |||
| 1044 | /** | ||
| 1045 | * VFIO_MIG_GET_PRECOPY_INFO - _IO(VFIO_TYPE, VFIO_BASE + 21) | ||
| 1046 | * | ||
| 1047 | * This ioctl is used on the migration data FD in the precopy phase of the | ||
| 1048 | * migration data transfer. It returns an estimate of the current data sizes | ||
| 1049 | * remaining to be transferred. It allows the user to judge when it is | ||
| 1050 | * appropriate to leave PRE_COPY for STOP_COPY. | ||
| 1051 | * | ||
| 1052 | * This ioctl is valid only in PRE_COPY states and kernel driver should | ||
| 1053 | * return -EINVAL from any other migration state. | ||
| 1054 | * | ||
| 1055 | * The vfio_precopy_info data structure returned by this ioctl provides | ||
| 1056 | * estimates of data available from the device during the PRE_COPY states. | ||
| 1057 | * This estimate is split into two categories, initial_bytes and | ||
| 1058 | * dirty_bytes. | ||
| 1059 | * | ||
| 1060 | * The initial_bytes field indicates the amount of initial precopy | ||
| 1061 | * data available from the device. This field should have a non-zero initial | ||
| 1062 | * value and decrease as migration data is read from the device. | ||
| 1063 | * It is recommended to leave PRE_COPY for STOP_COPY only after this field | ||
| 1064 | * reaches zero. Leaving PRE_COPY earlier might make things slower. | ||
| 1065 | * | ||
| 1066 | * The dirty_bytes field tracks device state changes relative to data | ||
| 1067 | * previously retrieved. This field starts at zero and may increase as | ||
| 1068 | * the internal device state is modified or decrease as that modified | ||
| 1069 | * state is read from the device. | ||
| 1070 | * | ||
| 1071 | * Userspace may use the combination of these fields to estimate the | ||
| 1072 | * potential data size available during the PRE_COPY phases, as well as | ||
| 1073 | * trends relative to the rate the device is dirtying its internal | ||
| 1074 | * state, but these fields are not required to have any bearing relative | ||
| 1075 | * to the data size available during the STOP_COPY phase. | ||
| 1076 | * | ||
| 1077 | * Drivers have a lot of flexibility in when and what they transfer during the | ||
| 1078 | * PRE_COPY phase, and how they report this from VFIO_MIG_GET_PRECOPY_INFO. | ||
| 1079 | * | ||
| 1080 | * During pre-copy the migration data FD has a temporary "end of stream" that is | ||
| 1081 | * reached when both initial_bytes and dirty_byte are zero. For instance, this | ||
| 1082 | * may indicate that the device is idle and not currently dirtying any internal | ||
| 1083 | * state. When read() is done on this temporary end of stream the kernel driver | ||
| 1084 | * should return ENOMSG from read(). Userspace can wait for more data (which may | ||
| 1085 | * never come) by using poll. | ||
| 1086 | * | ||
| 1087 | * Once in STOP_COPY the migration data FD has a permanent end of stream | ||
| 1088 | * signaled in the usual way by read() always returning 0 and poll always | ||
| 1089 | * returning readable. ENOMSG may not be returned in STOP_COPY. | ||
| 1090 | * Support for this ioctl is mandatory if a driver claims to support | ||
| 1091 | * VFIO_MIGRATION_PRE_COPY. | ||
| 1092 | * | ||
| 1093 | * Return: 0 on success, -1 and errno set on failure. | ||
| 1094 | */ | ||
| 1095 | struct vfio_precopy_info { | ||
| 1096 | __u32 argsz; | ||
| 1097 | __u32 flags; | ||
| 1098 | __aligned_u64 initial_bytes; | ||
| 1099 | __aligned_u64 dirty_bytes; | ||
| 987 | }; | 1100 | }; |
| 988 | 1101 | ||
| 1102 | #define VFIO_MIG_GET_PRECOPY_INFO _IO(VFIO_TYPE, VFIO_BASE + 21) | ||
| 1103 | |||
| 989 | /* | 1104 | /* |
| 990 | * Upon VFIO_DEVICE_FEATURE_SET, allow the device to be moved into a low power | 1105 | * Upon VFIO_DEVICE_FEATURE_SET, allow the device to be moved into a low power |
| 991 | * state with the platform-based power management. Device use of lower power | 1106 | * state with the platform-based power management. Device use of lower power |
| @@ -1128,6 +1243,19 @@ struct vfio_device_feature_dma_logging_report { · 136 unmodified lines | |||
| 1128 | 1243 | ||
| 1129 | #define VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT 8 | 1244 | #define VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT 8 |
| 1130 | 1245 | ||
| 1246 | /* | ||
| 1247 | * Upon VFIO_DEVICE_FEATURE_GET read back the estimated data length that will | ||
| 1248 | * be required to complete stop copy. | ||
| 1249 | * | ||
| 1250 | * Note: Can be called on each device state. | ||
| 1251 | */ | ||
| 1252 | |||
| 1253 | struct vfio_device_feature_mig_data_size { | ||
| 1254 | __aligned_u64 stop_copy_length; | ||
| 1255 | }; | ||
| 1256 | |||
| 1257 | #define VFIO_DEVICE_FEATURE_MIG_DATA_SIZE 9 | ||
| 1258 | |||
| 1131 | /* -------- API for Type1 VFIO IOMMU -------- */ | 1259 | /* -------- API for Type1 VFIO IOMMU -------- */ |
| 1132 | 1260 | ||
| 1133 | /** | 1261 | /** |