| @@ -89,6 +89,7 @@ · 88 unmodified lines |
| 89 | #include <drm/i915_drm.h> | 89 | #include <drm/i915_drm.h> |
| 90 | #include "i915_drv.h" | 90 | #include "i915_drv.h" |
| 91 | #include "i915_trace.h" | 91 | #include "i915_trace.h" |
| | 92 | #include "intel_lrc_reg.h" |
| 92 | #include "intel_workarounds.h" | 93 | #include "intel_workarounds.h" |
| 93 | | 94 | |
| 94 | #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 | 95 | #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 |
| @@ -321,6 +322,15 @@ static u32 default_desc_template(const struct drm_i915_private *i915, · 226 unmodified lines |
| 321 | return desc; | 322 | return desc; |
| 322 | } | 323 | } |
| 323 | | 324 | |
| | 325 | static void intel_context_retire(struct i915_gem_active *active, |
| | 326 | struct i915_request *rq) |
| | 327 | { |
| | 328 | struct intel_context *ce = |
| | 329 | container_of(active, typeof(*ce), active_tracker); |
| | 330 | |
| | 331 | intel_context_unpin(ce); |
| | 332 | } |
| | 333 | |
| 324 | void | 334 | void |
| 325 | intel_context_init(struct intel_context *ce, | 335 | intel_context_init(struct intel_context *ce, |
| 326 | struct i915_gem_context *ctx, | 336 | struct i915_gem_context *ctx, |
| @@ -333,6 +343,8 @@ intel_context_init(struct intel_context *ce, · 6 unmodified lines |
| 333 | | 343 | |
| 334 | /* Use the whole device by default */ | 344 | /* Use the whole device by default */ |
| 335 | ce->sseu = intel_device_default_sseu(ctx->i915); | 345 | ce->sseu = intel_device_default_sseu(ctx->i915); |
| | 346 | |
| | 347 | init_request_active(&ce->active_tracker, intel_context_retire); |
| 336 | } | 348 | } |
| 337 | | 349 | |
| 338 | static struct i915_gem_context * | 350 | static struct i915_gem_context * |
| @@ -850,6 +862,56 @@ out: · 511 unmodified lines |
| 850 | return 0; | 862 | return 0; |
| 851 | } | 863 | } |
| 852 | | 864 | |
| | 865 | static int get_sseu(struct i915_gem_context *ctx, |
| | 866 | struct drm_i915_gem_context_param *args) |
| | 867 | { |
| | 868 | struct drm_i915_gem_context_param_sseu user_sseu; |
| | 869 | struct intel_engine_cs *engine; |
| | 870 | struct intel_context *ce; |
| | 871 | int ret; |
| | 872 | |
| | 873 | if (args->size == 0) |
| | 874 | goto out; |
| | 875 | else if (args->size < sizeof(user_sseu)) |
| | 876 | return -EINVAL; |
| | 877 | |
| | 878 | if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), |
| | 879 | sizeof(user_sseu))) |
| | 880 | return -EFAULT; |
| | 881 | |
| | 882 | if (user_sseu.flags || user_sseu.rsvd) |
| | 883 | return -EINVAL; |
| | 884 | |
| | 885 | engine = intel_engine_lookup_user(ctx->i915, |
| | 886 | user_sseu.engine_class, |
| | 887 | user_sseu.engine_instance); |
| | 888 | if (!engine) |
| | 889 | return -EINVAL; |
| | 890 | |
| | 891 | /* Only use for mutex here is to serialize get_param and set_param. */ |
| | 892 | ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); |
| | 893 | if (ret) |
| | 894 | return ret; |
| | 895 | |
| | 896 | ce = to_intel_context(ctx, engine); |
| | 897 | |
| | 898 | user_sseu.slice_mask = ce->sseu.slice_mask; |
| | 899 | user_sseu.subslice_mask = ce->sseu.subslice_mask; |
| | 900 | user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; |
| | 901 | user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; |
| | 902 | |
| | 903 | mutex_unlock(&ctx->i915->drm.struct_mutex); |
| | 904 | |
| | 905 | if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, |
| | 906 | sizeof(user_sseu))) |
| | 907 | return -EFAULT; |
| | 908 | |
| | 909 | out: |
| | 910 | args->size = sizeof(user_sseu); |
| | 911 | |
| | 912 | return 0; |
| | 913 | } |
| | 914 | |
| 853 | int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, | 915 | int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, |
| 854 | struct drm_file *file) | 916 | struct drm_file *file) |
| 855 | { | 917 | { |
| @@ -862,15 +924,17 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, · 6 unmodified lines |
| 862 | if (!ctx) | 924 | if (!ctx) |
| 863 | return -ENOENT; | 925 | return -ENOENT; |
| 864 | | 926 | |
| 865 | args->size = 0; | | |
| 866 | switch (args->param) { | 927 | switch (args->param) { |
| 867 | case I915_CONTEXT_PARAM_BAN_PERIOD: | 928 | case I915_CONTEXT_PARAM_BAN_PERIOD: |
| 868 | ret = -EINVAL; | 929 | ret = -EINVAL; |
| 869 | break; | 930 | break; |
| 870 | case I915_CONTEXT_PARAM_NO_ZEROMAP: | 931 | case I915_CONTEXT_PARAM_NO_ZEROMAP: |
| | 932 | args->size = 0; |
| 871 | args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); | 933 | args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); |
| 872 | break; | 934 | break; |
| 873 | case I915_CONTEXT_PARAM_GTT_SIZE: | 935 | case I915_CONTEXT_PARAM_GTT_SIZE: |
| | 936 | args->size = 0; |
| | 937 | |
| 874 | if (ctx->ppgtt) | 938 | if (ctx->ppgtt) |
| 875 | args->value = ctx->ppgtt->vm.total; | 939 | args->value = ctx->ppgtt->vm.total; |
| 876 | else if (to_i915(dev)->mm.aliasing_ppgtt) | 940 | else if (to_i915(dev)->mm.aliasing_ppgtt) |
| @@ -879,14 +943,20 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, · 2 unmodified lines |
| 879 | args->value = to_i915(dev)->ggtt.vm.total; | 943 | args->value = to_i915(dev)->ggtt.vm.total; |
| 880 | break; | 944 | break; |
| 881 | case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: | 945 | case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: |
| | 946 | args->size = 0; |
| 882 | args->value = i915_gem_context_no_error_capture(ctx); | 947 | args->value = i915_gem_context_no_error_capture(ctx); |
| 883 | break; | 948 | break; |
| 884 | case I915_CONTEXT_PARAM_BANNABLE: | 949 | case I915_CONTEXT_PARAM_BANNABLE: |
| | 950 | args->size = 0; |
| 885 | args->value = i915_gem_context_is_bannable(ctx); | 951 | args->value = i915_gem_context_is_bannable(ctx); |
| 886 | break; | 952 | break; |
| 887 | case I915_CONTEXT_PARAM_PRIORITY: | 953 | case I915_CONTEXT_PARAM_PRIORITY: |
| | 954 | args->size = 0; |
| 888 | args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; | 955 | args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; |
| 889 | break; | 956 | break; |
| | 957 | case I915_CONTEXT_PARAM_SSEU: |
| | 958 | ret = get_sseu(ctx, args); |
| | 959 | break; |
| 890 | default: | 960 | default: |
| 891 | ret = -EINVAL; | 961 | ret = -EINVAL; |
| 892 | break; | 962 | break; |
| @@ -896,6 +966,270 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, · 3 unmodified lines |
| 896 | return ret; | 966 | return ret; |
| 897 | } | 967 | } |
| 898 | | 968 | |
| | 969 | static int gen8_emit_rpcs_config(struct i915_request *rq, |
| | 970 | struct intel_context *ce, |
| | 971 | struct intel_sseu sseu) |
| | 972 | { |
| | 973 | u64 offset; |
| | 974 | u32 *cs; |
| | 975 | |
| | 976 | cs = intel_ring_begin(rq, 4); |
| | 977 | if (IS_ERR(cs)) |
| | 978 | return PTR_ERR(cs); |
| | 979 | |
| | 980 | offset = i915_ggtt_offset(ce->state) + |
| | 981 | LRC_STATE_PN * PAGE_SIZE + |
| | 982 | (CTX_R_PWR_CLK_STATE + 1) * 4; |
| | 983 | |
| | 984 | *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; |
| | 985 | *cs++ = lower_32_bits(offset); |
| | 986 | *cs++ = upper_32_bits(offset); |
| | 987 | *cs++ = gen8_make_rpcs(rq->i915, &sseu); |
| | 988 | |
| | 989 | intel_ring_advance(rq, cs); |
| | 990 | |
| | 991 | return 0; |
| | 992 | } |
| | 993 | |
| | 994 | static int |
| | 995 | gen8_modify_rpcs_gpu(struct intel_context *ce, |
| | 996 | struct intel_engine_cs *engine, |
| | 997 | struct intel_sseu sseu) |
| | 998 | { |
| | 999 | struct drm_i915_private *i915 = engine->i915; |
| | 1000 | struct i915_request *rq, *prev; |
| | 1001 | intel_wakeref_t wakeref; |
| | 1002 | int ret; |
| | 1003 | |
| | 1004 | GEM_BUG_ON(!ce->pin_count); |
| | 1005 | |
| | 1006 | lockdep_assert_held(&i915->drm.struct_mutex); |
| | 1007 | |
| | 1008 | /* Submitting requests etc needs the hw awake. */ |
| | 1009 | wakeref = intel_runtime_pm_get(i915); |
| | 1010 | |
| | 1011 | rq = i915_request_alloc(engine, i915->kernel_context); |
| | 1012 | if (IS_ERR(rq)) { |
| | 1013 | ret = PTR_ERR(rq); |
| | 1014 | goto out_put; |
| | 1015 | } |
| | 1016 | |
| | 1017 | /* Queue this switch after all other activity by this context. */ |
| | 1018 | prev = i915_gem_active_raw(&ce->ring->timeline->last_request, |
| | 1019 | &i915->drm.struct_mutex); |
| | 1020 | if (prev && !i915_request_completed(prev)) { |
| | 1021 | ret = i915_request_await_dma_fence(rq, &prev->fence); |
| | 1022 | if (ret < 0) |
| | 1023 | goto out_add; |
| | 1024 | } |
| | 1025 | |
| | 1026 | /* Order all following requests to be after. */ |
| | 1027 | ret = i915_timeline_set_barrier(ce->ring->timeline, rq); |
| | 1028 | if (ret) |
| | 1029 | goto out_add; |
| | 1030 | |
| | 1031 | ret = gen8_emit_rpcs_config(rq, ce, sseu); |
| | 1032 | if (ret) |
| | 1033 | goto out_add; |
| | 1034 | |
| | 1035 | /* |
| | 1036 | * Guarantee context image and the timeline remains pinned until the |
| | 1037 | * modifying request is retired by setting the ce activity tracker. |
| | 1038 | * |
| | 1039 | * But we only need to take one pin on the account of it. Or in other |
| | 1040 | * words transfer the pinned ce object to tracked active request. |
| | 1041 | */ |
| | 1042 | if (!i915_gem_active_isset(&ce->active_tracker)) |
| | 1043 | __intel_context_pin(ce); |
| | 1044 | i915_gem_active_set(&ce->active_tracker, rq); |
| | 1045 | |
| | 1046 | out_add: |
| | 1047 | i915_request_add(rq); |
| | 1048 | out_put: |
| | 1049 | intel_runtime_pm_put(i915, wakeref); |
| | 1050 | |
| | 1051 | return ret; |
| | 1052 | } |
| | 1053 | |
| | 1054 | static int |
| | 1055 | i915_gem_context_reconfigure_sseu(struct i915_gem_context *ctx, |
| | 1056 | struct intel_engine_cs *engine, |
| | 1057 | struct intel_sseu sseu) |
| | 1058 | { |
| | 1059 | struct intel_context *ce = to_intel_context(ctx, engine); |
| | 1060 | int ret; |
| | 1061 | |
| | 1062 | GEM_BUG_ON(INTEL_GEN(ctx->i915) < 8); |
| | 1063 | GEM_BUG_ON(engine->id != RCS); |
| | 1064 | |
| | 1065 | ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); |
| | 1066 | if (ret) |
| | 1067 | return ret; |
| | 1068 | |
| | 1069 | /* Nothing to do if unmodified. */ |
| | 1070 | if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) |
| | 1071 | goto out; |
| | 1072 | |
| | 1073 | /* |
| | 1074 | * If context is not idle we have to submit an ordered request to modify |
| | 1075 | * its context image via the kernel context. Pristine and idle contexts |
| | 1076 | * will be configured on pinning. |
| | 1077 | */ |
| | 1078 | if (ce->pin_count) |
| | 1079 | ret = gen8_modify_rpcs_gpu(ce, engine, sseu); |
| | 1080 | |
| | 1081 | if (!ret) |
| | 1082 | ce->sseu = sseu; |
| | 1083 | |
| | 1084 | out: |
| | 1085 | mutex_unlock(&ctx->i915->drm.struct_mutex); |
| | 1086 | |
| | 1087 | return ret; |
| | 1088 | } |
| | 1089 | |
| | 1090 | static int |
| | 1091 | user_to_context_sseu(struct drm_i915_private *i915, |
| | 1092 | const struct drm_i915_gem_context_param_sseu *user, |
| | 1093 | struct intel_sseu *context) |
| | 1094 | { |
| | 1095 | const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; |
| | 1096 | |
| | 1097 | /* No zeros in any field. */ |
| | 1098 | if (!user->slice_mask || !user->subslice_mask || |
| | 1099 | !user->min_eus_per_subslice || !user->max_eus_per_subslice) |
| | 1100 | return -EINVAL; |
| | 1101 | |
| | 1102 | /* Max > min. */ |
| | 1103 | if (user->max_eus_per_subslice < user->min_eus_per_subslice) |
| | 1104 | return -EINVAL; |
| | 1105 | |
| | 1106 | /* |
| | 1107 | * Some future proofing on the types since the uAPI is wider than the |
| | 1108 | * current internal implementation. |
| | 1109 | */ |
| | 1110 | if (overflows_type(user->slice_mask, context->slice_mask) || |
| | 1111 | overflows_type(user->subslice_mask, context->subslice_mask) || |
| | 1112 | overflows_type(user->min_eus_per_subslice, |
| | 1113 | context->min_eus_per_subslice) || |
| | 1114 | overflows_type(user->max_eus_per_subslice, |
| | 1115 | context->max_eus_per_subslice)) |
| | 1116 | return -EINVAL; |
| | 1117 | |
| | 1118 | /* Check validity against hardware. */ |
| | 1119 | if (user->slice_mask & ~device->slice_mask) |
| | 1120 | return -EINVAL; |
| | 1121 | |
| | 1122 | if (user->subslice_mask & ~device->subslice_mask[0]) |
| | 1123 | return -EINVAL; |
| | 1124 | |
| | 1125 | if (user->max_eus_per_subslice > device->max_eus_per_subslice) |
| | 1126 | return -EINVAL; |
| | 1127 | |
| | 1128 | context->slice_mask = user->slice_mask; |
| | 1129 | context->subslice_mask = user->subslice_mask; |
| | 1130 | context->min_eus_per_subslice = user->min_eus_per_subslice; |
| | 1131 | context->max_eus_per_subslice = user->max_eus_per_subslice; |
| | 1132 | |
| | 1133 | /* Part specific restrictions. */ |
| | 1134 | if (IS_GEN(i915, 11)) { |
| | 1135 | unsigned int hw_s = hweight8(device->slice_mask); |
| | 1136 | unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); |
| | 1137 | unsigned int req_s = hweight8(context->slice_mask); |
| | 1138 | unsigned int req_ss = hweight8(context->subslice_mask); |
| | 1139 | |
| | 1140 | /* |
| | 1141 | * Only full subslice enablement is possible if more than one |
| | 1142 | * slice is turned on. |
| | 1143 | */ |
| | 1144 | if (req_s > 1 && req_ss != hw_ss_per_s) |
| | 1145 | return -EINVAL; |
| | 1146 | |
| | 1147 | /* |
| | 1148 | * If more than four (SScount bitfield limit) subslices are |
| | 1149 | * requested then the number has to be even. |
| | 1150 | */ |
| | 1151 | if (req_ss > 4 && (req_ss & 1)) |
| | 1152 | return -EINVAL; |
| | 1153 | |
| | 1154 | /* |
| | 1155 | * If only one slice is enabled and subslice count is below the |
| | 1156 | * device full enablement, it must be at most half of the all |
| | 1157 | * available subslices. |
| | 1158 | */ |
| | 1159 | if (req_s == 1 && req_ss < hw_ss_per_s && |
| | 1160 | req_ss > (hw_ss_per_s / 2)) |
| | 1161 | return -EINVAL; |
| | 1162 | |
| | 1163 | /* ABI restriction - VME use case only. */ |
| | 1164 | |
| | 1165 | /* All slices or one slice only. */ |
| | 1166 | if (req_s != 1 && req_s != hw_s) |
| | 1167 | return -EINVAL; |
| | 1168 | |
| | 1169 | /* |
| | 1170 | * Half subslices or full enablement only when one slice is |
| | 1171 | * enabled. |
| | 1172 | */ |
| | 1173 | if (req_s == 1 && |
| | 1174 | (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) |
| | 1175 | return -EINVAL; |
| | 1176 | |
| | 1177 | /* No EU configuration changes. */ |
| | 1178 | if ((user->min_eus_per_subslice != |
| | 1179 | device->max_eus_per_subslice) || |
| | 1180 | (user->max_eus_per_subslice != |
| | 1181 | device->max_eus_per_subslice)) |
| | 1182 | return -EINVAL; |
| | 1183 | } |
| | 1184 | |
| | 1185 | return 0; |
| | 1186 | } |
| | 1187 | |
| | 1188 | static int set_sseu(struct i915_gem_context *ctx, |
| | 1189 | struct drm_i915_gem_context_param *args) |
| | 1190 | { |
| | 1191 | struct drm_i915_private *i915 = ctx->i915; |
| | 1192 | struct drm_i915_gem_context_param_sseu user_sseu; |
| | 1193 | struct intel_engine_cs *engine; |
| | 1194 | struct intel_sseu sseu; |
| | 1195 | int ret; |
| | 1196 | |
| | 1197 | if (args->size < sizeof(user_sseu)) |
| | 1198 | return -EINVAL; |
| | 1199 | |
| | 1200 | if (!IS_GEN(i915, 11)) |
| | 1201 | return -ENODEV; |
| | 1202 | |
| | 1203 | if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), |
| | 1204 | sizeof(user_sseu))) |
| | 1205 | return -EFAULT; |
| | 1206 | |
| | 1207 | if (user_sseu.flags || user_sseu.rsvd) |
| | 1208 | return -EINVAL; |
| | 1209 | |
| | 1210 | engine = intel_engine_lookup_user(i915, |
| | 1211 | user_sseu.engine_class, |
| | 1212 | user_sseu.engine_instance); |
| | 1213 | if (!engine) |
| | 1214 | return -EINVAL; |
| | 1215 | |
| | 1216 | /* Only render engine supports RPCS configuration. */ |
| | 1217 | if (engine->class != RENDER_CLASS) |
| | 1218 | return -ENODEV; |
| | 1219 | |
| | 1220 | ret = user_to_context_sseu(i915, &user_sseu, &sseu); |
| | 1221 | if (ret) |
| | 1222 | return ret; |
| | 1223 | |
| | 1224 | ret = i915_gem_context_reconfigure_sseu(ctx, engine, sseu); |
| | 1225 | if (ret) |
| | 1226 | return ret; |
| | 1227 | |
| | 1228 | args->size = sizeof(user_sseu); |
| | 1229 | |
| | 1230 | return 0; |
| | 1231 | } |
| | 1232 | |
| 899 | int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, | 1233 | int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, |
| 900 | struct drm_file *file) | 1234 | struct drm_file *file) |
| 901 | { | 1235 | { |
| @@ -958,7 +1292,9 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, · 56 unmodified lines |
| 958 | I915_USER_PRIORITY(priority); | 1292 | I915_USER_PRIORITY(priority); |
| 959 | } | 1293 | } |
| 960 | break; | 1294 | break; |
| 961 | | 1295 | case I915_CONTEXT_PARAM_SSEU: |
| | 1296 | ret = set_sseu(ctx, args); |
| | 1297 | break; |
| 962 | default: | 1298 | default: |
| 963 | ret = -EINVAL; | 1299 | ret = -EINVAL; |
| 964 | break; | 1300 | break; |