History

habanalabs: expand clock throttling information uAPI

In addition to the clock throttling reason, user should be able to obtain also the start time and the duration of the throttling event.

Ofir Bitton committed 5 years ago

Commit 1679c7e1 parent 48f31166 files · +110 −14Browse files

Unified · Split

6 files
  1. drivers/misc/habanalabs/common/device.c +3 −0
  2. drivers/misc/habanalabs/common/habanalabs.h +29 −2
  3. drivers/misc/habanalabs/common/habanalabs_ioctl.c +25 −2
  4. drivers/misc/habanalabs/gaudi/gaudi.c +18 −4
  5. drivers/misc/habanalabs/goya/goya.c +21 −4
  6. include/uapi/misc/habanalabs.h +14 −2
drivers/misc/habanalabs/common/device.c+3−0
@@ -455,6 +455,7 @@ static int device_early_init(struct hl_device *hdev) · 454 unmodified lines
455455 INIT_LIST_HEAD(&hdev->fpriv_list);
456456 mutex_init(&hdev->fpriv_list_lock);
457457 atomic_set(&hdev->in_reset, 0);
458+ mutex_init(&hdev->clk_throttling.lock);
458459
459460 return 0;
460461
@@ -495,6 +496,8 @@ static void device_early_fini(struct hl_device *hdev) · 34 unmodified lines
495496
496497 mutex_destroy(&hdev->fpriv_list_lock);
497498
499+ mutex_destroy(&hdev->clk_throttling.lock);
500+
498501 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
499502
500503 kfree(hdev->hl_chip_info);
drivers/misc/habanalabs/common/habanalabs.h+29−2
@@ -2378,6 +2378,32 @@ struct multi_cs_data { · 2377 unmodified lines
23782378 u8 update_ts;
23792379 };
23802380
2381+/**
2382+ * struct hl_clk_throttle_timestamp - current/last clock throttling timestamp
2383+ * @start: timestamp taken when 'start' event is received in driver
2384+ * @end: timestamp taken when 'end' event is received in driver
2385+ */
2386+struct hl_clk_throttle_timestamp {
2387+ ktime_t start;
2388+ ktime_t end;
2389+};
2390+
2391+/**
2392+ * struct hl_clk_throttle - keeps current/last clock throttling timestamps
2393+ * @timestamp: timestamp taken by driver and firmware, index 0 refers to POWER
2394+ * index 1 refers to THERMAL
2395+ * @lock: protects this structure as it can be accessed from both event queue
2396+ * context and info_ioctl context
2397+ * @current_reason: bitmask represents the current clk throttling reasons
2398+ * @aggregated_reason: bitmask represents aggregated clk throttling reasons since driver load
2399+ */
2400+struct hl_clk_throttle {
2401+ struct hl_clk_throttle_timestamp timestamp[HL_CLK_THROTTLE_TYPE_MAX];
2402+ struct mutex lock;
2403+ u32 current_reason;
2404+ u32 aggregated_reason;
2405+};
2406+
23812407 /**
23822408 * struct hl_device - habanalabs device structure.
23832409 * @pdev: pointer to PCI device, can be NULL in case of simulator device.
@@ -2445,6 +2471,7 @@ struct multi_cs_data { · 61 unmodified lines
24452471 * @pci_mem_region: array of memory regions in the PCI
24462472 * @state_dump_specs: constants and dictionaries needed to dump system state.
24472473 * @multi_cs_completion: array of multi-CS completion.
2474+ * @clk_throttling: holds information about current/previous clock throttling events
24482475 * @dram_used_mem: current DRAM memory consumption.
24492476 * @timeout_jiffies: device CS timeout value.
24502477 * @max_power: the max power of the device, as configured by the sysadmin. This
@@ -2474,7 +2501,6 @@ struct multi_cs_data { · 23 unmodified lines
24742501 * @high_pll: high PLL profile frequency.
24752502 * @soft_reset_cnt: number of soft reset since the driver was loaded.
24762503 * @hard_reset_cnt: number of hard reset since the driver was loaded.
2477− * @clk_throttling_reason: bitmask represents the current clk throttling reasons
24782504 * @id: device minor.
24792505 * @id_control: minor of the control device
24802506 * @cpu_pci_msb_addr: 50-bit extension bits for the device CPU's 40-bit
@@ -2604,6 +2630,8 @@ struct hl_device { · 123 unmodified lines
26042630
26052631 struct multi_cs_completion multi_cs_completion[
26062632 MULTI_CS_MAX_USER_CTX];
2633+ struct hl_clk_throttle clk_throttling;
2634+
26072635 u32 *stream_master_qid_arr;
26082636 atomic64_t dram_used_mem;
26092637 u64 timeout_jiffies;
@@ -2622,7 +2650,6 @@ struct hl_device { · 12 unmodified lines
26222650 u32 high_pll;
26232651 u32 soft_reset_cnt;
26242652 u32 hard_reset_cnt;
2625− u32 clk_throttling_reason;
26262653 u16 id;
26272654 u16 id_control;
26282655 u16 cpu_pci_msb_addr;
drivers/misc/habanalabs/common/habanalabs_ioctl.c+25−2
@@ -313,15 +313,38 @@ static int pci_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args) · 312 unmodified lines
313313
314314 static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
315315 {
316+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
316317 struct hl_device *hdev = hpriv->hdev;
317318 struct hl_info_clk_throttle clk_throttle = {0};
319+ ktime_t end_time, zero_time = ktime_set(0, 0);
318320 u32 max_size = args->return_size;
319− void __user *out = (void __user *) (uintptr_t) args->return_pointer;
321+ int i;
320322
321323 if ((!max_size) || (!out))
322324 return -EINVAL;
323325
324− clk_throttle.clk_throttling_reason = hdev->clk_throttling_reason;
326+ mutex_lock(&hdev->clk_throttling.lock);
327+
328+ clk_throttle.clk_throttling_reason = hdev->clk_throttling.current_reason;
329+
330+ for (i = 0 ; i < HL_CLK_THROTTLE_TYPE_MAX ; i++) {
331+ if (!(hdev->clk_throttling.aggregated_reason & BIT(i)))
332+ continue;
333+
334+ clk_throttle.clk_throttling_timestamp_us[i] =
335+ ktime_to_us(hdev->clk_throttling.timestamp[i].start);
336+
337+ if (ktime_compare(hdev->clk_throttling.timestamp[i].end, zero_time))
338+ end_time = ktime_get();
339+ else
340+ end_time = hdev->clk_throttling.timestamp[i].end;
341+
342+ clk_throttle.clk_throttling_duration_ns[i] =
343+ ktime_to_ns(ktime_sub(end_time,
344+ hdev->clk_throttling.timestamp[i].start));
345+
346+ }
347+ mutex_unlock(&hdev->clk_throttling.lock);
325348
326349 return copy_to_user(out, &clk_throttle,
327350 min((size_t) max_size, sizeof(clk_throttle))) ? -EFAULT : 0;
drivers/misc/habanalabs/gaudi/gaudi.c+18−4
@@ -7925,27 +7925,39 @@ static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type) · 7924 unmodified lines
79257925 static void gaudi_print_clk_change_info(struct hl_device *hdev,
79267926 u16 event_type)
79277927 {
7928+ ktime_t zero_time = ktime_set(0, 0);
7929+
7930+ mutex_lock(&hdev->clk_throttling.lock);
7931+
79287932 switch (event_type) {
79297933 case GAUDI_EVENT_FIX_POWER_ENV_S:
7930− hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER;
7934+ hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
7935+ hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
7936+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
7937+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
79317938 dev_info_ratelimited(hdev->dev,
79327939 "Clock throttling due to power consumption\n");
79337940 break;
79347941
79357942 case GAUDI_EVENT_FIX_POWER_ENV_E:
7936− hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER;
7943+ hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
7944+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
79377945 dev_info_ratelimited(hdev->dev,
79387946 "Power envelop is safe, back to optimal clock\n");
79397947 break;
79407948
79417949 case GAUDI_EVENT_FIX_THERMAL_ENV_S:
7942− hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL;
7950+ hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
7951+ hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
7952+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
7953+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
79437954 dev_info_ratelimited(hdev->dev,
79447955 "Clock throttling due to overheating\n");
79457956 break;
79467957
79477958 case GAUDI_EVENT_FIX_THERMAL_ENV_E:
7948− hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL;
7959+ hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
7960+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
79497961 dev_info_ratelimited(hdev->dev,
79507962 "Thermal envelop is safe, back to optimal clock\n");
79517963 break;
@@ -7955,6 +7967,8 @@ static void gaudi_print_clk_change_info(struct hl_device *hdev, · 3 unmodified lines
79557967 event_type);
79567968 break;
79577969 }
7970+
7971+ mutex_unlock(&hdev->clk_throttling.lock);
79587972 }
79597973
79607974 static void gaudi_handle_eqe(struct hl_device *hdev,
drivers/misc/habanalabs/goya/goya.c+21−4
@@ -4768,24 +4768,39 @@ static int goya_unmask_irq(struct hl_device *hdev, u16 event_type) · 4767 unmodified lines
47684768
47694769 static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type)
47704770 {
4771+ ktime_t zero_time = ktime_set(0, 0);
4772+
4773+ mutex_lock(&hdev->clk_throttling.lock);
4774+
47714775 switch (event_type) {
47724776 case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_S:
4773− hdev->clk_throttling_reason |= HL_CLK_THROTTLE_POWER;
4777+ hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;
4778+ hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;
4779+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();
4780+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;
47744781 dev_info_ratelimited(hdev->dev,
47754782 "Clock throttling due to power consumption\n");
47764783 break;
4784+
47774785 case GOYA_ASYNC_EVENT_ID_FIX_POWER_ENV_E:
4778− hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_POWER;
4786+ hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;
4787+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();
47794788 dev_info_ratelimited(hdev->dev,
47804789 "Power envelop is safe, back to optimal clock\n");
47814790 break;
4791+
47824792 case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_S:
4783− hdev->clk_throttling_reason |= HL_CLK_THROTTLE_THERMAL;
4793+ hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;
4794+ hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;
4795+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();
4796+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;
47844797 dev_info_ratelimited(hdev->dev,
47854798 "Clock throttling due to overheating\n");
47864799 break;
4800+
47874801 case GOYA_ASYNC_EVENT_ID_FIX_THERMAL_ENV_E:
4788− hdev->clk_throttling_reason &= ~HL_CLK_THROTTLE_THERMAL;
4802+ hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;
4803+ hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();
47894804 dev_info_ratelimited(hdev->dev,
47904805 "Thermal envelop is safe, back to optimal clock\n");
47914806 break;
@@ -4795,6 +4810,8 @@ static void goya_print_clk_change_info(struct hl_device *hdev, u16 event_type) · 3 unmodified lines
47954810 event_type);
47964811 break;
47974812 }
4813+
4814+ mutex_unlock(&hdev->clk_throttling.lock);
47984815 }
47994816
48004817 void goya_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)
include/uapi/misc/habanalabs.h+14−2
@@ -473,15 +473,27 @@ struct hl_info_pci_counters { · 472 unmodified lines
473473 __u64 replay_cnt;
474474 };
475475
476−#define HL_CLK_THROTTLE_POWER 0x1
477−#define HL_CLK_THROTTLE_THERMAL 0x2
476+enum hl_clk_throttling_type {
477+ HL_CLK_THROTTLE_TYPE_POWER,
478+ HL_CLK_THROTTLE_TYPE_THERMAL,
479+ HL_CLK_THROTTLE_TYPE_MAX
480+};
481+
482+/* clk_throttling_reason masks */
483+#define HL_CLK_THROTTLE_POWER (1 << HL_CLK_THROTTLE_TYPE_POWER)
484+#define HL_CLK_THROTTLE_THERMAL (1 << HL_CLK_THROTTLE_TYPE_THERMAL)
478485
479486 /**
480487 * struct hl_info_clk_throttle - clock throttling reason
481488 * @clk_throttling_reason: each bit represents a clk throttling reason
489+ * @clk_throttling_timestamp_us: represents CPU timestamp in microseconds of the start-event
490+ * @clk_throttling_duration_ns: the clock throttle time in nanosec
482491 */
483492 struct hl_info_clk_throttle {
484493 __u32 clk_throttling_reason;
494+ __u32 pad;
495+ __u64 clk_throttling_timestamp_us[HL_CLK_THROTTLE_TYPE_MAX];
496+ __u64 clk_throttling_duration_ns[HL_CLK_THROTTLE_TYPE_MAX];
485497 };
486498
487499 /**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t