2015-08-12 22:43:39 +08:00
|
|
|
/*
|
|
|
|
* Copyright © 2014 Intel Corporation
|
|
|
|
*
|
|
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
|
|
* to deal in the Software without restriction, including without limitation
|
|
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
|
|
*
|
|
|
|
* The above copyright notice and this permission notice (including the next
|
|
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
|
|
* Software.
|
|
|
|
*
|
|
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
|
|
|
* IN THE SOFTWARE.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2017-10-05 02:13:40 +08:00
|
|
|
#include <linux/circ_buf.h>
|
2017-03-16 20:56:18 +08:00
|
|
|
#include <trace/events/dma_fence.h>
|
|
|
|
|
2017-11-16 21:32:41 +08:00
|
|
|
#include "intel_guc_submission.h"
|
2018-02-27 00:37:59 +08:00
|
|
|
#include "intel_lrc_reg.h"
|
2017-10-05 02:13:40 +08:00
|
|
|
#include "i915_drv.h"
|
|
|
|
|
2018-02-27 00:37:59 +08:00
|
|
|
#define GUC_PREEMPT_FINISHED 0x1
|
|
|
|
#define GUC_PREEMPT_BREADCRUMB_DWORDS 0x8
|
|
|
|
#define GUC_PREEMPT_BREADCRUMB_BYTES \
|
|
|
|
(sizeof(u32) * GUC_PREEMPT_BREADCRUMB_DWORDS)
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/**
|
2015-10-20 07:10:54 +08:00
|
|
|
* DOC: GuC-based command submission
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
2017-03-23 01:39:50 +08:00
|
|
|
* GuC client:
|
2017-11-16 21:32:40 +08:00
|
|
|
* A intel_guc_client refers to a submission path through GuC. Currently, there
|
2017-10-26 22:17:37 +08:00
|
|
|
* are two clients. One of them (the execbuf_client) is charged with all
|
|
|
|
* submissions to the GuC, the other one (preempt_client) is responsible for
|
|
|
|
* preempting the execbuf_client. This struct is the owner of a doorbell, a
|
|
|
|
* process descriptor and a workqueue (all of them inside a single gem object
|
|
|
|
* that contains all required pages for these elements).
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
2017-03-23 01:39:53 +08:00
|
|
|
* GuC stage descriptor:
|
2017-03-23 01:39:50 +08:00
|
|
|
* During initialization, the driver allocates a static pool of 1024 such
|
|
|
|
* descriptors, and shares them with the GuC.
|
2017-11-16 21:32:40 +08:00
|
|
|
* Currently, there exists a 1:1 mapping between a intel_guc_client and a
|
2017-03-23 01:39:53 +08:00
|
|
|
* guc_stage_desc (via the client's stage_id), so effectively only one
|
|
|
|
* gets used. This stage descriptor lets the GuC know about the doorbell,
|
|
|
|
* workqueue and process descriptor. Theoretically, it also lets the GuC
|
|
|
|
* know about our HW contexts (context ID, etc...), but we actually
|
2017-03-23 01:39:50 +08:00
|
|
|
* employ a kind of submission where the GuC uses the LRCA sent via the work
|
2017-03-23 01:39:53 +08:00
|
|
|
* item instead (the single guc_stage_desc associated to execbuf client
|
2017-03-23 01:39:50 +08:00
|
|
|
* contains information about the default kernel context only, but this is
|
|
|
|
* essentially unused). This is called a "proxy" submission.
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
|
|
|
* The Scratch registers:
|
|
|
|
* There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
|
|
|
|
* a value to the action register (SOFT_SCRATCH_0) along with any data. It then
|
|
|
|
* triggers an interrupt on the GuC via another register write (0xC4C8).
|
|
|
|
* Firmware writes a success/fail code back to the action register after
|
|
|
|
* processes the request. The kernel driver polls waiting for this update and
|
|
|
|
* then proceeds.
|
2016-11-26 01:59:35 +08:00
|
|
|
* See intel_guc_send()
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
|
|
|
* Doorbells:
|
|
|
|
* Doorbells are interrupts to uKernel. A doorbell is a single cache line (QW)
|
|
|
|
* mapped into process space.
|
|
|
|
*
|
|
|
|
* Work Items:
|
|
|
|
* There are several types of work items that the host may place into a
|
|
|
|
* workqueue, each with its own requirements and limitations. Currently only
|
|
|
|
* WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
|
|
|
|
* represents in-order queue. The kernel driver packs ring tail pointer and an
|
|
|
|
* ELSP context descriptor dword into Work Item.
|
2017-10-26 04:00:14 +08:00
|
|
|
* See guc_add_request()
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
2018-02-22 22:22:29 +08:00
|
|
|
static inline struct i915_priolist *to_priolist(struct rb_node *rb)
|
|
|
|
{
|
|
|
|
return rb_entry(rb, struct i915_priolist, node);
|
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static inline bool is_high_priority(struct intel_guc_client *client)
|
2017-03-23 01:39:44 +08:00
|
|
|
{
|
2017-10-26 22:17:37 +08:00
|
|
|
return (client->priority == GUC_CLIENT_PRIORITY_KMD_HIGH ||
|
|
|
|
client->priority == GUC_CLIENT_PRIORITY_HIGH);
|
2017-03-23 01:39:44 +08:00
|
|
|
}
|
|
|
|
|
2017-12-14 06:13:50 +08:00
|
|
|
static int reserve_doorbell(struct intel_guc_client *client)
|
2017-03-23 01:39:44 +08:00
|
|
|
{
|
|
|
|
unsigned long offset;
|
|
|
|
unsigned long end;
|
|
|
|
u16 id;
|
|
|
|
|
|
|
|
GEM_BUG_ON(client->doorbell_id != GUC_DOORBELL_INVALID);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The bitmap tracks which doorbell registers are currently in use.
|
|
|
|
* It is split into two halves; the first half is used for normal
|
|
|
|
* priority contexts, the second half for high-priority ones.
|
|
|
|
*/
|
|
|
|
offset = 0;
|
2017-11-16 21:32:41 +08:00
|
|
|
end = GUC_NUM_DOORBELLS / 2;
|
2017-03-23 01:39:44 +08:00
|
|
|
if (is_high_priority(client)) {
|
|
|
|
offset = end;
|
|
|
|
end += offset;
|
|
|
|
}
|
|
|
|
|
2017-05-31 08:05:46 +08:00
|
|
|
id = find_next_zero_bit(client->guc->doorbell_bitmap, end, offset);
|
2017-03-23 01:39:44 +08:00
|
|
|
if (id == end)
|
|
|
|
return -ENOSPC;
|
|
|
|
|
|
|
|
__set_bit(id, client->guc->doorbell_bitmap);
|
|
|
|
client->doorbell_id = id;
|
|
|
|
DRM_DEBUG_DRIVER("client %u (high prio=%s) reserved doorbell: %d\n",
|
2017-03-23 01:39:53 +08:00
|
|
|
client->stage_id, yesno(is_high_priority(client)),
|
2017-03-23 01:39:44 +08:00
|
|
|
id);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-05-01 15:52:03 +08:00
|
|
|
static bool has_doorbell(struct intel_guc_client *client)
|
|
|
|
{
|
|
|
|
if (client->doorbell_id == GUC_DOORBELL_INVALID)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return test_bit(client->doorbell_id, client->guc->doorbell_bitmap);
|
|
|
|
}
|
|
|
|
|
2017-12-14 06:13:50 +08:00
|
|
|
static void unreserve_doorbell(struct intel_guc_client *client)
|
2017-03-23 01:39:44 +08:00
|
|
|
{
|
2018-05-01 15:52:03 +08:00
|
|
|
GEM_BUG_ON(!has_doorbell(client));
|
2017-03-23 01:39:44 +08:00
|
|
|
|
|
|
|
__clear_bit(client->doorbell_id, client->guc->doorbell_bitmap);
|
|
|
|
client->doorbell_id = GUC_DOORBELL_INVALID;
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
|
|
|
* Tell the GuC to allocate or deallocate a specific doorbell
|
|
|
|
*/
|
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
static int __guc_allocate_doorbell(struct intel_guc *guc, u32 stage_id)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2016-11-26 01:59:35 +08:00
|
|
|
u32 action[] = {
|
|
|
|
INTEL_GUC_ACTION_ALLOCATE_DOORBELL,
|
2017-03-23 01:39:53 +08:00
|
|
|
stage_id
|
2016-11-26 01:59:35 +08:00
|
|
|
};
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2016-11-26 01:59:35 +08:00
|
|
|
return intel_guc_send(guc, action, ARRAY_SIZE(action));
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
static int __guc_deallocate_doorbell(struct intel_guc *guc, u32 stage_id)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2016-11-26 01:59:35 +08:00
|
|
|
u32 action[] = {
|
|
|
|
INTEL_GUC_ACTION_DEALLOCATE_DOORBELL,
|
2017-03-23 01:39:53 +08:00
|
|
|
stage_id
|
2016-11-26 01:59:35 +08:00
|
|
|
};
|
2016-10-13 00:24:41 +08:00
|
|
|
|
2016-11-26 01:59:35 +08:00
|
|
|
return intel_guc_send(guc, action, ARRAY_SIZE(action));
|
2016-10-13 00:24:41 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static struct guc_stage_desc *__get_stage_desc(struct intel_guc_client *client)
|
2017-03-23 01:39:45 +08:00
|
|
|
{
|
2017-03-23 01:39:53 +08:00
|
|
|
struct guc_stage_desc *base = client->guc->stage_desc_pool_vaddr;
|
2017-03-23 01:39:45 +08:00
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
return &base[client->stage_id];
|
2017-03-23 01:39:45 +08:00
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
|
|
|
* Initialise, update, or clear doorbell data shared with the GuC
|
|
|
|
*
|
|
|
|
* These functions modify shared data and so need access to the mapped
|
|
|
|
* client object which contains the page being used for the doorbell
|
|
|
|
*/
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static void __update_doorbell_desc(struct intel_guc_client *client, u16 new_id)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-03-23 01:39:53 +08:00
|
|
|
struct guc_stage_desc *desc;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2016-06-14 00:57:32 +08:00
|
|
|
/* Update the GuC's idea of the doorbell ID */
|
2017-03-23 01:39:53 +08:00
|
|
|
desc = __get_stage_desc(client);
|
2017-03-23 01:39:45 +08:00
|
|
|
desc->db_id = new_id;
|
2017-03-23 01:39:44 +08:00
|
|
|
}
|
2016-06-14 00:57:32 +08:00
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static struct guc_doorbell_info *__get_doorbell(struct intel_guc_client *client)
|
2017-03-23 01:39:44 +08:00
|
|
|
{
|
|
|
|
return client->vaddr + client->doorbell_offset;
|
|
|
|
}
|
|
|
|
|
2018-10-23 07:04:22 +08:00
|
|
|
static void __init_doorbell(struct intel_guc_client *client)
|
2017-03-23 01:39:44 +08:00
|
|
|
{
|
|
|
|
struct guc_doorbell_info *doorbell;
|
|
|
|
|
|
|
|
doorbell = __get_doorbell(client);
|
2016-06-14 00:57:32 +08:00
|
|
|
doorbell->db_status = GUC_DOORBELL_ENABLED;
|
2017-09-14 18:51:23 +08:00
|
|
|
doorbell->cookie = 0;
|
2016-06-14 00:57:32 +08:00
|
|
|
}
|
|
|
|
|
2018-10-23 07:04:22 +08:00
|
|
|
static void __fini_doorbell(struct intel_guc_client *client)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-03-23 01:39:51 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(client->guc);
|
2017-03-23 01:39:44 +08:00
|
|
|
struct guc_doorbell_info *doorbell;
|
2017-03-23 01:39:51 +08:00
|
|
|
u16 db_id = client->doorbell_id;
|
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
doorbell = __get_doorbell(client);
|
|
|
|
doorbell->db_status = GUC_DOORBELL_DISABLED;
|
|
|
|
|
2017-03-23 01:39:51 +08:00
|
|
|
/* Doorbell release flow requires that we wait for GEN8_DRB_VALID bit
|
|
|
|
* to go to zero after updating db_status before we call the GuC to
|
2017-11-16 21:32:41 +08:00
|
|
|
* release the doorbell
|
|
|
|
*/
|
2017-03-23 01:39:51 +08:00
|
|
|
if (wait_for_us(!(I915_READ(GEN8_DRBREGL(db_id)) & GEN8_DRB_VALID), 10))
|
|
|
|
WARN_ONCE(true, "Doorbell never became invalid after disable\n");
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static int create_doorbell(struct intel_guc_client *client)
|
2017-03-23 01:39:52 +08:00
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
2018-05-01 15:52:03 +08:00
|
|
|
if (WARN_ON(!has_doorbell(client)))
|
|
|
|
return -ENODEV; /* internal setup error, should never happen */
|
|
|
|
|
2017-03-23 01:39:52 +08:00
|
|
|
__update_doorbell_desc(client, client->doorbell_id);
|
2018-10-23 07:04:22 +08:00
|
|
|
__init_doorbell(client);
|
2017-03-23 01:39:52 +08:00
|
|
|
|
2017-12-14 06:13:50 +08:00
|
|
|
ret = __guc_allocate_doorbell(client->guc, client->stage_id);
|
|
|
|
if (ret) {
|
2018-10-23 07:04:22 +08:00
|
|
|
__fini_doorbell(client);
|
2017-12-14 06:13:50 +08:00
|
|
|
__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
|
2018-03-19 17:53:47 +08:00
|
|
|
DRM_DEBUG_DRIVER("Couldn't create client %u doorbell: %d\n",
|
|
|
|
client->stage_id, ret);
|
2017-12-14 06:13:50 +08:00
|
|
|
return ret;
|
|
|
|
}
|
2017-03-23 01:39:52 +08:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static int destroy_doorbell(struct intel_guc_client *client)
|
2016-06-14 00:57:33 +08:00
|
|
|
{
|
2017-12-14 06:13:50 +08:00
|
|
|
int ret;
|
2016-06-14 00:57:33 +08:00
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
GEM_BUG_ON(!has_doorbell(client));
|
|
|
|
|
2018-10-23 07:04:22 +08:00
|
|
|
__fini_doorbell(client);
|
2017-12-14 06:13:50 +08:00
|
|
|
ret = __guc_deallocate_doorbell(client->guc, client->stage_id);
|
|
|
|
if (ret)
|
|
|
|
DRM_ERROR("Couldn't destroy client %u doorbell: %d\n",
|
|
|
|
client->stage_id, ret);
|
2016-06-14 00:57:33 +08:00
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
__update_doorbell_desc(client, GUC_DOORBELL_INVALID);
|
2016-06-14 00:57:33 +08:00
|
|
|
|
2017-12-14 06:13:50 +08:00
|
|
|
return ret;
|
2017-03-23 01:39:44 +08:00
|
|
|
}
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-11-16 21:32:41 +08:00
|
|
|
static unsigned long __select_cacheline(struct intel_guc *guc)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-03-23 01:39:44 +08:00
|
|
|
unsigned long offset;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
/* Doorbell uses a single cache line within a page */
|
|
|
|
offset = offset_in_page(guc->db_cacheline);
|
|
|
|
|
|
|
|
/* Moving to next cache line to reduce contention */
|
2017-03-23 01:39:44 +08:00
|
|
|
guc->db_cacheline += cache_line_size();
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
DRM_DEBUG_DRIVER("reserved cacheline 0x%lx, next 0x%x, linesize %u\n",
|
2017-11-16 21:32:41 +08:00
|
|
|
offset, guc->db_cacheline, cache_line_size());
|
2015-08-12 22:43:41 +08:00
|
|
|
return offset;
|
|
|
|
}
|
|
|
|
|
2017-03-24 07:00:00 +08:00
|
|
|
static inline struct guc_process_desc *
|
2017-11-16 21:32:40 +08:00
|
|
|
__get_process_desc(struct intel_guc_client *client)
|
2017-03-24 07:00:00 +08:00
|
|
|
{
|
|
|
|
return client->vaddr + client->proc_desc_offset;
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
|
|
|
* Initialise the process descriptor shared with the GuC firmware.
|
|
|
|
*/
|
2018-10-03 05:54:28 +08:00
|
|
|
static void guc_proc_desc_init(struct intel_guc_client *client)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
|
|
|
struct guc_process_desc *desc;
|
|
|
|
|
2017-03-24 07:00:00 +08:00
|
|
|
desc = memset(__get_process_desc(client), 0, sizeof(*desc));
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* XXX: pDoorbell and WQVBaseAddress are pointers in process address
|
|
|
|
* space for ring3 clients (set them as in mmap_ioctl) or kernel
|
|
|
|
* space for kernel clients (map on demand instead? May make debug
|
|
|
|
* easier to have it mapped).
|
|
|
|
*/
|
|
|
|
desc->wq_base_addr = 0;
|
|
|
|
desc->db_base_addr = 0;
|
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
desc->stage_id = client->stage_id;
|
2017-09-18 17:25:35 +08:00
|
|
|
desc->wq_size_bytes = GUC_WQ_SIZE;
|
2015-08-12 22:43:41 +08:00
|
|
|
desc->wq_status = WQ_STATUS_ACTIVE;
|
|
|
|
desc->priority = client->priority;
|
|
|
|
}
|
|
|
|
|
2018-10-03 05:54:28 +08:00
|
|
|
static void guc_proc_desc_fini(struct intel_guc_client *client)
|
|
|
|
{
|
|
|
|
struct guc_process_desc *desc;
|
|
|
|
|
|
|
|
desc = __get_process_desc(client);
|
|
|
|
memset(desc, 0, sizeof(*desc));
|
|
|
|
}
|
|
|
|
|
2017-10-26 04:00:10 +08:00
|
|
|
static int guc_stage_desc_pool_create(struct intel_guc *guc)
|
|
|
|
{
|
|
|
|
struct i915_vma *vma;
|
|
|
|
void *vaddr;
|
|
|
|
|
|
|
|
vma = intel_guc_allocate_vma(guc,
|
|
|
|
PAGE_ALIGN(sizeof(struct guc_stage_desc) *
|
|
|
|
GUC_MAX_STAGE_DESCRIPTORS));
|
|
|
|
if (IS_ERR(vma))
|
|
|
|
return PTR_ERR(vma);
|
|
|
|
|
|
|
|
vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
|
|
|
|
if (IS_ERR(vaddr)) {
|
2018-07-21 20:50:37 +08:00
|
|
|
i915_vma_unpin_and_release(&vma, 0);
|
2017-10-26 04:00:10 +08:00
|
|
|
return PTR_ERR(vaddr);
|
|
|
|
}
|
|
|
|
|
|
|
|
guc->stage_desc_pool = vma;
|
|
|
|
guc->stage_desc_pool_vaddr = vaddr;
|
|
|
|
ida_init(&guc->stage_ids);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void guc_stage_desc_pool_destroy(struct intel_guc *guc)
|
|
|
|
{
|
|
|
|
ida_destroy(&guc->stage_ids);
|
2018-07-21 20:50:37 +08:00
|
|
|
i915_vma_unpin_and_release(&guc->stage_desc_pool, I915_VMA_RELEASE_MAP);
|
2017-10-26 04:00:10 +08:00
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
2017-03-23 01:39:53 +08:00
|
|
|
* Initialise/clear the stage descriptor shared with the GuC firmware.
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
|
|
|
* This descriptor tells the GuC where (in GGTT space) to find the important
|
|
|
|
* data structures relating to this client (doorbell, process descriptor,
|
|
|
|
* write queue, etc).
|
|
|
|
*/
|
2018-10-03 05:54:28 +08:00
|
|
|
static void guc_stage_desc_init(struct intel_guc_client *client)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2018-10-03 05:54:28 +08:00
|
|
|
struct intel_guc *guc = client->guc;
|
2016-01-24 03:58:14 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
2016-03-16 19:00:36 +08:00
|
|
|
struct intel_engine_cs *engine;
|
2016-05-24 21:53:34 +08:00
|
|
|
struct i915_gem_context *ctx = client->owner;
|
2017-03-23 01:39:53 +08:00
|
|
|
struct guc_stage_desc *desc;
|
2016-08-27 15:54:01 +08:00
|
|
|
unsigned int tmp;
|
2016-04-19 23:08:36 +08:00
|
|
|
u32 gfx_addr;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
desc = __get_stage_desc(client);
|
2017-03-23 01:39:45 +08:00
|
|
|
memset(desc, 0, sizeof(*desc));
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-11-16 21:32:41 +08:00
|
|
|
desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE |
|
|
|
|
GUC_STAGE_DESC_ATTR_KERNEL;
|
2017-10-26 22:17:37 +08:00
|
|
|
if (is_high_priority(client))
|
|
|
|
desc->attribute |= GUC_STAGE_DESC_ATTR_PREEMPT;
|
2017-03-23 01:39:53 +08:00
|
|
|
desc->stage_id = client->stage_id;
|
2017-03-23 01:39:45 +08:00
|
|
|
desc->priority = client->priority;
|
|
|
|
desc->db_id = client->doorbell_id;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2016-08-27 15:54:01 +08:00
|
|
|
for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
|
2018-04-30 21:15:01 +08:00
|
|
|
struct intel_context *ce = to_intel_context(ctx, engine);
|
2017-10-06 16:49:40 +08:00
|
|
|
u32 guc_engine_id = engine->guc_id;
|
2017-03-23 01:39:45 +08:00
|
|
|
struct guc_execlist_context *lrc = &desc->lrc[guc_engine_id];
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
|
|
|
|
/* TODO: We have a design issue to be solved here. Only when we
|
|
|
|
* receive the first batch, we know which engine is used by the
|
|
|
|
* user. But here GuC expects the lrc and ring to be pinned. It
|
|
|
|
* is not an issue for default context, which is the only one
|
|
|
|
* for now who owns a GuC client. But for future owner of GuC
|
|
|
|
* client, need to make sure lrc is pinned prior to enter here.
|
|
|
|
*/
|
2016-05-24 21:53:37 +08:00
|
|
|
if (!ce->state)
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
break; /* XXX: continue? */
|
|
|
|
|
2017-03-23 01:39:50 +08:00
|
|
|
/*
|
2017-03-23 01:39:53 +08:00
|
|
|
* XXX: When this is a GUC_STAGE_DESC_ATTR_KERNEL client (proxy
|
2017-03-23 01:39:50 +08:00
|
|
|
* submission or, in other words, not using a direct submission
|
|
|
|
* model) the KMD's LRCA is not used for any work submission.
|
|
|
|
* Instead, the GuC uses the LRCA of the user mode context (see
|
2017-10-26 04:00:14 +08:00
|
|
|
* guc_add_request below).
|
2017-03-23 01:39:50 +08:00
|
|
|
*/
|
2016-05-24 21:53:37 +08:00
|
|
|
lrc->context_desc = lower_32_bits(ce->lrc_desc);
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
|
|
|
|
/* The state page is after PPHWSP */
|
2018-03-14 08:32:49 +08:00
|
|
|
lrc->ring_lrca = intel_guc_ggtt_offset(guc, ce->state) +
|
|
|
|
LRC_STATE_PN * PAGE_SIZE;
|
2017-03-23 01:39:53 +08:00
|
|
|
|
|
|
|
/* XXX: In direct submission, the GuC wants the HW context id
|
2017-11-16 21:32:41 +08:00
|
|
|
* here. In proxy submission, it wants the stage id
|
|
|
|
*/
|
2017-03-23 01:39:53 +08:00
|
|
|
lrc->context_id = (client->stage_id << GUC_ELC_CTXID_OFFSET) |
|
2016-08-09 22:19:22 +08:00
|
|
|
(guc_engine_id << GUC_ELC_ENGINE_OFFSET);
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
|
2018-03-14 08:32:49 +08:00
|
|
|
lrc->ring_begin = intel_guc_ggtt_offset(guc, ce->ring->vma);
|
2016-08-15 17:48:57 +08:00
|
|
|
lrc->ring_end = lrc->ring_begin + ce->ring->size - 1;
|
|
|
|
lrc->ring_next_free_location = lrc->ring_begin;
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
lrc->ring_current_tail_pointer_value = 0;
|
|
|
|
|
2017-03-23 01:39:45 +08:00
|
|
|
desc->engines_used |= (1 << guc_engine_id);
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
}
|
|
|
|
|
2016-08-09 22:19:21 +08:00
|
|
|
DRM_DEBUG_DRIVER("Host engines 0x%x => GuC engines used 0x%x\n",
|
2017-11-16 21:32:41 +08:00
|
|
|
client->engines, desc->engines_used);
|
2017-03-23 01:39:45 +08:00
|
|
|
WARN_ON(desc->engines_used == 0);
|
drm/i915: Integrate GuC-based command submission
GuC-based submission is mostly the same as execlist mode, up to
intel_logical_ring_advance_and_submit(), where the context being
dispatched would be added to the execlist queue; at this point
we submit the context to the GuC backend instead.
There are, however, a few other changes also required, notably:
1. Contexts must be pinned at GGTT addresses accessible by the GuC
i.e. NOT in the range [0..WOPCM_SIZE), so we have to add the
PIN_OFFSET_BIAS flag to the relevant GGTT-pinning calls.
2. The GuC's TLB must be invalidated after a context is pinned at
a new GGTT address.
3. GuC firmware uses the one page before Ring Context as shared data.
Therefore, whenever driver wants to get base address of LRC, we
will offset one page for it. LRC_PPHWSP_PN is defined as the page
number of LRCA.
4. In the work queue used to pass requests to the GuC, the GuC
firmware requires the ring-tail-offset to be represented as an
11-bit value, expressed in QWords. Therefore, the ringbuffer
size must be reduced to the representable range (4 pages).
v2:
Defer adding #defines until needed [Chris Wilson]
Rationalise type declarations [Chris Wilson]
v4:
Squashed kerneldoc patch into here [Daniel Vetter]
v5:
Update request->tail in code common to both GuC and execlist modes.
Add a private version of lr_context_update(), as sharing the
execlist version leads to race conditions when the CPU and
the GuC both update TAIL in the context image.
Conversion of error-captured HWS page to string must account
for offset from start of object to actual HWS (LRC_PPHWSP_PN).
Issue: VIZ-4884
Signed-off-by: Alex Dai <yu.dai@intel.com>
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tom O'Rourke <Tom.O'Rourke@intel.com>
Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
2015-08-12 22:43:43 +08:00
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
2016-04-19 23:08:36 +08:00
|
|
|
* The doorbell, process descriptor, and workqueue are all parts
|
|
|
|
* of the client object, which the GuC will reference via the GGTT
|
2015-08-12 22:43:41 +08:00
|
|
|
*/
|
2018-03-14 08:32:49 +08:00
|
|
|
gfx_addr = intel_guc_ggtt_offset(guc, client->vma);
|
2017-03-23 01:39:45 +08:00
|
|
|
desc->db_trigger_phy = sg_dma_address(client->vma->pages->sgl) +
|
2016-04-19 23:08:36 +08:00
|
|
|
client->doorbell_offset;
|
2017-10-06 21:08:44 +08:00
|
|
|
desc->db_trigger_cpu = ptr_to_u64(__get_doorbell(client));
|
2017-03-23 01:39:45 +08:00
|
|
|
desc->db_trigger_uk = gfx_addr + client->doorbell_offset;
|
|
|
|
desc->process_desc = gfx_addr + client->proc_desc_offset;
|
2017-09-18 17:25:35 +08:00
|
|
|
desc->wq_addr = gfx_addr + GUC_DB_SIZE;
|
|
|
|
desc->wq_size = GUC_WQ_SIZE;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-10-06 21:08:44 +08:00
|
|
|
desc->desc_private = ptr_to_u64(client);
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2018-10-03 05:54:28 +08:00
|
|
|
static void guc_stage_desc_fini(struct intel_guc_client *client)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-03-23 01:39:53 +08:00
|
|
|
struct guc_stage_desc *desc;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
desc = __get_stage_desc(client);
|
2017-03-23 01:39:45 +08:00
|
|
|
memset(desc, 0, sizeof(*desc));
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2016-09-13 04:19:37 +08:00
|
|
|
/* Construct a Work Item and append it to the GuC's Work Queue */
|
2017-11-16 21:32:40 +08:00
|
|
|
static void guc_wq_item_append(struct intel_guc_client *client,
|
2017-10-26 04:00:14 +08:00
|
|
|
u32 target_engine, u32 context_desc,
|
|
|
|
u32 ring_tail, u32 fence_id)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2016-05-13 22:36:34 +08:00
|
|
|
/* wqi_len is in DWords, and does not include the one-word header */
|
|
|
|
const size_t wqi_size = sizeof(struct guc_wq_item);
|
2017-09-13 05:36:37 +08:00
|
|
|
const u32 wqi_len = wqi_size / sizeof(u32) - 1;
|
2017-03-24 07:00:00 +08:00
|
|
|
struct guc_process_desc *desc = __get_process_desc(client);
|
2015-08-12 22:43:41 +08:00
|
|
|
struct guc_wq_item *wqi;
|
2017-10-26 04:00:14 +08:00
|
|
|
u32 wq_off;
|
2015-12-17 03:45:55 +08:00
|
|
|
|
2017-09-18 17:25:35 +08:00
|
|
|
lockdep_assert_held(&client->wq_lock);
|
2016-05-13 22:36:34 +08:00
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
|
|
|
|
* should not have the case where structure wqi is across page, neither
|
|
|
|
* wrapped to the beginning. This simplifies the implementation below.
|
|
|
|
*
|
|
|
|
* XXX: if not the case, we need save data to a temp wqi and copy it to
|
|
|
|
* workqueue buffer dw by dw.
|
|
|
|
*/
|
2016-05-13 22:36:34 +08:00
|
|
|
BUILD_BUG_ON(wqi_size != 16);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2018-08-28 06:36:14 +08:00
|
|
|
/* We expect the WQ to be active if we're appending items to it */
|
|
|
|
GEM_BUG_ON(desc->wq_status != WQ_STATUS_ACTIVE);
|
|
|
|
|
2017-09-18 17:25:35 +08:00
|
|
|
/* Free space is guaranteed. */
|
|
|
|
wq_off = READ_ONCE(desc->tail);
|
|
|
|
GEM_BUG_ON(CIRC_SPACE(wq_off, READ_ONCE(desc->head),
|
|
|
|
GUC_WQ_SIZE) < wqi_size);
|
2016-09-09 21:11:57 +08:00
|
|
|
GEM_BUG_ON(wq_off & (wqi_size - 1));
|
2016-05-13 22:36:34 +08:00
|
|
|
|
|
|
|
/* WQ starts from the page after doorbell / process_desc */
|
2016-12-16 03:53:21 +08:00
|
|
|
wqi = client->vaddr + wq_off + GUC_DB_SIZE;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2018-08-28 06:36:14 +08:00
|
|
|
if (I915_SELFTEST_ONLY(client->use_nop_wqi)) {
|
|
|
|
wqi->header = WQ_TYPE_NOOP | (wqi_len << WQ_LEN_SHIFT);
|
|
|
|
} else {
|
|
|
|
/* Now fill in the 4-word work queue item */
|
|
|
|
wqi->header = WQ_TYPE_INORDER |
|
|
|
|
(wqi_len << WQ_LEN_SHIFT) |
|
|
|
|
(target_engine << WQ_TARGET_SHIFT) |
|
|
|
|
WQ_NO_WCFLUSH_WAIT;
|
|
|
|
wqi->context_desc = context_desc;
|
|
|
|
wqi->submit_element_info = ring_tail << WQ_RING_TAIL_SHIFT;
|
|
|
|
GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
|
|
|
|
wqi->fence_id = fence_id;
|
|
|
|
}
|
2017-09-18 17:25:35 +08:00
|
|
|
|
2017-10-26 04:00:14 +08:00
|
|
|
/* Make the update visible to GuC */
|
2017-09-18 17:25:35 +08:00
|
|
|
WRITE_ONCE(desc->tail, (wq_off + wqi_size) & (GUC_WQ_SIZE - 1));
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static void guc_ring_doorbell(struct intel_guc_client *client)
|
2016-06-14 00:57:31 +08:00
|
|
|
{
|
2017-09-14 18:51:23 +08:00
|
|
|
struct guc_doorbell_info *db;
|
|
|
|
u32 cookie;
|
2016-06-14 00:57:31 +08:00
|
|
|
|
2017-09-18 17:25:35 +08:00
|
|
|
lockdep_assert_held(&client->wq_lock);
|
2016-06-14 00:57:31 +08:00
|
|
|
|
|
|
|
/* pointer of current doorbell cacheline */
|
2017-09-14 18:51:23 +08:00
|
|
|
db = __get_doorbell(client);
|
2016-06-14 00:57:31 +08:00
|
|
|
|
2017-10-26 04:00:09 +08:00
|
|
|
/*
|
|
|
|
* We're not expecting the doorbell cookie to change behind our back,
|
|
|
|
* we also need to treat 0 as a reserved value.
|
|
|
|
*/
|
2017-09-14 18:51:23 +08:00
|
|
|
cookie = READ_ONCE(db->cookie);
|
2017-10-26 04:00:09 +08:00
|
|
|
WARN_ON_ONCE(xchg(&db->cookie, cookie + 1 ?: cookie + 2) != cookie);
|
2016-06-14 00:57:31 +08:00
|
|
|
|
2017-09-14 18:51:23 +08:00
|
|
|
/* XXX: doorbell was lost and need to acquire it again */
|
|
|
|
GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED);
|
2016-06-14 00:57:31 +08:00
|
|
|
}
|
|
|
|
|
2018-02-21 17:56:36 +08:00
|
|
|
static void guc_add_request(struct intel_guc *guc, struct i915_request *rq)
|
2017-10-26 04:00:14 +08:00
|
|
|
{
|
2017-11-16 21:32:40 +08:00
|
|
|
struct intel_guc_client *client = guc->execbuf_client;
|
2017-10-26 04:00:14 +08:00
|
|
|
struct intel_engine_cs *engine = rq->engine;
|
2018-05-18 05:26:32 +08:00
|
|
|
u32 ctx_desc = lower_32_bits(rq->hw_context->lrc_desc);
|
2017-10-26 04:00:14 +08:00
|
|
|
u32 ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
|
|
|
|
|
|
|
|
spin_lock(&client->wq_lock);
|
|
|
|
|
|
|
|
guc_wq_item_append(client, engine->guc_id, ctx_desc,
|
|
|
|
ring_tail, rq->global_seqno);
|
|
|
|
guc_ring_doorbell(client);
|
|
|
|
|
|
|
|
client->submissions[engine->id] += 1;
|
|
|
|
|
|
|
|
spin_unlock(&client->wq_lock);
|
|
|
|
}
|
|
|
|
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
/*
|
|
|
|
* When we're doing submissions using regular execlists backend, writing to
|
|
|
|
* ELSP from CPU side is enough to make sure that writes to ringbuffer pages
|
|
|
|
* pinned in mappable aperture portion of GGTT are visible to command streamer.
|
|
|
|
* Writes done by GuC on our behalf are not guaranteeing such ordering,
|
|
|
|
* therefore, to ensure the flush, we're issuing a POSTING READ.
|
|
|
|
*/
|
|
|
|
static void flush_ggtt_writes(struct i915_vma *vma)
|
|
|
|
{
|
2018-06-05 23:37:58 +08:00
|
|
|
struct drm_i915_private *dev_priv = vma->vm->i915;
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
|
|
|
|
if (i915_vma_is_map_and_fenceable(vma))
|
|
|
|
POSTING_READ_FW(GUC_STATUS);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void inject_preempt_context(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct guc_preempt_work *preempt_work =
|
|
|
|
container_of(work, typeof(*preempt_work), work);
|
|
|
|
struct intel_engine_cs *engine = preempt_work->engine;
|
|
|
|
struct intel_guc *guc = container_of(preempt_work, typeof(*guc),
|
|
|
|
preempt_work[engine->id]);
|
2017-11-16 21:32:40 +08:00
|
|
|
struct intel_guc_client *client = guc->preempt_client;
|
2017-11-02 06:16:30 +08:00
|
|
|
struct guc_stage_desc *stage_desc = __get_stage_desc(client);
|
2018-09-20 04:54:32 +08:00
|
|
|
struct intel_context *ce = to_intel_context(client->owner, engine);
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
u32 data[7];
|
|
|
|
|
2018-09-20 04:54:32 +08:00
|
|
|
if (!ce->ring->emit) { /* recreate upon load/resume */
|
|
|
|
u32 addr = intel_hws_preempt_done_address(engine);
|
|
|
|
u32 *cs;
|
|
|
|
|
|
|
|
cs = ce->ring->vaddr;
|
|
|
|
if (engine->id == RCS) {
|
|
|
|
cs = gen8_emit_ggtt_write_rcs(cs,
|
|
|
|
GUC_PREEMPT_FINISHED,
|
|
|
|
addr);
|
|
|
|
} else {
|
|
|
|
cs = gen8_emit_ggtt_write(cs,
|
|
|
|
GUC_PREEMPT_FINISHED,
|
|
|
|
addr);
|
|
|
|
*cs++ = MI_NOOP;
|
|
|
|
*cs++ = MI_NOOP;
|
|
|
|
}
|
|
|
|
*cs++ = MI_USER_INTERRUPT;
|
|
|
|
*cs++ = MI_NOOP;
|
|
|
|
|
|
|
|
ce->ring->emit = GUC_PREEMPT_BREADCRUMB_BYTES;
|
|
|
|
GEM_BUG_ON((void *)cs - ce->ring->vaddr != ce->ring->emit);
|
|
|
|
|
|
|
|
flush_ggtt_writes(ce->ring->vma);
|
|
|
|
}
|
|
|
|
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
spin_lock_irq(&client->wq_lock);
|
2018-09-20 04:54:32 +08:00
|
|
|
guc_wq_item_append(client, engine->guc_id, lower_32_bits(ce->lrc_desc),
|
2018-02-27 00:37:59 +08:00
|
|
|
GUC_PREEMPT_BREADCRUMB_BYTES / sizeof(u64), 0);
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
spin_unlock_irq(&client->wq_lock);
|
|
|
|
|
2017-11-02 06:16:30 +08:00
|
|
|
/*
|
|
|
|
* If GuC firmware performs an engine reset while that engine had
|
|
|
|
* a preemption pending, it will set the terminated attribute bit
|
|
|
|
* on our preemption stage descriptor. GuC firmware retains all
|
|
|
|
* pending work items for a high-priority GuC client, unlike the
|
|
|
|
* normal-priority GuC client where work items are dropped. It
|
|
|
|
* wants to make sure the preempt-to-idle work doesn't run when
|
|
|
|
* scheduling resumes, and uses this bit to inform its scheduler
|
|
|
|
* and presumably us as well. Our job is to clear it for the next
|
|
|
|
* preemption after reset, otherwise that and future preemptions
|
|
|
|
* will never complete. We'll just clear it every time.
|
|
|
|
*/
|
|
|
|
stage_desc->attribute &= ~GUC_STAGE_DESC_ATTR_TERMINATED;
|
|
|
|
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
data[0] = INTEL_GUC_ACTION_REQUEST_PREEMPTION;
|
|
|
|
data[1] = client->stage_id;
|
|
|
|
data[2] = INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q |
|
|
|
|
INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q;
|
|
|
|
data[3] = engine->guc_id;
|
|
|
|
data[4] = guc->execbuf_client->priority;
|
|
|
|
data[5] = guc->execbuf_client->stage_id;
|
2018-03-14 08:32:49 +08:00
|
|
|
data[6] = intel_guc_ggtt_offset(guc, guc->shared_data);
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
|
|
|
|
if (WARN_ON(intel_guc_send(guc, data, ARRAY_SIZE(data)))) {
|
|
|
|
execlists_clear_active(&engine->execlists,
|
|
|
|
EXECLISTS_ACTIVE_PREEMPT);
|
2017-11-16 21:32:37 +08:00
|
|
|
tasklet_schedule(&engine->execlists.tasklet);
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We're using user interrupt and HWSP value to mark that preemption has
|
|
|
|
* finished and GPU is idle. Normally, we could unwind and continue similar to
|
|
|
|
* execlists submission path. Unfortunately, with GuC we also need to wait for
|
|
|
|
* it to finish its own postprocessing, before attempting to submit. Otherwise
|
|
|
|
* GuC may silently ignore our submissions, and thus we risk losing request at
|
|
|
|
* best, executing out-of-order and causing kernel panic at worst.
|
|
|
|
*/
|
|
|
|
#define GUC_PREEMPT_POSTPROCESS_DELAY_MS 10
|
|
|
|
static void wait_for_guc_preempt_report(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct intel_guc *guc = &engine->i915->guc;
|
|
|
|
struct guc_shared_ctx_data *data = guc->shared_data_vaddr;
|
|
|
|
struct guc_ctx_report *report =
|
|
|
|
&data->preempt_ctx_report[engine->guc_id];
|
|
|
|
|
|
|
|
WARN_ON(wait_for_atomic(report->report_return_status ==
|
|
|
|
INTEL_GUC_REPORT_STATUS_COMPLETE,
|
|
|
|
GUC_PREEMPT_POSTPROCESS_DELAY_MS));
|
|
|
|
/*
|
|
|
|
* GuC is expecting that we're also going to clear the affected context
|
|
|
|
* counter, let's also reset the return status to not depend on GuC
|
|
|
|
* resetting it after recieving another preempt action
|
|
|
|
*/
|
|
|
|
report->affected_count = 0;
|
|
|
|
report->report_return_status = INTEL_GUC_REPORT_STATUS_UNKNOWN;
|
|
|
|
}
|
|
|
|
|
2018-05-17 02:33:50 +08:00
|
|
|
static void complete_preempt_context(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct intel_engine_execlists *execlists = &engine->execlists;
|
|
|
|
|
|
|
|
GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT));
|
|
|
|
|
2018-07-16 21:21:54 +08:00
|
|
|
if (inject_preempt_hang(execlists))
|
|
|
|
return;
|
|
|
|
|
2018-05-17 02:33:50 +08:00
|
|
|
execlists_cancel_port_requests(execlists);
|
|
|
|
execlists_unwind_incomplete_requests(execlists);
|
|
|
|
|
|
|
|
wait_for_guc_preempt_report(engine);
|
|
|
|
intel_write_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX, 0);
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/**
|
2017-11-16 21:32:38 +08:00
|
|
|
* guc_submit() - Submit commands through GuC
|
2017-09-14 16:32:13 +08:00
|
|
|
* @engine: engine associated with the commands
|
2016-05-13 22:36:32 +08:00
|
|
|
*
|
|
|
|
* The only error here arises if the doorbell hardware isn't functioning
|
|
|
|
* as expected, which really shouln't happen.
|
2015-08-12 22:43:41 +08:00
|
|
|
*/
|
2017-11-16 21:32:38 +08:00
|
|
|
static void guc_submit(struct intel_engine_cs *engine)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
struct intel_guc *guc = &engine->i915->guc;
|
2017-09-22 20:43:03 +08:00
|
|
|
struct intel_engine_execlists * const execlists = &engine->execlists;
|
|
|
|
struct execlist_port *port = execlists->port;
|
2017-09-14 16:32:13 +08:00
|
|
|
unsigned int n;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-10-10 19:48:57 +08:00
|
|
|
for (n = 0; n < execlists_num_ports(execlists); n++) {
|
2018-02-21 17:56:36 +08:00
|
|
|
struct i915_request *rq;
|
2017-09-14 16:32:13 +08:00
|
|
|
unsigned int count;
|
drm/i915/guc: WA to address the Ringbuffer coherency issue
Driver accesses the ringbuffer pages, via GMADR BAR, if the pages are
pinned in mappable aperture portion of GGTT and for ringbuffer pages
allocated from Stolen memory, access can only be done through GMADR BAR.
In case of GuC based submission, updates done in ringbuffer via GMADR
may not get committed to memory by the time the Command streamer starts
reading them, resulting in fetching of stale data.
For Host based submission, such problem is not there as the write to Ring
Tail or ELSP register happens from the Host side prior to submission.
Access to any GFX register from CPU side goes to GTTMMADR BAR and Hw already
enforces the ordering between outstanding GMADR writes & new GTTMADR access.
MMIO writes from GuC side do not go to GTTMMADR BAR as GuC communication to
registers within GT is contained within GT, so ordering is not enforced
resulting in a race, which can manifest in form of a hang.
To ensure the flush of in-flight GMADR writes, a POSTING READ is done to
GuC register prior to doorbell ring.
There is already a similar WA in i915_gem_object_flush_gtt_write_domain(),
which takes care of GMADR writes from User space to GEM buffers, but not the
ringbuffer writes from KMD.
This WA is needed on all recent HW.
v2:
- Use POSTING_READ_FW instead of POSTING_READ as GuC register do not lie
in any forcewake domain range and so the overhead of spinlock & search
in the forcewake table is avoidable. (Chris)
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Sagar Arun Kamble <sagar.a.kamble@intel.com>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: http://patchwork.freedesktop.org/patch/msgid/1477413323-1880-1-git-send-email-akash.goel@intel.com
2016-10-26 00:35:23 +08:00
|
|
|
|
2017-09-14 16:32:13 +08:00
|
|
|
rq = port_unpack(&port[n], &count);
|
|
|
|
if (rq && count == 0) {
|
|
|
|
port_set(&port[n], port_pack(rq, ++count));
|
2017-02-28 19:28:03 +08:00
|
|
|
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
flush_ggtt_writes(rq->ring->vma);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-10-26 04:00:14 +08:00
|
|
|
guc_add_request(guc, rq);
|
2017-09-14 16:32:13 +08:00
|
|
|
}
|
|
|
|
}
|
drm/i915/guc: Split hw submission for replay after GPU reset
Something I missed before sending off the partial series was that the
non-scheduler guc reset path was broken (in the full series, this is
pushed to the execlists reset handler). The issue is that after a reset,
we have to refill the GuC workqueues, which we do by resubmitting the
requests. However, if we already have submitted them, the fences within
them have already been used and triggering them again is an error.
Instead, just repopulate the guc workqueue.
[ 115.858560] [IGT] gem_busy: starting subtest hang-render
[ 135.839867] [drm] GPU HANG: ecode 9:0:0xe757fefe, in gem_busy [1716], reason: Hang on render ring, action: reset
[ 135.839902] drm/i915: Resetting chip after gpu hang
[ 135.839957] [drm] RC6 on
[ 135.858351] ------------[ cut here ]------------
[ 135.858357] WARNING: CPU: 2 PID: 45 at drivers/gpu/drm/i915/i915_sw_fence.c:108 i915_sw_fence_complete+0x25/0x30
[ 135.858357] Modules linked in: rfcomm bnep binfmt_misc nls_iso8859_1 input_leds snd_hda_codec_hdmi snd_hda_codec_realtek snd_hda_codec_generic snd_hda_intel snd_hda_codec snd_hda_core btusb btrtl snd_hwdep snd_pcm 8250_dw snd_seq_midi hid_lenovo snd_seq_midi_event snd_rawmidi iwlwifi x86_pkg_temp_thermal coretemp snd_seq crct10dif_pclmul snd_seq_device hci_uart snd_timer crc32_pclmul ghash_clmulni_intel idma64 aesni_intel virt_dma btbcm snd btqca aes_x86_64 btintel lrw cfg80211 bluetooth gf128mul glue_helper ablk_helper cryptd soundcore intel_lpss_pci intel_pch_thermal intel_lpss_acpi intel_lpss acpi_als mfd_core kfifo_buf acpi_pad industrialio autofs4 hid_plantronics usbhid dm_mirror dm_region_hash dm_log sdhci_pci ahci sdhci libahci i2c_hid hid
[ 135.858389] CPU: 2 PID: 45 Comm: kworker/2:1 Tainted: G W 4.9.0-rc4+ #238
[ 135.858389] Hardware name: /NUC6i3SYB, BIOS SYSKLi35.86A.0024.2015.1027.2142 10/27/2015
[ 135.858392] Workqueue: events_long i915_hangcheck_elapsed
[ 135.858394] ffffc900001bf9b8 ffffffff812bb238 0000000000000000 0000000000000000
[ 135.858396] ffffc900001bf9f8 ffffffff8104f621 0000006c00000000 ffff8808296137f8
[ 135.858398] 0000000000000a00 ffff8808457a0000 ffff880845764e60 ffff880845760000
[ 135.858399] Call Trace:
[ 135.858403] [<ffffffff812bb238>] dump_stack+0x4d/0x65
[ 135.858405] [<ffffffff8104f621>] __warn+0xc1/0xe0
[ 135.858406] [<ffffffff8104f748>] warn_slowpath_null+0x18/0x20
[ 135.858408] [<ffffffff813f8c15>] i915_sw_fence_complete+0x25/0x30
[ 135.858410] [<ffffffff813f8fad>] i915_sw_fence_commit+0xd/0x30
[ 135.858412] [<ffffffff8142e591>] __i915_gem_request_submit+0xe1/0xf0
[ 135.858413] [<ffffffff8142e5c8>] i915_gem_request_submit+0x28/0x40
[ 135.858415] [<ffffffff814433e7>] i915_guc_submit+0x47/0x210
[ 135.858417] [<ffffffff81443e98>] i915_guc_submission_enable+0x468/0x540
[ 135.858419] [<ffffffff81442495>] intel_guc_setup+0x715/0x810
[ 135.858421] [<ffffffff8142b6b4>] i915_gem_init_hw+0x114/0x2a0
[ 135.858423] [<ffffffff813eeaa8>] i915_reset+0xe8/0x120
[ 135.858424] [<ffffffff813f3937>] i915_reset_and_wakeup+0x157/0x180
[ 135.858426] [<ffffffff813f79db>] i915_handle_error+0x1ab/0x230
[ 135.858428] [<ffffffff812c760d>] ? scnprintf+0x4d/0x90
[ 135.858430] [<ffffffff81435985>] i915_hangcheck_elapsed+0x275/0x3d0
[ 135.858432] [<ffffffff810668cf>] process_one_work+0x12f/0x410
[ 135.858433] [<ffffffff81066bf3>] worker_thread+0x43/0x4d0
[ 135.858435] [<ffffffff81066bb0>] ? process_one_work+0x410/0x410
[ 135.858436] [<ffffffff81066bb0>] ? process_one_work+0x410/0x410
[ 135.858438] [<ffffffff8106bbb4>] kthread+0xd4/0xf0
[ 135.858440] [<ffffffff8106bae0>] ? kthread_park+0x60/0x60
v2: Only resubmit submitted requests
v3: Don't forget the pending requests have reserved space.
Fixes: d55ac5bf97c6 ("drm/i915: Defer transfer onto execution timeline to actual hw submission")
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20161129121024.22650-6-chris@chris-wilson.co.uk
2016-11-29 20:10:24 +08:00
|
|
|
}
|
|
|
|
|
2018-02-21 17:56:36 +08:00
|
|
|
static void port_assign(struct execlist_port *port, struct i915_request *rq)
|
2017-05-17 20:10:00 +08:00
|
|
|
{
|
2017-11-24 21:00:29 +08:00
|
|
|
GEM_BUG_ON(port_isset(port));
|
2017-05-17 20:10:00 +08:00
|
|
|
|
2018-02-21 17:56:36 +08:00
|
|
|
port_set(port, i915_request_get(rq));
|
2017-05-17 20:10:00 +08:00
|
|
|
}
|
|
|
|
|
2018-04-04 02:35:37 +08:00
|
|
|
static inline int rq_prio(const struct i915_request *rq)
|
|
|
|
{
|
2018-04-19 02:40:52 +08:00
|
|
|
return rq->sched.attr.priority;
|
2018-04-04 02:35:37 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline int port_prio(const struct execlist_port *port)
|
|
|
|
{
|
|
|
|
return rq_prio(port_request(port));
|
|
|
|
}
|
|
|
|
|
2018-05-09 05:03:18 +08:00
|
|
|
static bool __guc_dequeue(struct intel_engine_cs *engine)
|
2017-03-16 20:56:18 +08:00
|
|
|
{
|
2017-09-22 20:43:03 +08:00
|
|
|
struct intel_engine_execlists * const execlists = &engine->execlists;
|
|
|
|
struct execlist_port *port = execlists->port;
|
2018-02-21 17:56:36 +08:00
|
|
|
struct i915_request *last = NULL;
|
2017-09-22 20:43:07 +08:00
|
|
|
const struct execlist_port * const last_port =
|
|
|
|
&execlists->port[execlists->port_mask];
|
2017-03-16 20:56:18 +08:00
|
|
|
bool submit = false;
|
2017-09-14 16:32:13 +08:00
|
|
|
struct rb_node *rb;
|
|
|
|
|
2018-05-09 05:03:18 +08:00
|
|
|
lockdep_assert_held(&engine->timeline.lock);
|
|
|
|
|
2017-11-24 21:37:44 +08:00
|
|
|
if (port_isset(port)) {
|
2018-04-04 02:35:37 +08:00
|
|
|
if (intel_engine_has_preemption(engine)) {
|
2017-11-24 21:37:44 +08:00
|
|
|
struct guc_preempt_work *preempt_work =
|
|
|
|
&engine->i915->guc.preempt_work[engine->id];
|
2018-04-04 02:35:37 +08:00
|
|
|
int prio = execlists->queue_priority;
|
2017-11-24 21:37:44 +08:00
|
|
|
|
2018-04-04 02:35:37 +08:00
|
|
|
if (__execlists_need_preempt(prio, port_prio(port))) {
|
2017-11-24 21:37:44 +08:00
|
|
|
execlists_set_active(execlists,
|
|
|
|
EXECLISTS_ACTIVE_PREEMPT);
|
|
|
|
queue_work(engine->i915->guc.preempt_wq,
|
|
|
|
&preempt_work->work);
|
2018-05-09 05:03:18 +08:00
|
|
|
return false;
|
2017-11-24 21:37:44 +08:00
|
|
|
}
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
port++;
|
2017-11-24 21:37:44 +08:00
|
|
|
if (port_isset(port))
|
2018-05-09 05:03:18 +08:00
|
|
|
return false;
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
}
|
2017-11-24 21:37:44 +08:00
|
|
|
GEM_BUG_ON(port_isset(port));
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
|
2018-06-29 15:53:20 +08:00
|
|
|
while ((rb = rb_first_cached(&execlists->queue))) {
|
2018-02-22 22:22:29 +08:00
|
|
|
struct i915_priolist *p = to_priolist(rb);
|
2018-02-21 17:56:36 +08:00
|
|
|
struct i915_request *rq, *rn;
|
2018-10-01 20:32:04 +08:00
|
|
|
int i;
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
|
2018-10-01 20:32:04 +08:00
|
|
|
priolist_for_each_request_consume(rq, rn, p, i) {
|
2018-05-18 05:26:32 +08:00
|
|
|
if (last && rq->hw_context != last->hw_context) {
|
2018-10-01 20:32:04 +08:00
|
|
|
if (port == last_port)
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
goto done;
|
|
|
|
|
2017-05-23 18:23:59 +08:00
|
|
|
if (submit)
|
|
|
|
port_assign(port, last);
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
port++;
|
|
|
|
}
|
|
|
|
|
2018-10-01 20:32:04 +08:00
|
|
|
list_del_init(&rq->sched.link);
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
|
2018-02-21 17:56:36 +08:00
|
|
|
__i915_request_submit(rq);
|
|
|
|
trace_i915_request_in(rq, port_index(port, execlists));
|
2018-10-01 20:32:04 +08:00
|
|
|
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
last = rq;
|
|
|
|
submit = true;
|
2017-03-16 20:56:18 +08:00
|
|
|
}
|
|
|
|
|
2018-06-29 15:53:20 +08:00
|
|
|
rb_erase_cached(&p->node, &execlists->queue);
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
if (p->priority != I915_PRIORITY_NORMAL)
|
2017-05-17 20:10:04 +08:00
|
|
|
kmem_cache_free(engine->i915->priorities, p);
|
2018-02-22 22:22:29 +08:00
|
|
|
}
|
drm/i915: Split execlist priority queue into rbtree + linked list
All the requests at the same priority are executed in FIFO order. They
do not need to be stored in the rbtree themselves, as they are a simple
list within a level. If we move the requests at one priority into a list,
we can then reduce the rbtree to the set of priorities. This should keep
the height of the rbtree small, as the number of active priorities can not
exceed the number of active requests and should be typically only a few.
Currently, we have ~2k possible different priority levels, that may
increase to allow even more fine grained selection. Allocating those in
advance seems a waste (and may be impossible), so we opt for allocating
upon first use, and freeing after its requests are depleted. To avoid
the possibility of an allocation failure causing us to lose a request,
we preallocate the default priority (0) and bump any request to that
priority if we fail to allocate it the appropriate plist. Having a
request (that is ready to run, so not leading to corruption) execute
out-of-order is better than leaking the request (and its dependency
tree) entirely.
There should be a benefit to reducing execlists_dequeue() to principally
using a simple list (and reducing the frequency of both rbtree iteration
and balancing on erase) but for typical workloads, request coalescing
should be small enough that we don't notice any change. The main gain is
from improving PI calls to schedule, and the explicit list within a
level should make request unwinding simpler (we just need to insert at
the head of the list rather than the tail and not have to make the
rbtree search more complicated).
v2: Avoid use-after-free when deleting a depleted priolist
v3: Michał found the solution to handling the allocation failure
gracefully. If we disable all priority scheduling following the
allocation failure, those requests will be executed in fifo and we will
ensure that this request and its dependencies are in strict fifo (even
when it doesn't realise it is only a single list). Normal scheduling is
restored once we know the device is idle, until the next failure!
Suggested-by: Michał Wajdeczko <michal.wajdeczko@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michał Winiarski <michal.winiarski@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Reviewed-by: Michał Winiarski <michal.winiarski@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/20170517121007.27224-8-chris@chris-wilson.co.uk
2017-05-17 20:10:03 +08:00
|
|
|
done:
|
2018-02-22 22:22:29 +08:00
|
|
|
execlists->queue_priority = rb ? to_priolist(rb)->priority : INT_MIN;
|
2018-05-09 05:03:18 +08:00
|
|
|
if (submit)
|
2017-05-17 20:10:00 +08:00
|
|
|
port_assign(port, last);
|
2018-05-09 05:03:18 +08:00
|
|
|
if (last)
|
2018-03-31 21:06:26 +08:00
|
|
|
execlists_user_begin(execlists, execlists->port);
|
2018-02-16 00:25:53 +08:00
|
|
|
|
|
|
|
/* We must always keep the beast fed if we have work piled up */
|
|
|
|
GEM_BUG_ON(port_isset(execlists->port) &&
|
|
|
|
!execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
|
2018-06-29 15:53:20 +08:00
|
|
|
GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
|
|
|
|
!port_isset(execlists->port));
|
2018-02-16 00:25:53 +08:00
|
|
|
|
2018-05-09 05:03:18 +08:00
|
|
|
return submit;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void guc_dequeue(struct intel_engine_cs *engine)
|
|
|
|
{
|
2018-09-25 16:31:59 +08:00
|
|
|
if (__guc_dequeue(engine))
|
2018-05-09 05:03:18 +08:00
|
|
|
guc_submit(engine);
|
2017-03-16 20:56:18 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:37 +08:00
|
|
|
static void guc_submission_tasklet(unsigned long data)
|
2017-03-16 20:56:18 +08:00
|
|
|
{
|
2017-09-22 20:43:03 +08:00
|
|
|
struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
|
2017-09-22 20:43:06 +08:00
|
|
|
struct intel_engine_execlists * const execlists = &engine->execlists;
|
|
|
|
struct execlist_port *port = execlists->port;
|
2018-02-21 17:56:36 +08:00
|
|
|
struct i915_request *rq;
|
2018-09-25 16:31:59 +08:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
spin_lock_irqsave(&engine->timeline.lock, flags);
|
2017-03-16 20:56:18 +08:00
|
|
|
|
2018-03-31 21:06:26 +08:00
|
|
|
rq = port_request(port);
|
2018-02-21 17:56:36 +08:00
|
|
|
while (rq && i915_request_completed(rq)) {
|
|
|
|
trace_i915_request_out(rq);
|
|
|
|
i915_request_put(rq);
|
2017-05-17 20:10:00 +08:00
|
|
|
|
2018-03-31 21:06:26 +08:00
|
|
|
port = execlists_port_complete(execlists, port);
|
|
|
|
if (port_isset(port)) {
|
|
|
|
execlists_user_begin(execlists, port);
|
|
|
|
rq = port_request(port);
|
|
|
|
} else {
|
|
|
|
execlists_user_end(execlists);
|
|
|
|
rq = NULL;
|
|
|
|
}
|
2017-09-14 16:32:13 +08:00
|
|
|
}
|
2017-03-16 20:56:18 +08:00
|
|
|
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
if (execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT) &&
|
|
|
|
intel_read_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX) ==
|
2018-05-17 02:33:50 +08:00
|
|
|
GUC_PREEMPT_FINISHED)
|
|
|
|
complete_preempt_context(engine);
|
drm/i915/guc: Preemption! With GuC
Pretty similar to what we have on execlists.
We're reusing most of the GEM code, however, due to GuC quirks we need a
couple of extra bits.
Preemption is implemented as GuC action, and actions can be pretty slow.
Because of that, we're using a mutex to serialize them. Since we're
requesting preemption from the tasklet, the task of creating a workitem
and wrapping it in GuC action is delegated to a worker.
To distinguish that preemption has finished, we're using additional
piece of HWSP, and since we're not getting context switch interrupts,
we're also adding a user interrupt.
The fact that our special preempt context has completed unfortunately
doesn't mean that we're ready to submit new work. We also need to wait
for GuC to finish its own processing.
v2: Don't compile out the wait for GuC, handle workqueue flush on reset,
no need for ordered workqueue, put on a reviewer hat when looking at my own
patches (Chris)
Move struct work around in intel_guc, move user interruput outside of
conditional (Michał)
Keep ring around rather than chase though intel_context
v3: Extract WA for flushing ggtt writes to a helper (Chris)
Keep work_struct in intel_guc rather than engine (Michał)
Use ordered workqueue for inject_preempt worker to avoid GuC quirks.
v4: Drop now unused INTEL_GUC_PREEMPT_OPTION_IMMEDIATE (Daniele)
Drop stray newlines, use container_of for intel_guc in worker,
check for presence of workqueue when flushing it, rather than
enable_guc_submission modparam, reorder preempt postprocessing (Chris)
v5: Make wq NULL after destroying it
v6: Swap struct guc_preempt_work members (Michał)
Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Jeff McGee <jeff.mcgee@intel.com>
Cc: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Cc: Oscar Mateo <oscar.mateo@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Link: https://patchwork.freedesktop.org/patch/msgid/20171026133558.19580-1-michal.winiarski@intel.com
2017-10-26 21:35:58 +08:00
|
|
|
|
|
|
|
if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT))
|
2017-11-16 21:32:38 +08:00
|
|
|
guc_dequeue(engine);
|
2018-09-25 16:31:59 +08:00
|
|
|
|
|
|
|
spin_unlock_irqrestore(&engine->timeline.lock, flags);
|
2017-03-16 20:56:18 +08:00
|
|
|
}
|
|
|
|
|
2018-05-17 02:33:52 +08:00
|
|
|
static struct i915_request *
|
|
|
|
guc_reset_prepare(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
struct intel_engine_execlists * const execlists = &engine->execlists;
|
|
|
|
|
|
|
|
GEM_TRACE("%s\n", engine->name);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Prevent request submission to the hardware until we have
|
|
|
|
* completed the reset in i915_gem_reset_finish(). If a request
|
|
|
|
* is completed by one engine, it may then queue a request
|
|
|
|
* to a second via its execlists->tasklet *just* as we are
|
|
|
|
* calling engine->init_hw() and also writing the ELSP.
|
|
|
|
* Turning off the execlists->tasklet until the reset is over
|
|
|
|
* prevents the race.
|
|
|
|
*/
|
|
|
|
__tasklet_disable_sync_once(&execlists->tasklet);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We're using worker to queue preemption requests from the tasklet in
|
|
|
|
* GuC submission mode.
|
|
|
|
* Even though tasklet was disabled, we may still have a worker queued.
|
|
|
|
* Let's make sure that all workers scheduled before disabling the
|
|
|
|
* tasklet are completed before continuing with the reset.
|
|
|
|
*/
|
|
|
|
if (engine->i915->guc.preempt_wq)
|
|
|
|
flush_workqueue(engine->i915->guc.preempt_wq);
|
|
|
|
|
|
|
|
return i915_gem_find_active_request(engine);
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/*
|
|
|
|
* Everything below here is concerned with setup & teardown, and is
|
|
|
|
* therefore not part of the somewhat time-critical batch-submission
|
2017-11-16 21:32:38 +08:00
|
|
|
* path of guc_submit() above.
|
2015-08-12 22:43:41 +08:00
|
|
|
*/
|
|
|
|
|
2016-08-09 22:19:20 +08:00
|
|
|
/* Check that a doorbell register is in the expected state */
|
2017-03-23 01:39:44 +08:00
|
|
|
static bool doorbell_ok(struct intel_guc *guc, u16 db_id)
|
2016-08-09 22:19:20 +08:00
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
2017-03-23 01:39:44 +08:00
|
|
|
u32 drbregl;
|
|
|
|
bool valid;
|
|
|
|
|
|
|
|
GEM_BUG_ON(db_id >= GUC_DOORBELL_INVALID);
|
|
|
|
|
|
|
|
drbregl = I915_READ(GEN8_DRBREGL(db_id));
|
|
|
|
valid = drbregl & GEN8_DRB_VALID;
|
2016-08-09 22:19:20 +08:00
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
if (test_bit(db_id, guc->doorbell_bitmap) == valid)
|
2016-08-09 22:19:20 +08:00
|
|
|
return true;
|
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
DRM_DEBUG_DRIVER("Doorbell %d has unexpected state (0x%x): valid=%s\n",
|
|
|
|
db_id, drbregl, yesno(valid));
|
2016-08-09 22:19:20 +08:00
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-12-14 06:13:52 +08:00
|
|
|
static bool guc_verify_doorbells(struct intel_guc *guc)
|
drm/i915/guc: (re)initialise doorbell h/w when enabling GuC submission
During a hibernate/resume cycle, the whole system is reset, including
the GuC and the doorbell hardware. Then the system is booted up, drivers
are loaded, etc -- the GuC firmware may be loaded and set running at
this point. But then, the booted kernel is replaced by the hibernated
image, and this resumed kernel will also try to reload the GuC firmware
(which will fail). To recover, we reset the GuC and try again (which
should work). But this GuC reset doesn't also reset the doorbell
hardware, so it can be left in a state inconsistent with that assumed
by the driver and/or the newly-loaded GuC firmware.
It would be better if the GuC reset also cleared all doorbell state,
but that's not how the hardware currently works; also, the driver cannot
directly reprogram the doorbell hardware (only the GuC can do that).
So this patch cycles through all doorbells, assigning and releasing each
in turn, so that all the doorbell hardware is left in a consistent
state, no matter how it was programmed by the previously-running kernel
and/or GuC firmware.
v2: don't use kmap_atomic() now that client page 0 is kept mapped.
Signed-off-by: Dave Gordon <david.s.gordon@intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1465837054-16245-2-git-send-email-david.s.gordon@intel.com
2016-06-14 00:57:34 +08:00
|
|
|
{
|
2017-03-23 01:39:52 +08:00
|
|
|
u16 db_id;
|
2017-12-14 06:13:52 +08:00
|
|
|
|
|
|
|
for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id)
|
|
|
|
if (!doorbell_ok(guc, db_id))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
/**
|
2017-11-16 21:32:40 +08:00
|
|
|
* guc_client_alloc() - Allocate an intel_guc_client
|
2016-06-11 01:29:25 +08:00
|
|
|
* @dev_priv: driver private data structure
|
2016-08-17 20:42:42 +08:00
|
|
|
* @engines: The set of engines to enable for this client
|
2015-08-12 22:43:41 +08:00
|
|
|
* @priority: four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
|
2017-11-16 21:32:41 +08:00
|
|
|
* The kernel client to replace ExecList submission is created with
|
|
|
|
* NORMAL priority. Priority of a client for scheduler can be HIGH,
|
|
|
|
* while a preemption context can use CRITICAL.
|
2015-10-20 07:10:54 +08:00
|
|
|
* @ctx: the context that owns the client (we use the default render
|
2017-11-16 21:32:41 +08:00
|
|
|
* context)
|
2015-08-12 22:43:41 +08:00
|
|
|
*
|
2017-11-16 21:32:40 +08:00
|
|
|
* Return: An intel_guc_client object if success, else NULL.
|
2015-08-12 22:43:41 +08:00
|
|
|
*/
|
2017-11-16 21:32:40 +08:00
|
|
|
static struct intel_guc_client *
|
2016-06-11 01:29:25 +08:00
|
|
|
guc_client_alloc(struct drm_i915_private *dev_priv,
|
2017-10-06 16:49:40 +08:00
|
|
|
u32 engines,
|
|
|
|
u32 priority,
|
2016-06-11 01:29:25 +08:00
|
|
|
struct i915_gem_context *ctx)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-11-16 21:32:40 +08:00
|
|
|
struct intel_guc_client *client;
|
2015-08-12 22:43:41 +08:00
|
|
|
struct intel_guc *guc = &dev_priv->guc;
|
2016-08-15 17:48:51 +08:00
|
|
|
struct i915_vma *vma;
|
2016-11-03 01:50:47 +08:00
|
|
|
void *vaddr;
|
2017-03-23 01:39:44 +08:00
|
|
|
int ret;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
client = kzalloc(sizeof(*client), GFP_KERNEL);
|
|
|
|
if (!client)
|
2017-03-23 01:39:44 +08:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
client->guc = guc;
|
2017-03-23 01:39:44 +08:00
|
|
|
client->owner = ctx;
|
2016-08-09 22:19:21 +08:00
|
|
|
client->engines = engines;
|
|
|
|
client->priority = priority;
|
2017-03-23 01:39:44 +08:00
|
|
|
client->doorbell_id = GUC_DOORBELL_INVALID;
|
|
|
|
spin_lock_init(&client->wq_lock);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
ret = ida_simple_get(&guc->stage_ids, 0, GUC_MAX_STAGE_DESCRIPTORS,
|
2017-11-16 21:32:41 +08:00
|
|
|
GFP_KERNEL);
|
2017-03-23 01:39:44 +08:00
|
|
|
if (ret < 0)
|
|
|
|
goto err_client;
|
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
client->stage_id = ret;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
/* The first page is doorbell/proc_desc. Two followed pages are wq. */
|
2017-01-14 01:41:57 +08:00
|
|
|
vma = intel_guc_allocate_vma(guc, GUC_DB_SIZE + GUC_WQ_SIZE);
|
2017-03-23 01:39:44 +08:00
|
|
|
if (IS_ERR(vma)) {
|
|
|
|
ret = PTR_ERR(vma);
|
|
|
|
goto err_id;
|
|
|
|
}
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2016-04-19 23:08:34 +08:00
|
|
|
/* We'll keep just the first (doorbell/proc) page permanently kmap'd. */
|
2016-08-15 17:48:51 +08:00
|
|
|
client->vma = vma;
|
2016-11-03 01:50:47 +08:00
|
|
|
|
|
|
|
vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
|
2017-03-23 01:39:44 +08:00
|
|
|
if (IS_ERR(vaddr)) {
|
|
|
|
ret = PTR_ERR(vaddr);
|
|
|
|
goto err_vma;
|
|
|
|
}
|
2016-11-03 01:50:47 +08:00
|
|
|
client->vaddr = vaddr;
|
2016-09-09 21:11:57 +08:00
|
|
|
|
2018-10-23 07:04:23 +08:00
|
|
|
ret = reserve_doorbell(client);
|
|
|
|
if (ret)
|
|
|
|
goto err_vaddr;
|
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
client->doorbell_offset = __select_cacheline(guc);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Since the doorbell only requires a single cacheline, we can save
|
|
|
|
* space by putting the application process descriptor in the same
|
|
|
|
* page. Use the half of the page that doesn't include the doorbell.
|
|
|
|
*/
|
|
|
|
if (client->doorbell_offset >= (GUC_DB_SIZE / 2))
|
|
|
|
client->proc_desc_offset = 0;
|
|
|
|
else
|
|
|
|
client->proc_desc_offset = (GUC_DB_SIZE / 2);
|
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
DRM_DEBUG_DRIVER("new priority %u client %p for engine(s) 0x%x: stage_id %u\n",
|
|
|
|
priority, client, client->engines, client->stage_id);
|
2017-03-23 01:39:44 +08:00
|
|
|
DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%lx\n",
|
|
|
|
client->doorbell_id, client->doorbell_offset);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
|
|
|
return client;
|
2017-03-23 01:39:52 +08:00
|
|
|
|
|
|
|
err_vaddr:
|
|
|
|
i915_gem_object_unpin_map(client->vma->obj);
|
2017-03-23 01:39:44 +08:00
|
|
|
err_vma:
|
2018-07-21 20:50:37 +08:00
|
|
|
i915_vma_unpin_and_release(&client->vma, 0);
|
2017-03-23 01:39:44 +08:00
|
|
|
err_id:
|
2017-03-23 01:39:53 +08:00
|
|
|
ida_simple_remove(&guc->stage_ids, client->stage_id);
|
2017-03-23 01:39:44 +08:00
|
|
|
err_client:
|
|
|
|
kfree(client);
|
|
|
|
return ERR_PTR(ret);
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:40 +08:00
|
|
|
static void guc_client_free(struct intel_guc_client *client)
|
2017-03-23 01:39:52 +08:00
|
|
|
{
|
2017-12-14 06:13:50 +08:00
|
|
|
unreserve_doorbell(client);
|
2018-07-21 20:50:37 +08:00
|
|
|
i915_vma_unpin_and_release(&client->vma, I915_VMA_RELEASE_MAP);
|
2017-03-23 01:39:53 +08:00
|
|
|
ida_simple_remove(&client->guc->stage_ids, client->stage_id);
|
2017-03-23 01:39:52 +08:00
|
|
|
kfree(client);
|
|
|
|
}
|
|
|
|
|
2018-02-27 00:37:59 +08:00
|
|
|
static inline bool ctx_save_restore_disabled(struct intel_context *ce)
|
|
|
|
{
|
|
|
|
u32 sr = ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1];
|
|
|
|
|
|
|
|
#define SR_DISABLED \
|
|
|
|
_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT | \
|
|
|
|
CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)
|
|
|
|
|
|
|
|
return (sr & SR_DISABLED) == SR_DISABLED;
|
|
|
|
|
|
|
|
#undef SR_DISABLED
|
|
|
|
}
|
|
|
|
|
2017-10-26 22:17:37 +08:00
|
|
|
static int guc_clients_create(struct intel_guc *guc)
|
|
|
|
{
|
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
2017-11-16 21:32:40 +08:00
|
|
|
struct intel_guc_client *client;
|
2017-10-26 22:17:37 +08:00
|
|
|
|
|
|
|
GEM_BUG_ON(guc->execbuf_client);
|
|
|
|
GEM_BUG_ON(guc->preempt_client);
|
|
|
|
|
|
|
|
client = guc_client_alloc(dev_priv,
|
|
|
|
INTEL_INFO(dev_priv)->ring_mask,
|
|
|
|
GUC_CLIENT_PRIORITY_KMD_NORMAL,
|
|
|
|
dev_priv->kernel_context);
|
|
|
|
if (IS_ERR(client)) {
|
|
|
|
DRM_ERROR("Failed to create GuC client for submission!\n");
|
|
|
|
return PTR_ERR(client);
|
|
|
|
}
|
|
|
|
guc->execbuf_client = client;
|
|
|
|
|
2018-02-08 05:05:44 +08:00
|
|
|
if (dev_priv->preempt_context) {
|
|
|
|
client = guc_client_alloc(dev_priv,
|
|
|
|
INTEL_INFO(dev_priv)->ring_mask,
|
|
|
|
GUC_CLIENT_PRIORITY_KMD_HIGH,
|
|
|
|
dev_priv->preempt_context);
|
|
|
|
if (IS_ERR(client)) {
|
|
|
|
DRM_ERROR("Failed to create GuC client for preemption!\n");
|
|
|
|
guc_client_free(guc->execbuf_client);
|
|
|
|
guc->execbuf_client = NULL;
|
|
|
|
return PTR_ERR(client);
|
|
|
|
}
|
|
|
|
guc->preempt_client = client;
|
2017-10-26 22:17:37 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void guc_clients_destroy(struct intel_guc *guc)
|
|
|
|
{
|
2017-11-16 21:32:40 +08:00
|
|
|
struct intel_guc_client *client;
|
2017-10-26 22:17:37 +08:00
|
|
|
|
|
|
|
client = fetch_and_zero(&guc->preempt_client);
|
2018-02-08 05:05:42 +08:00
|
|
|
if (client)
|
|
|
|
guc_client_free(client);
|
|
|
|
|
|
|
|
client = fetch_and_zero(&guc->execbuf_client);
|
2018-07-13 04:20:27 +08:00
|
|
|
if (client)
|
|
|
|
guc_client_free(client);
|
2017-10-26 22:17:37 +08:00
|
|
|
}
|
|
|
|
|
2018-10-03 05:54:28 +08:00
|
|
|
static int __guc_client_enable(struct intel_guc_client *client)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
guc_proc_desc_init(client);
|
|
|
|
guc_stage_desc_init(client);
|
|
|
|
|
|
|
|
ret = create_doorbell(client);
|
|
|
|
if (ret)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
fail:
|
|
|
|
guc_stage_desc_fini(client);
|
|
|
|
guc_proc_desc_fini(client);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __guc_client_disable(struct intel_guc_client *client)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* By the time we're here, GuC may have already been reset. if that is
|
|
|
|
* the case, instead of trying (in vain) to communicate with it, let's
|
|
|
|
* just cleanup the doorbell HW and our internal state.
|
|
|
|
*/
|
|
|
|
if (intel_guc_is_alive(client->guc))
|
|
|
|
destroy_doorbell(client);
|
|
|
|
else
|
2018-10-23 07:04:22 +08:00
|
|
|
__fini_doorbell(client);
|
2018-10-03 05:54:28 +08:00
|
|
|
|
|
|
|
guc_stage_desc_fini(client);
|
|
|
|
guc_proc_desc_fini(client);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int guc_clients_enable(struct intel_guc *guc)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = __guc_client_enable(guc->execbuf_client);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (guc->preempt_client) {
|
|
|
|
ret = __guc_client_enable(guc->preempt_client);
|
|
|
|
if (ret) {
|
|
|
|
__guc_client_disable(guc->execbuf_client);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void guc_clients_disable(struct intel_guc *guc)
|
|
|
|
{
|
|
|
|
if (guc->preempt_client)
|
|
|
|
__guc_client_disable(guc->preempt_client);
|
|
|
|
|
|
|
|
if (guc->execbuf_client)
|
|
|
|
__guc_client_disable(guc->execbuf_client);
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:39 +08:00
|
|
|
/*
|
2017-03-23 01:39:52 +08:00
|
|
|
* Set up the memory resources to be shared with the GuC (via the GGTT)
|
|
|
|
* at firmware loading time.
|
2015-08-12 22:43:39 +08:00
|
|
|
*/
|
2017-11-16 21:32:39 +08:00
|
|
|
int intel_guc_submission_init(struct intel_guc *guc)
|
2015-08-12 22:43:39 +08:00
|
|
|
{
|
2017-12-14 06:13:47 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
2017-03-23 01:39:46 +08:00
|
|
|
int ret;
|
2015-08-12 22:43:39 +08:00
|
|
|
|
2017-03-23 01:39:53 +08:00
|
|
|
if (guc->stage_desc_pool)
|
2017-03-23 01:39:46 +08:00
|
|
|
return 0;
|
2015-08-12 22:43:39 +08:00
|
|
|
|
2017-10-26 04:00:10 +08:00
|
|
|
ret = guc_stage_desc_pool_create(guc);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
2017-11-06 19:48:33 +08:00
|
|
|
/*
|
|
|
|
* Keep static analysers happy, let them know that we allocated the
|
|
|
|
* vma after testing that it didn't exist earlier.
|
|
|
|
*/
|
|
|
|
GEM_BUG_ON(!guc->stage_desc_pool);
|
2017-03-23 01:39:45 +08:00
|
|
|
|
2017-12-14 06:13:52 +08:00
|
|
|
WARN_ON(!guc_verify_doorbells(guc));
|
2017-12-14 06:13:51 +08:00
|
|
|
ret = guc_clients_create(guc);
|
|
|
|
if (ret)
|
2018-06-05 20:05:47 +08:00
|
|
|
goto err_pool;
|
2017-12-14 06:13:51 +08:00
|
|
|
|
2017-12-14 06:13:47 +08:00
|
|
|
for_each_engine(engine, dev_priv, id) {
|
|
|
|
guc->preempt_work[id].engine = engine;
|
|
|
|
INIT_WORK(&guc->preempt_work[id].work, inject_preempt_context);
|
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:39 +08:00
|
|
|
return 0;
|
2016-11-29 20:10:23 +08:00
|
|
|
|
2018-06-05 20:05:47 +08:00
|
|
|
err_pool:
|
|
|
|
guc_stage_desc_pool_destroy(guc);
|
|
|
|
return ret;
|
2017-03-23 01:39:46 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:39 +08:00
|
|
|
void intel_guc_submission_fini(struct intel_guc *guc)
|
2017-03-23 01:39:46 +08:00
|
|
|
{
|
2017-12-14 06:13:47 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
|
|
|
|
for_each_engine(engine, dev_priv, id)
|
|
|
|
cancel_work_sync(&guc->preempt_work[id].work);
|
|
|
|
|
2017-12-14 06:13:51 +08:00
|
|
|
guc_clients_destroy(guc);
|
2017-12-14 06:13:52 +08:00
|
|
|
WARN_ON(!guc_verify_doorbells(guc));
|
|
|
|
|
2018-07-14 01:26:58 +08:00
|
|
|
if (guc->stage_desc_pool)
|
|
|
|
guc_stage_desc_pool_destroy(guc);
|
2016-11-29 20:10:23 +08:00
|
|
|
}
|
|
|
|
|
2017-03-09 21:20:04 +08:00
|
|
|
static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
2017-10-11 05:30:06 +08:00
|
|
|
struct intel_rps *rps = &dev_priv->gt_pm.rps;
|
2017-03-09 21:20:04 +08:00
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
int irqs;
|
|
|
|
|
2017-11-16 21:32:41 +08:00
|
|
|
/* tell all command streamers to forward interrupts (but not vblank)
|
|
|
|
* to GuC
|
|
|
|
*/
|
2017-03-09 21:20:04 +08:00
|
|
|
irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
|
|
|
|
for_each_engine(engine, dev_priv, id)
|
|
|
|
I915_WRITE(RING_MODE_GEN7(engine), irqs);
|
|
|
|
|
|
|
|
/* route USER_INTERRUPT to Host, all others are sent to GuC. */
|
|
|
|
irqs = GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
|
|
|
|
GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
|
|
|
|
/* These three registers have the same bit definitions */
|
|
|
|
I915_WRITE(GUC_BCS_RCS_IER, ~irqs);
|
|
|
|
I915_WRITE(GUC_VCS2_VCS1_IER, ~irqs);
|
|
|
|
I915_WRITE(GUC_WD_VECS_IER, ~irqs);
|
2017-03-11 10:37:01 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The REDIRECT_TO_GUC bit of the PMINTRMSK register directs all
|
|
|
|
* (unmasked) PM interrupts to the GuC. All other bits of this
|
|
|
|
* register *disable* generation of a specific interrupt.
|
|
|
|
*
|
|
|
|
* 'pm_intrmsk_mbz' indicates bits that are NOT to be set when
|
|
|
|
* writing to the PM interrupt mask register, i.e. interrupts
|
|
|
|
* that must not be disabled.
|
|
|
|
*
|
|
|
|
* If the GuC is handling these interrupts, then we must not let
|
|
|
|
* the PM code disable ANY interrupt that the GuC is expecting.
|
|
|
|
* So for each ENABLED (0) bit in this register, we must SET the
|
|
|
|
* bit in pm_intrmsk_mbz so that it's left enabled for the GuC.
|
|
|
|
* GuC needs ARAT expired interrupt unmasked hence it is set in
|
|
|
|
* pm_intrmsk_mbz.
|
|
|
|
*
|
|
|
|
* Here we CLEAR REDIRECT_TO_GUC bit in pm_intrmsk_mbz, which will
|
|
|
|
* result in the register bit being left SET!
|
|
|
|
*/
|
2017-10-11 05:30:06 +08:00
|
|
|
rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
|
|
|
|
rps->pm_intrmsk_mbz &= ~GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
|
2017-03-09 21:20:04 +08:00
|
|
|
}
|
|
|
|
|
2017-03-23 01:39:55 +08:00
|
|
|
static void guc_interrupts_release(struct drm_i915_private *dev_priv)
|
|
|
|
{
|
2017-10-11 05:30:06 +08:00
|
|
|
struct intel_rps *rps = &dev_priv->gt_pm.rps;
|
2017-03-23 01:39:55 +08:00
|
|
|
struct intel_engine_cs *engine;
|
|
|
|
enum intel_engine_id id;
|
|
|
|
int irqs;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* tell all command streamers NOT to forward interrupts or vblank
|
|
|
|
* to GuC.
|
|
|
|
*/
|
|
|
|
irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER);
|
|
|
|
irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING);
|
|
|
|
for_each_engine(engine, dev_priv, id)
|
|
|
|
I915_WRITE(RING_MODE_GEN7(engine), irqs);
|
|
|
|
|
|
|
|
/* route all GT interrupts to the host */
|
|
|
|
I915_WRITE(GUC_BCS_RCS_IER, 0);
|
|
|
|
I915_WRITE(GUC_VCS2_VCS1_IER, 0);
|
|
|
|
I915_WRITE(GUC_WD_VECS_IER, 0);
|
|
|
|
|
2017-10-11 05:30:06 +08:00
|
|
|
rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
|
|
|
|
rps->pm_intrmsk_mbz &= ~ARAT_EXPIRED_INTRMSK;
|
2017-03-23 01:39:55 +08:00
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:38 +08:00
|
|
|
static void guc_submission_park(struct intel_engine_cs *engine)
|
2017-10-25 22:39:42 +08:00
|
|
|
{
|
|
|
|
intel_engine_unpin_breadcrumbs_irq(engine);
|
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:38 +08:00
|
|
|
static void guc_submission_unpark(struct intel_engine_cs *engine)
|
2017-10-25 22:39:42 +08:00
|
|
|
{
|
|
|
|
intel_engine_pin_breadcrumbs_irq(engine);
|
|
|
|
}
|
|
|
|
|
2018-07-18 04:29:32 +08:00
|
|
|
static void guc_set_default_submission(struct intel_engine_cs *engine)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We inherit a bunch of functions from execlists that we'd like
|
|
|
|
* to keep using:
|
|
|
|
*
|
|
|
|
* engine->submit_request = execlists_submit_request;
|
|
|
|
* engine->cancel_requests = execlists_cancel_requests;
|
|
|
|
* engine->schedule = execlists_schedule;
|
|
|
|
*
|
|
|
|
* But we need to override the actual submission backend in order
|
|
|
|
* to talk to the GuC.
|
|
|
|
*/
|
|
|
|
intel_execlists_set_default_submission(engine);
|
|
|
|
|
|
|
|
engine->execlists.tasklet.func = guc_submission_tasklet;
|
|
|
|
|
|
|
|
engine->park = guc_submission_park;
|
|
|
|
engine->unpark = guc_submission_unpark;
|
|
|
|
|
|
|
|
engine->reset.prepare = guc_reset_prepare;
|
|
|
|
|
|
|
|
engine->flags &= ~I915_ENGINE_SUPPORTS_STATS;
|
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:39 +08:00
|
|
|
int intel_guc_submission_enable(struct intel_guc *guc)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-11-16 21:32:39 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
2016-08-03 05:50:31 +08:00
|
|
|
struct intel_engine_cs *engine;
|
drm/i915: Allocate intel_engine_cs structure only for the enabled engines
With the possibility of addition of many more number of rings in future,
the drm_i915_private structure could bloat as an array, of type
intel_engine_cs, is embedded inside it.
struct intel_engine_cs engine[I915_NUM_ENGINES];
Though this is still fine as generally there is only a single instance of
drm_i915_private structure used, but not all of the possible rings would be
enabled or active on most of the platforms. Some memory can be saved by
allocating intel_engine_cs structure only for the enabled/active engines.
Currently the engine/ring ID is kept static and dev_priv->engine[] is simply
indexed using the enums defined in intel_engine_id.
To save memory and continue using the static engine/ring IDs, 'engine' is
defined as an array of pointers.
struct intel_engine_cs *engine[I915_NUM_ENGINES];
dev_priv->engine[engine_ID] will be NULL for disabled engine instances.
There is a text size reduction of 928 bytes, from 1028200 to 1027272, for
i915.o file (but for i915.ko file text size remain same as 1193131 bytes).
v2:
- Remove the engine iterator field added in drm_i915_private structure,
instead pass a local iterator variable to the for_each_engine**
macros. (Chris)
- Do away with intel_engine_initialized() and instead directly use the
NULL pointer check on engine pointer. (Chris)
v3:
- Remove for_each_engine_id() macro, as the updated macro for_each_engine()
can be used in place of it. (Chris)
- Protect the access to Render engine Fault register with a NULL check, as
engine specific init is done later in Driver load sequence.
v4:
- Use !!dev_priv->engine[VCS] style for the engine check in getparam. (Chris)
- Kill the superfluous init_engine_lists().
v5:
- Cleanup the intel_engines_init() & intel_engines_setup(), with respect to
allocation of intel_engine_cs structure. (Chris)
v6:
- Rebase.
v7:
- Optimize the for_each_engine_masked() macro. (Chris)
- Change the type of 'iter' local variable to enum intel_engine_id. (Chris)
- Rebase.
v8: Rebase.
v9: Rebase.
v10:
- For index calculation use engine ID instead of pointer based arithmetic in
intel_engine_sync_index() as engine pointers are not contiguous now (Chris)
- For appropriateness, rename local enum variable 'iter' to 'id'. (Joonas)
- Use for_each_engine macro for cleanup in intel_engines_init() and remove
check for NULL engine pointer in cleanup() routines. (Joonas)
v11: Rebase.
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Signed-off-by: Akash Goel <akash.goel@intel.com>
Reviewed-by: Joonas Lahtinen <joonas.lahtinen@linux.intel.com>
Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Link: http://patchwork.freedesktop.org/patch/msgid/1476378888-7372-1-git-send-email-akash.goel@intel.com
2016-10-14 01:14:48 +08:00
|
|
|
enum intel_engine_id id;
|
2017-03-23 01:39:44 +08:00
|
|
|
int err;
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-09-14 16:32:13 +08:00
|
|
|
/*
|
|
|
|
* We're using GuC work items for submitting work through GuC. Since
|
|
|
|
* we're coalescing multiple requests from a single context into a
|
|
|
|
* single work item prior to assigning it to execlist_port, we can
|
|
|
|
* never have more work items than the total number of ports (for all
|
|
|
|
* engines). The GuC firmware is controlling the HEAD of work queue,
|
|
|
|
* and it is guaranteed that it will remove the work item from the
|
|
|
|
* queue before our request is completed.
|
|
|
|
*/
|
2017-09-22 20:43:03 +08:00
|
|
|
BUILD_BUG_ON(ARRAY_SIZE(engine->execlists.port) *
|
2017-09-14 16:32:13 +08:00
|
|
|
sizeof(struct guc_wq_item) *
|
|
|
|
I915_NUM_ENGINES > GUC_WQ_SIZE);
|
|
|
|
|
2017-12-14 06:13:51 +08:00
|
|
|
GEM_BUG_ON(!guc->execbuf_client);
|
|
|
|
|
2017-03-23 01:39:44 +08:00
|
|
|
err = intel_guc_sample_forcewake(guc);
|
|
|
|
if (err)
|
2017-12-14 06:13:51 +08:00
|
|
|
return err;
|
2017-03-23 01:39:52 +08:00
|
|
|
|
2018-10-03 05:54:28 +08:00
|
|
|
err = guc_clients_enable(guc);
|
2017-03-23 01:39:44 +08:00
|
|
|
if (err)
|
2017-12-14 06:13:51 +08:00
|
|
|
return err;
|
2015-08-19 05:34:47 +08:00
|
|
|
|
2016-08-03 05:50:31 +08:00
|
|
|
/* Take over from manual control of ELSP (execlists) */
|
2017-03-09 21:20:04 +08:00
|
|
|
guc_interrupts_capture(dev_priv);
|
|
|
|
|
|
|
|
for_each_engine(engine, dev_priv, id) {
|
2018-07-18 04:29:32 +08:00
|
|
|
engine->set_default_submission = guc_set_default_submission;
|
|
|
|
engine->set_default_submission(engine);
|
2016-09-09 21:11:53 +08:00
|
|
|
}
|
|
|
|
|
2015-08-12 22:43:41 +08:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-11-16 21:32:39 +08:00
|
|
|
void intel_guc_submission_disable(struct intel_guc *guc)
|
2015-08-12 22:43:41 +08:00
|
|
|
{
|
2017-11-16 21:32:39 +08:00
|
|
|
struct drm_i915_private *dev_priv = guc_to_i915(guc);
|
2015-08-12 22:43:41 +08:00
|
|
|
|
2017-10-25 22:39:42 +08:00
|
|
|
GEM_BUG_ON(dev_priv->gt.awake); /* GT should be parked first */
|
|
|
|
|
2017-03-11 10:36:59 +08:00
|
|
|
guc_interrupts_release(dev_priv);
|
2018-10-03 05:54:28 +08:00
|
|
|
guc_clients_disable(guc);
|
2015-08-12 22:43:41 +08:00
|
|
|
}
|
2017-11-17 06:06:31 +08:00
|
|
|
|
|
|
|
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
|
|
|
|
#include "selftests/intel_guc.c"
|
|
|
|
#endif
|