Skip to content

Commit 9da3da6

Browse files
ickledanvet
ickle
authored and
danvet
committed
drm/i915: Replace the array of pages with a scatterlist
Rather than have multiple data structures for describing our page layout in conjunction with the array of pages, we can migrate all users over to a scatterlist. One major advantage, other than unifying the page tracking structures, this offers is that we replace the vmalloc'ed array (which can be up to a megabyte in size) with a chain of individual pages which helps reduce memory pressure. The disadvantage is that we then do not have a simple array to iterate, or to access randomly. The common case for this is in the relocation processing, which will typically fit within a single scatterlist page and so be almost the same cost as the simple array. For iterating over the array, the extra function call could be optimised away, but in reality is an insignificant cost of either binding the pages, or performing the pwrite/pread. v2: Fix drm_clflush_sg() to not invoke wbinvd as well! And fix the trivial compile error from rebasing. Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk> Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
1 parent f60d7f0 commit 9da3da6

12 files changed

Lines changed: 239 additions & 220 deletions

File tree

‎drivers/char/agp/intel-gtt.c‎

Lines changed: 22 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -84,40 +84,33 @@ static struct _intel_private {
8484
#define IS_IRONLAKE intel_private.driver->is_ironlake
8585
#define HAS_PGTBL_EN intel_private.driver->has_pgtbl_enable
8686

87-
int intel_gtt_map_memory(struct page **pages, unsigned int num_entries,
88-
struct scatterlist **sg_list, int *num_sg)
87+
static int intel_gtt_map_memory(struct page **pages,
88+
unsigned int num_entries,
89+
struct sg_table *st)
8990
{
90-
struct sg_table st;
9191
struct scatterlist *sg;
9292
int i;
9393

94-
if (*sg_list)
95-
return 0; /* already mapped (for e.g. resume */
96-
9794
DBG("try mapping %lu pages\n", (unsigned long)num_entries);
9895

99-
if (sg_alloc_table(&st, num_entries, GFP_KERNEL))
96+
if (sg_alloc_table(st, num_entries, GFP_KERNEL))
10097
goto err;
10198

102-
*sg_list = sg = st.sgl;
103-
104-
for (i = 0 ; i < num_entries; i++, sg = sg_next(sg))
99+
for_each_sg(st->sgl, sg, num_entries, i)
105100
sg_set_page(sg, pages[i], PAGE_SIZE, 0);
106101

107-
*num_sg = pci_map_sg(intel_private.pcidev, *sg_list,
108-
num_entries, PCI_DMA_BIDIRECTIONAL);
109-
if (unlikely(!*num_sg))
102+
if (!pci_map_sg(intel_private.pcidev,
103+
st->sgl, st->nents, PCI_DMA_BIDIRECTIONAL))
110104
goto err;
111105

112106
return 0;
113107

114108
err:
115-
sg_free_table(&st);
109+
sg_free_table(st);
116110
return -ENOMEM;
117111
}
118-
EXPORT_SYMBOL(intel_gtt_map_memory);
119112

120-
void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
113+
static void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
121114
{
122115
struct sg_table st;
123116
DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
@@ -130,7 +123,6 @@ void intel_gtt_unmap_memory(struct scatterlist *sg_list, int num_sg)
130123

131124
sg_free_table(&st);
132125
}
133-
EXPORT_SYMBOL(intel_gtt_unmap_memory);
134126

135127
static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
136128
{
@@ -879,8 +871,7 @@ static bool i830_check_flags(unsigned int flags)
879871
return false;
880872
}
881873

882-
void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
883-
unsigned int sg_len,
874+
void intel_gtt_insert_sg_entries(struct sg_table *st,
884875
unsigned int pg_start,
885876
unsigned int flags)
886877
{
@@ -892,21 +883,22 @@ void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
892883

893884
/* sg may merge pages, but we have to separate
894885
* per-page addr for GTT */
895-
for_each_sg(sg_list, sg, sg_len, i) {
886+
for_each_sg(st->sgl, sg, st->nents, i) {
896887
len = sg_dma_len(sg) >> PAGE_SHIFT;
897888
for (m = 0; m < len; m++) {
898889
dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
899-
intel_private.driver->write_entry(addr,
900-
j, flags);
890+
intel_private.driver->write_entry(addr, j, flags);
901891
j++;
902892
}
903893
}
904894
readl(intel_private.gtt+j-1);
905895
}
906896
EXPORT_SYMBOL(intel_gtt_insert_sg_entries);
907897

908-
void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
909-
struct page **pages, unsigned int flags)
898+
static void intel_gtt_insert_pages(unsigned int first_entry,
899+
unsigned int num_entries,
900+
struct page **pages,
901+
unsigned int flags)
910902
{
911903
int i, j;
912904

@@ -917,7 +909,6 @@ void intel_gtt_insert_pages(unsigned int first_entry, unsigned int num_entries,
917909
}
918910
readl(intel_private.gtt+j-1);
919911
}
920-
EXPORT_SYMBOL(intel_gtt_insert_pages);
921912

922913
static int intel_fake_agp_insert_entries(struct agp_memory *mem,
923914
off_t pg_start, int type)
@@ -953,13 +944,15 @@ static int intel_fake_agp_insert_entries(struct agp_memory *mem,
953944
global_cache_flush();
954945

955946
if (intel_private.base.needs_dmar) {
956-
ret = intel_gtt_map_memory(mem->pages, mem->page_count,
957-
&mem->sg_list, &mem->num_sg);
947+
struct sg_table st;
948+
949+
ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st);
958950
if (ret != 0)
959951
return ret;
960952

961-
intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
962-
pg_start, type);
953+
intel_gtt_insert_sg_entries(&st, pg_start, type);
954+
mem->sg_list = st.sgl;
955+
mem->num_sg = st.nents;
963956
} else
964957
intel_gtt_insert_pages(pg_start, mem->page_count, mem->pages,
965958
type);

‎drivers/gpu/drm/drm_cache.c‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,31 @@ drm_clflush_pages(struct page *pages[], unsigned long num_pages)
100100
}
101101
EXPORT_SYMBOL(drm_clflush_pages);
102102

103+
void
104+
drm_clflush_sg(struct sg_table *st)
105+
{
106+
#if defined(CONFIG_X86)
107+
if (cpu_has_clflush) {
108+
struct scatterlist *sg;
109+
int i;
110+
111+
mb();
112+
for_each_sg(st->sgl, sg, st->nents, i)
113+
drm_clflush_page(sg_page(sg));
114+
mb();
115+
116+
return;
117+
}
118+
119+
if (on_each_cpu(drm_clflush_ipi_handler, NULL, 1) != 0)
120+
printk(KERN_ERR "Timed out waiting for cache flush.\n");
121+
#else
122+
printk(KERN_ERR "Architecture has no drm_cache.c support\n");
123+
WARN_ON_ONCE(1);
124+
#endif
125+
}
126+
EXPORT_SYMBOL(drm_clflush_sg);
127+
103128
void
104129
drm_clflush_virt_range(char *addr, unsigned long length)
105130
{

‎drivers/gpu/drm/i915/i915_drv.h‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,16 +1006,11 @@ struct drm_i915_gem_object {
10061006

10071007
unsigned int has_aliasing_ppgtt_mapping:1;
10081008
unsigned int has_global_gtt_mapping:1;
1009+
unsigned int has_dma_mapping:1;
10091010

1010-
struct page **pages;
1011+
struct sg_table *pages;
10111012
int pages_pin_count;
10121013

1013-
/**
1014-
* DMAR support
1015-
*/
1016-
struct scatterlist *sg_list;
1017-
int num_sg;
1018-
10191014
/* prime dma-buf support */
10201015
struct sg_table *sg_table;
10211016
void *dma_buf_vmapping;
@@ -1342,6 +1337,15 @@ void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
13421337
void i915_gem_lastclose(struct drm_device *dev);
13431338

13441339
int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
1340+
static inline struct page *i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
1341+
{
1342+
struct scatterlist *sg = obj->pages->sgl;
1343+
while (n >= SG_MAX_SINGLE_ALLOC) {
1344+
sg = sg_chain_ptr(sg + SG_MAX_SINGLE_ALLOC - 1);
1345+
n -= SG_MAX_SINGLE_ALLOC - 1;
1346+
}
1347+
return sg_page(sg+n);
1348+
}
13451349
static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
13461350
{
13471351
BUG_ON(obj->pages == NULL);

‎drivers/gpu/drm/i915/i915_gem.c‎

Lines changed: 55 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,8 @@ i915_gem_shmem_pread(struct drm_device *dev,
411411
int hit_slowpath = 0;
412412
int prefaulted = 0;
413413
int needs_clflush = 0;
414+
struct scatterlist *sg;
415+
int i;
414416

415417
user_data = (char __user *) (uintptr_t) args->data_ptr;
416418
remain = args->size;
@@ -439,9 +441,15 @@ i915_gem_shmem_pread(struct drm_device *dev,
439441

440442
offset = args->offset;
441443

442-
while (remain > 0) {
444+
for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
443445
struct page *page;
444446

447+
if (i < offset >> PAGE_SHIFT)
448+
continue;
449+
450+
if (remain <= 0)
451+
break;
452+
445453
/* Operation in this page
446454
*
447455
* shmem_page_offset = offset within page in shmem file
@@ -452,7 +460,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
452460
if ((shmem_page_offset + page_length) > PAGE_SIZE)
453461
page_length = PAGE_SIZE - shmem_page_offset;
454462

455-
page = obj->pages[offset >> PAGE_SHIFT];
463+
page = sg_page(sg);
456464
page_do_bit17_swizzling = obj_do_bit17_swizzling &&
457465
(page_to_phys(page) & (1 << 17)) != 0;
458466

@@ -731,6 +739,8 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
731739
int hit_slowpath = 0;
732740
int needs_clflush_after = 0;
733741
int needs_clflush_before = 0;
742+
int i;
743+
struct scatterlist *sg;
734744

735745
user_data = (char __user *) (uintptr_t) args->data_ptr;
736746
remain = args->size;
@@ -765,10 +775,16 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
765775
offset = args->offset;
766776
obj->dirty = 1;
767777

768-
while (remain > 0) {
778+
for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
769779
struct page *page;
770780
int partial_cacheline_write;
771781

782+
if (i < offset >> PAGE_SHIFT)
783+
continue;
784+
785+
if (remain <= 0)
786+
break;
787+
772788
/* Operation in this page
773789
*
774790
* shmem_page_offset = offset within page in shmem file
@@ -787,7 +803,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
787803
((shmem_page_offset | page_length)
788804
& (boot_cpu_data.x86_clflush_size - 1));
789805

790-
page = obj->pages[offset >> PAGE_SHIFT];
806+
page = sg_page(sg);
791807
page_do_bit17_swizzling = obj_do_bit17_swizzling &&
792808
(page_to_phys(page) & (1 << 17)) != 0;
793809

@@ -1633,6 +1649,7 @@ static void
16331649
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
16341650
{
16351651
int page_count = obj->base.size / PAGE_SIZE;
1652+
struct scatterlist *sg;
16361653
int ret, i;
16371654

16381655
BUG_ON(obj->madv == __I915_MADV_PURGED);
@@ -1653,19 +1670,21 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
16531670
if (obj->madv == I915_MADV_DONTNEED)
16541671
obj->dirty = 0;
16551672

1656-
for (i = 0; i < page_count; i++) {
1673+
for_each_sg(obj->pages->sgl, sg, page_count, i) {
1674+
struct page *page = sg_page(sg);
1675+
16571676
if (obj->dirty)
1658-
set_page_dirty(obj->pages[i]);
1677+
set_page_dirty(page);
16591678

16601679
if (obj->madv == I915_MADV_WILLNEED)
1661-
mark_page_accessed(obj->pages[i]);
1680+
mark_page_accessed(page);
16621681

1663-
page_cache_release(obj->pages[i]);
1682+
page_cache_release(page);
16641683
}
16651684
obj->dirty = 0;
16661685

1667-
drm_free_large(obj->pages);
1668-
obj->pages = NULL;
1686+
sg_free_table(obj->pages);
1687+
kfree(obj->pages);
16691688
}
16701689

16711690
static int
@@ -1682,6 +1701,7 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
16821701
return -EBUSY;
16831702

16841703
ops->put_pages(obj);
1704+
obj->pages = NULL;
16851705

16861706
list_del(&obj->gtt_list);
16871707
if (i915_gem_object_is_purgeable(obj))
@@ -1739,6 +1759,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
17391759
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
17401760
int page_count, i;
17411761
struct address_space *mapping;
1762+
struct sg_table *st;
1763+
struct scatterlist *sg;
17421764
struct page *page;
17431765
gfp_t gfp;
17441766

@@ -1749,20 +1771,27 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
17491771
BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
17501772
BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
17511773

1752-
/* Get the list of pages out of our struct file. They'll be pinned
1753-
* at this point until we release them.
1754-
*/
1774+
st = kmalloc(sizeof(*st), GFP_KERNEL);
1775+
if (st == NULL)
1776+
return -ENOMEM;
1777+
17551778
page_count = obj->base.size / PAGE_SIZE;
1756-
obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1757-
if (obj->pages == NULL)
1779+
if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1780+
sg_free_table(st);
1781+
kfree(st);
17581782
return -ENOMEM;
1783+
}
17591784

1760-
/* Fail silently without starting the shrinker */
1785+
/* Get the list of pages out of our struct file. They'll be pinned
1786+
* at this point until we release them.
1787+
*
1788+
* Fail silently without starting the shrinker
1789+
*/
17611790
mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
17621791
gfp = mapping_gfp_mask(mapping);
17631792
gfp |= __GFP_NORETRY | __GFP_NOWARN;
17641793
gfp &= ~(__GFP_IO | __GFP_WAIT);
1765-
for (i = 0; i < page_count; i++) {
1794+
for_each_sg(st->sgl, sg, page_count, i) {
17661795
page = shmem_read_mapping_page_gfp(mapping, i, gfp);
17671796
if (IS_ERR(page)) {
17681797
i915_gem_purge(dev_priv, page_count);
@@ -1785,20 +1814,20 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
17851814
gfp &= ~(__GFP_IO | __GFP_WAIT);
17861815
}
17871816

1788-
obj->pages[i] = page;
1817+
sg_set_page(sg, page, PAGE_SIZE, 0);
17891818
}
17901819

17911820
if (i915_gem_object_needs_bit17_swizzle(obj))
17921821
i915_gem_object_do_bit_17_swizzle(obj);
17931822

1823+
obj->pages = st;
17941824
return 0;
17951825

17961826
err_pages:
1797-
while (i--)
1798-
page_cache_release(obj->pages[i]);
1799-
1800-
drm_free_large(obj->pages);
1801-
obj->pages = NULL;
1827+
for_each_sg(st->sgl, sg, i, page_count)
1828+
page_cache_release(sg_page(sg));
1829+
sg_free_table(st);
1830+
kfree(st);
18021831
return PTR_ERR(page);
18031832
}
18041833

@@ -2981,7 +3010,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj)
29813010

29823011
trace_i915_gem_object_clflush(obj);
29833012

2984-
drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
3013+
drm_clflush_sg(obj->pages);
29853014
}
29863015

29873016
/** Flushes the GTT write domain for the object if it's dirty. */
@@ -3731,6 +3760,8 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
37313760
i915_gem_object_put_pages(obj);
37323761
i915_gem_object_free_mmap_offset(obj);
37333762

3763+
BUG_ON(obj->pages);
3764+
37343765
drm_gem_object_release(&obj->base);
37353766
i915_gem_info_remove_obj(dev_priv, obj->base.size);
37363767

0 commit comments

Comments
 (0)