summaryrefslogtreecommitdiff
path: root/drivers/virtio/virtio_ring.c
blob: 7f1cbc59329ef3b7b14e257ca4eb86b03dea5b72 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
// SPDX-License-Identifier: GPL-2.0+
/*
 * Copyright (C) 2018, Tuomas Tynkkynen <tuomas.tynkkynen@iki.fi>
 * Copyright (C) 2018, Bin Meng <bmeng.cn@gmail.com>
 *
 * virtio ring implementation
 */

#include <common.h>
#include <dm.h>
#include <log.h>
#include <malloc.h>
#include <virtio_types.h>
#include <virtio.h>
#include <virtio_ring.h>
#include <linux/bug.h>
#include <linux/compat.h>

int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[],
		  unsigned int out_sgs, unsigned int in_sgs)
{
	struct vring_desc *desc;
	unsigned int total_sg = out_sgs + in_sgs;
	unsigned int i, n, avail, descs_used, uninitialized_var(prev);
	int head;

	WARN_ON(total_sg == 0);

	head = vq->free_head;

	desc = vq->vring.desc;
	i = head;
	descs_used = total_sg;

	if (vq->num_free < descs_used) {
		debug("Can't add buf len %i - avail = %i\n",
		      descs_used, vq->num_free);
		/*
		 * FIXME: for historical reasons, we force a notify here if
		 * there are outgoing parts to the buffer.  Presumably the
		 * host should service the ring ASAP.
		 */
		if (out_sgs)
			virtio_notify(vq->vdev, vq);
		return -ENOSPC;
	}

	for (n = 0; n < out_sgs; n++) {
		struct virtio_sg *sg = sgs[n];

		desc[i].flags = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT);
		desc[i].addr = cpu_to_virtio64(vq->vdev, (u64)(size_t)sg->addr);
		desc[i].len = cpu_to_virtio32(vq->vdev, sg->length);

		prev = i;
		i = virtio16_to_cpu(vq->vdev, desc[i].next);
	}
	for (; n < (out_sgs + in_sgs); n++) {
		struct virtio_sg *sg = sgs[n];

		desc[i].flags = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT |
						VRING_DESC_F_WRITE);
		desc[i].addr = cpu_to_virtio64(vq->vdev,
					       (u64)(uintptr_t)sg->addr);
		desc[i].len = cpu_to_virtio32(vq->vdev, sg->length);

		prev = i;
		i = virtio16_to_cpu(vq->vdev, desc[i].next);
	}
	/* Last one doesn't continue */
	desc[prev].flags &= cpu_to_virtio16(vq->vdev, ~VRING_DESC_F_NEXT);

	/* We're using some buffers from the free list. */
	vq->num_free -= descs_used;

	/* Update free pointer */
	vq->free_head = i;

	/*
	 * Put entry in available array (but don't update avail->idx
	 * until they do sync).
	 */
	avail = vq->avail_idx_shadow & (vq->vring.num - 1);
	vq->vring.avail->ring[avail] = cpu_to_virtio16(vq->vdev, head);

	/*
	 * Descriptors and available array need to be set before we expose the
	 * new available array entries.
	 */
	virtio_wmb();
	vq->avail_idx_shadow++;
	vq->vring.avail->idx = cpu_to_virtio16(vq->vdev, vq->avail_idx_shadow);
	vq->num_added++;

	/*
	 * This is very unlikely, but theoretically possible.
	 * Kick just in case.
	 */
	if (unlikely(vq->num_added == (1 << 16) - 1))
		virtqueue_kick(vq);

	return 0;
}

static bool virtqueue_kick_prepare(struct virtqueue *vq)
{
	u16 new, old;
	bool needs_kick;

	/*
	 * We need to expose available array entries before checking
	 * avail event.
	 */
	virtio_mb();

	old = vq->avail_idx_shadow - vq->num_added;
	new = vq->avail_idx_shadow;
	vq->num_added = 0;

	if (vq->event) {
		needs_kick = vring_need_event(virtio16_to_cpu(vq->vdev,
				vring_avail_event(&vq->vring)), new, old);
	} else {
		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(vq->vdev,
				VRING_USED_F_NO_NOTIFY));
	}

	return needs_kick;
}

void virtqueue_kick(struct virtqueue *vq)
{
	if (virtqueue_kick_prepare(vq))
		virtio_notify(vq->vdev, vq);
}

static void detach_buf(struct virtqueue *vq, unsigned int head)
{
	unsigned int i;
	__virtio16 nextflag = cpu_to_virtio16(vq->vdev, VRING_DESC_F_NEXT);

	/* Put back on free list: unmap first-level descriptors and find end */
	i = head;

	while (vq->vring.desc[i].flags & nextflag) {
		i = virtio16_to_cpu(vq->vdev, vq->vring.desc[i].next);
		vq->num_free++;
	}

	vq->vring.desc[i].next = cpu_to_virtio16(vq->vdev, vq->free_head);
	vq->free_head = head;

	/* Plus final descriptor */
	vq->num_free++;
}

static inline bool more_used(const struct virtqueue *vq)
{
	return vq->last_used_idx != virtio16_to_cpu(vq->vdev,
			vq->vring.used->idx);
}

void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len)
{
	unsigned int i;
	u16 last_used;

	if (!more_used(vq)) {
		debug("(%s.%d): No more buffers in queue\n",
		      vq->vdev->name, vq->index);
		return NULL;
	}

	/* Only get used array entries after they have been exposed by host */
	virtio_rmb();

	last_used = (vq->last_used_idx & (vq->vring.num - 1));
	i = virtio32_to_cpu(vq->vdev, vq->vring.used->ring[last_used].id);
	if (len) {
		*len = virtio32_to_cpu(vq->vdev,
				       vq->vring.used->ring[last_used].len);
		debug("(%s.%d): last used idx %u with len %u\n",
		      vq->vdev->name, vq->index, i, *len);
	}

	if (unlikely(i >= vq->vring.num)) {
		printf("(%s.%d): id %u out of range\n",
		       vq->vdev->name, vq->index, i);
		return NULL;
	}

	detach_buf(vq, i);
	vq->last_used_idx++;
	/*
	 * If we expect an interrupt for the next entry, tell host
	 * by writing event index and flush out the write before
	 * the read in the next get_buf call.
	 */
	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
		virtio_store_mb(&vring_used_event(&vq->vring),
				cpu_to_virtio16(vq->vdev, vq->last_used_idx));

	return (void *)(uintptr_t)virtio64_to_cpu(vq->vdev,
						  vq->vring.desc[i].addr);
}

static struct virtqueue *__vring_new_virtqueue(unsigned int index,
					       struct vring vring,
					       struct udevice *udev)
{
	unsigned int i;
	struct virtqueue *vq;
	struct virtio_dev_priv *uc_priv = dev_get_uclass_priv(udev);
	struct udevice *vdev = uc_priv->vdev;

	vq = malloc(sizeof(*vq));
	if (!vq)
		return NULL;

	vq->vdev = vdev;
	vq->index = index;
	vq->num_free = vring.num;
	vq->vring = vring;
	vq->last_used_idx = 0;
	vq->avail_flags_shadow = 0;
	vq->avail_idx_shadow = 0;
	vq->num_added = 0;
	list_add_tail(&vq->list, &uc_priv->vqs);

	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);

	/* Tell other side not to bother us */
	vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
	if (!vq->event)
		vq->vring.avail->flags = cpu_to_virtio16(vdev,
				vq->avail_flags_shadow);

	/* Put everything in free lists */
	vq->free_head = 0;
	for (i = 0; i < vring.num - 1; i++)
		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);

	return vq;
}

struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num,
					 unsigned int vring_align,
					 struct udevice *udev)
{
	struct virtqueue *vq;
	void *queue = NULL;
	struct vring vring;

	/* We assume num is a power of 2 */
	if (num & (num - 1)) {
		printf("Bad virtqueue length %u\n", num);
		return NULL;
	}

	/* TODO: allocate each queue chunk individually */
	for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
		queue = memalign(PAGE_SIZE, vring_size(num, vring_align));
		if (queue)
			break;
	}

	if (!num)
		return NULL;

	if (!queue) {
		/* Try to get a single page. You are my only hope! */
		queue = memalign(PAGE_SIZE, vring_size(num, vring_align));
	}
	if (!queue)
		return NULL;

	memset(queue, 0, vring_size(num, vring_align));
	vring_init(&vring, num, queue, vring_align);

	vq = __vring_new_virtqueue(index, vring, udev);
	if (!vq) {
		free(queue);
		return NULL;
	}
	debug("(%s): created vring @ %p for vq @ %p with num %u\n", udev->name,
	      queue, vq, num);

	return vq;
}

void vring_del_virtqueue(struct virtqueue *vq)
{
	free(vq->vring.desc);
	list_del(&vq->list);
	free(vq);
}

unsigned int virtqueue_get_vring_size(struct virtqueue *vq)
{
	return vq->vring.num;
}

ulong virtqueue_get_desc_addr(struct virtqueue *vq)
{
	return (ulong)vq->vring.desc;
}

ulong virtqueue_get_avail_addr(struct virtqueue *vq)
{
	return (ulong)vq->vring.desc +
	       ((char *)vq->vring.avail - (char *)vq->vring.desc);
}

ulong virtqueue_get_used_addr(struct virtqueue *vq)
{
	return (ulong)vq->vring.desc +
	       ((char *)vq->vring.used - (char *)vq->vring.desc);
}

bool virtqueue_poll(struct virtqueue *vq, u16 last_used_idx)
{
	virtio_mb();

	return last_used_idx != virtio16_to_cpu(vq->vdev, vq->vring.used->idx);
}

void virtqueue_dump(struct virtqueue *vq)
{
	unsigned int i;

	printf("virtqueue %p for dev %s:\n", vq, vq->vdev->name);
	printf("\tindex %u, phys addr %p num %u\n",
	       vq->index, vq->vring.desc, vq->vring.num);
	printf("\tfree_head %u, num_added %u, num_free %u\n",
	       vq->free_head, vq->num_added, vq->num_free);
	printf("\tlast_used_idx %u, avail_flags_shadow %u, avail_idx_shadow %u\n",
	       vq->last_used_idx, vq->avail_flags_shadow, vq->avail_idx_shadow);

	printf("Descriptor dump:\n");
	for (i = 0; i < vq->vring.num; i++) {
		printf("\tdesc[%u] = { 0x%llx, len %u, flags %u, next %u }\n",
		       i, vq->vring.desc[i].addr, vq->vring.desc[i].len,
		       vq->vring.desc[i].flags, vq->vring.desc[i].next);
	}

	printf("Avail ring dump:\n");
	printf("\tflags %u, idx %u\n",
	       vq->vring.avail->flags, vq->vring.avail->idx);
	for (i = 0; i < vq->vring.num; i++) {
		printf("\tavail[%u] = %u\n",
		       i, vq->vring.avail->ring[i]);
	}

	printf("Used ring dump:\n");
	printf("\tflags %u, idx %u\n",
	       vq->vring.used->flags, vq->vring.used->idx);
	for (i = 0; i < vq->vring.num; i++) {
		printf("\tused[%u] = { %u, %u }\n", i,
		       vq->vring.used->ring[i].id, vq->vring.used->ring[i].len);
	}
}