Branch data Line data Source code
1 : : /*-
2 : : * BSD LICENSE
3 : : *
4 : : * Copyright (c) Intel Corporation.
5 : : * All rights reserved.
6 : : *
7 : : * Redistribution and use in source and binary forms, with or without
8 : : * modification, are permitted provided that the following conditions
9 : : * are met:
10 : : *
11 : : * * Redistributions of source code must retain the above copyright
12 : : * notice, this list of conditions and the following disclaimer.
13 : : * * Redistributions in binary form must reproduce the above copyright
14 : : * notice, this list of conditions and the following disclaimer in
15 : : * the documentation and/or other materials provided with the
16 : : * distribution.
17 : : * * Neither the name of Intel Corporation nor the names of its
18 : : * contributors may be used to endorse or promote products derived
19 : : * from this software without specific prior written permission.
20 : : *
21 : : * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 : : * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 : : * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 : : * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 : : * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 : : * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 : : * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 : : * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 : : * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 : : * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 : : * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 : : */
33 : :
34 : : #include "aio_getevents.h"
35 : : #include "spdk/stdinc.h"
36 : : #include "spdk/barrier.h"
37 : : #include "spdk/likely.h"
38 : : #include "spdk/util.h"
39 : :
40 : : /* For user space reaping of completions */
41 : : struct spdk_aio_ring {
42 : : uint32_t id;
43 : : uint32_t size;
44 : : uint32_t head;
45 : : uint32_t tail;
46 : :
47 : : uint32_t version;
48 : : uint32_t compat_features;
49 : : uint32_t incompat_features;
50 : : uint32_t header_length;
51 : : };
52 : :
53 : : #define SPDK_AIO_RING_VERSION 0xa10a10a1
54 : :
55 : 0 : int user_io_getevents(io_context_t io_ctx, unsigned int max, struct io_event *uevents)
56 : : {
57 : 0 : uint32_t head, tail, count;
58 : 0 : struct spdk_aio_ring *ring;
59 : 0 : struct timespec timeout;
60 : 0 : struct io_event *kevents;
61 : :
62 : 0 : ring = (struct spdk_aio_ring *)io_ctx;
63 : :
64 [ # # # # ]: 0 : if (spdk_unlikely(ring->version != SPDK_AIO_RING_VERSION || ring->incompat_features != 0)) {
65 : 0 : timeout.tv_sec = 0;
66 : 0 : timeout.tv_nsec = 0;
67 : :
68 : 0 : return io_getevents(io_ctx, 0, max, uevents, &timeout);
69 : : }
70 : :
71 : : /* Read the current state out of the ring */
72 : 0 : head = ring->head;
73 : 0 : tail = ring->tail;
74 : :
75 : : /* This memory barrier is required to prevent the loads above
76 : : * from being re-ordered with stores to the events array
77 : : * potentially occurring on other threads. */
78 : 0 : spdk_smp_rmb();
79 : :
80 : : /* Calculate how many items are in the circular ring */
81 : 0 : count = tail - head;
82 [ # # ]: 0 : if (tail < head) {
83 : 0 : count += ring->size;
84 : : }
85 : :
86 : : /* Reduce the count to the limit provided by the user */
87 : 0 : count = spdk_min(max, count);
88 : :
89 : : /* Grab the memory location of the event array */
90 : 0 : kevents = (struct io_event *)((uintptr_t)ring + ring->header_length);
91 : :
92 : : /* Copy the events out of the ring. */
93 [ # # ]: 0 : if ((head + count) <= ring->size) {
94 : : /* Only one copy is required */
95 : 0 : memcpy(uevents, &kevents[head], count * sizeof(struct io_event));
96 : : } else {
97 : 0 : uint32_t first_part = ring->size - head;
98 : : /* Two copies are required */
99 : 0 : memcpy(uevents, &kevents[head], first_part * sizeof(struct io_event));
100 : 0 : memcpy(&uevents[first_part], &kevents[0], (count - first_part) * sizeof(struct io_event));
101 : : }
102 : :
103 : : /* Update the head pointer. On x86, stores will not be reordered with older loads,
104 : : * so the copies out of the event array will always be complete prior to this
105 : : * update becoming visible. On other architectures this is not guaranteed, so
106 : : * add a barrier. */
107 : : #if defined(__i386__) || defined(__x86_64__)
108 : 0 : spdk_compiler_barrier();
109 : : #else
110 : : spdk_smp_mb();
111 : : #endif
112 : 0 : ring->head = (head + count) % ring->size;
113 : :
114 : 0 : return count;
115 : : }
|
