2 * Copyright 2010 Tilera Corporation. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation, version 2.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11 * NON INFRINGEMENT. See the GNU General Public License for
16 * @file drv_xgbe_intf.h
17 * Interface to the hypervisor XGBE driver.
20 #ifndef __DRV_XGBE_INTF_H__
21 #define __DRV_XGBE_INTF_H__
24 * An object for forwarding VAs and PAs to the hypervisor.
27 * This allows the supervisor to specify a number of areas of memory to
28 * store packet buffers.
32 /** The physical address of the memory. */
34 /** Page table entry for the memory. This is only used to derive the
35 * memory's caching mode; the PA bits are ignored. */
37 /** The virtual address of the memory. */
39 /** Size (in bytes) of the memory area. */
45 /** The various pread/pwrite offsets into the hypervisor-level driver.
50 /** Inform the Linux driver of the address of the NetIO arena memory.
51 * This offset is actually only used to convey information from netio
52 * to the Linux driver; it never makes it from there to the hypervisor.
53 * Write-only; takes a uint32_t specifying the VA address. */
54 NETIO_FIXED_ADDR
= 0x5000000000000000ULL
,
56 /** Inform the Linux driver of the size of the NetIO arena memory.
57 * This offset is actually only used to convey information from netio
58 * to the Linux driver; it never makes it from there to the hypervisor.
59 * Write-only; takes a uint32_t specifying the VA size. */
60 NETIO_FIXED_SIZE
= 0x5100000000000000ULL
,
62 /** Register current tile with IPP. Write then read: write, takes a
63 * netio_input_config_t, read returns a pointer to a netio_queue_impl_t. */
64 NETIO_IPP_INPUT_REGISTER_OFF
= 0x6000000000000000ULL
,
66 /** Unregister current tile from IPP. Write-only, takes a dummy argument. */
67 NETIO_IPP_INPUT_UNREGISTER_OFF
= 0x6100000000000000ULL
,
69 /** Start packets flowing. Write-only, takes a dummy argument. */
70 NETIO_IPP_INPUT_INIT_OFF
= 0x6200000000000000ULL
,
72 /** Stop packets flowing. Write-only, takes a dummy argument. */
73 NETIO_IPP_INPUT_UNINIT_OFF
= 0x6300000000000000ULL
,
75 /** Configure group (typically we group on VLAN). Write-only: takes an
76 * array of netio_group_t's, low 24 bits of the offset is the base group
77 * number times the size of a netio_group_t. */
78 NETIO_IPP_INPUT_GROUP_CFG_OFF
= 0x6400000000000000ULL
,
80 /** Configure bucket. Write-only: takes an array of netio_bucket_t's, low
81 * 24 bits of the offset is the base bucket number times the size of a
83 NETIO_IPP_INPUT_BUCKET_CFG_OFF
= 0x6500000000000000ULL
,
85 /** Get/set a parameter. Read or write: read or write data is the parameter
86 * value, low 32 bits of the offset is a __netio_getset_offset_t. */
87 NETIO_IPP_PARAM_OFF
= 0x6600000000000000ULL
,
89 /** Get fast I/O index. Read-only; returns a 4-byte base index value. */
90 NETIO_IPP_GET_FASTIO_OFF
= 0x6700000000000000ULL
,
92 /** Configure hijack IP address. Packets with this IPv4 dest address
93 * go to bucket NETIO_NUM_BUCKETS - 1. Write-only: takes an IP address
94 * in some standard form. FIXME: Define the form! */
95 NETIO_IPP_INPUT_HIJACK_CFG_OFF
= 0x6800000000000000ULL
,
98 * Offsets beyond this point are reserved for the supervisor (although that
99 * enforcement must be done by the supervisor driver itself).
101 NETIO_IPP_USER_MAX_OFF
= 0x6FFFFFFFFFFFFFFFULL
,
103 /** Register I/O memory. Write-only, takes a netio_ipp_address_t. */
104 NETIO_IPP_IOMEM_REGISTER_OFF
= 0x7000000000000000ULL
,
106 /** Unregister I/O memory. Write-only, takes a netio_ipp_address_t. */
107 NETIO_IPP_IOMEM_UNREGISTER_OFF
= 0x7100000000000000ULL
,
109 /* Offsets greater than 0x7FFFFFFF can't be used directly from Linux
110 * userspace code due to limitations in the pread/pwrite syscalls. */
112 /** Drain LIPP buffers. */
113 NETIO_IPP_DRAIN_OFF
= 0xFA00000000000000ULL
,
115 /** Supply a netio_ipp_address_t to be used as shared memory for the
116 * LEPP command queue. */
117 NETIO_EPP_SHM_OFF
= 0xFB00000000000000ULL
,
119 /* 0xFC... is currently unused. */
121 /** Stop IPP/EPP tiles. Write-only, takes a dummy argument. */
122 NETIO_IPP_STOP_SHIM_OFF
= 0xFD00000000000000ULL
,
124 /** Start IPP/EPP tiles. Write-only, takes a dummy argument. */
125 NETIO_IPP_START_SHIM_OFF
= 0xFE00000000000000ULL
,
127 /** Supply packet arena. Write-only, takes an array of
128 * netio_ipp_address_t values. */
129 NETIO_IPP_ADDRESS_OFF
= 0xFF00000000000000ULL
,
132 /** Extract the base offset from an offset */
133 #define NETIO_BASE_OFFSET(off) ((off) & 0xFF00000000000000ULL)
134 /** Extract the local offset from an offset */
135 #define NETIO_LOCAL_OFFSET(off) ((off) & 0x00FFFFFFFFFFFFFFULL)
145 uint64_t addr
:48; /**< Class-specific address */
146 unsigned int class:8; /**< Class (e.g., NETIO_PARAM) */
147 unsigned int opcode
:8; /**< High 8 bits of NETIO_IPP_PARAM_OFF */
149 bits
; /**< Bitfields */
150 uint64_t word
; /**< Aggregated value to use as the offset */
152 __netio_getset_offset_t
;
155 * Fast I/O index offsets (must be contiguous).
159 NETIO_FASTIO_ALLOCATE
= 0, /**< Get empty packet buffer */
160 NETIO_FASTIO_FREE_BUFFER
= 1, /**< Give buffer back to IPP */
161 NETIO_FASTIO_RETURN_CREDITS
= 2, /**< Give credits to IPP */
162 NETIO_FASTIO_SEND_PKT_NOCK
= 3, /**< Send a packet, no checksum */
163 NETIO_FASTIO_SEND_PKT_CK
= 4, /**< Send a packet, with checksum */
164 NETIO_FASTIO_SEND_PKT_VEC
= 5, /**< Send a vector of packets */
165 NETIO_FASTIO_SENDV_PKT
= 6, /**< Sendv one packet */
166 NETIO_FASTIO_NUM_INDEX
= 7, /**< Total number of fast I/O indices */
167 } netio_fastio_index_t
;
169 /** 3-word return type for Fast I/O call. */
172 int err
; /**< Error code. */
173 uint32_t val0
; /**< Value. Meaning depends upon the specific call. */
174 uint32_t val1
; /**< Value. Meaning depends upon the specific call. */
175 } netio_fastio_rv3_t
;
177 /** 0-argument fast I/O call */
178 int __netio_fastio0(uint32_t fastio_index
);
179 /** 1-argument fast I/O call */
180 int __netio_fastio1(uint32_t fastio_index
, uint32_t arg0
);
181 /** 3-argument fast I/O call, 2-word return value */
182 netio_fastio_rv3_t
__netio_fastio3_rv3(uint32_t fastio_index
, uint32_t arg0
,
183 uint32_t arg1
, uint32_t arg2
);
184 /** 4-argument fast I/O call */
185 int __netio_fastio4(uint32_t fastio_index
, uint32_t arg0
, uint32_t arg1
,
186 uint32_t arg2
, uint32_t arg3
);
187 /** 6-argument fast I/O call */
188 int __netio_fastio6(uint32_t fastio_index
, uint32_t arg0
, uint32_t arg1
,
189 uint32_t arg2
, uint32_t arg3
, uint32_t arg4
, uint32_t arg5
);
190 /** 9-argument fast I/O call */
191 int __netio_fastio9(uint32_t fastio_index
, uint32_t arg0
, uint32_t arg1
,
192 uint32_t arg2
, uint32_t arg3
, uint32_t arg4
, uint32_t arg5
,
193 uint32_t arg6
, uint32_t arg7
, uint32_t arg8
);
195 /** Allocate an empty packet.
196 * @param fastio_index Fast I/O index.
197 * @param size Size of the packet to allocate.
199 #define __netio_fastio_allocate(fastio_index, size) \
200 __netio_fastio1((fastio_index) + NETIO_FASTIO_ALLOCATE, size)
203 * @param fastio_index Fast I/O index.
204 * @param handle Handle for the packet to free.
206 #define __netio_fastio_free_buffer(fastio_index, handle) \
207 __netio_fastio1((fastio_index) + NETIO_FASTIO_FREE_BUFFER, handle)
209 /** Increment our receive credits.
210 * @param fastio_index Fast I/O index.
211 * @param credits Number of credits to add.
213 #define __netio_fastio_return_credits(fastio_index, credits) \
214 __netio_fastio1((fastio_index) + NETIO_FASTIO_RETURN_CREDITS, credits)
216 /** Send packet, no checksum.
217 * @param fastio_index Fast I/O index.
218 * @param ackflag Nonzero if we want an ack.
219 * @param size Size of the packet.
220 * @param va Virtual address of start of packet.
221 * @param handle Packet handle.
223 #define __netio_fastio_send_pkt_nock(fastio_index, ackflag, size, va, handle) \
224 __netio_fastio4((fastio_index) + NETIO_FASTIO_SEND_PKT_NOCK, ackflag, \
227 /** Send packet, calculate checksum.
228 * @param fastio_index Fast I/O index.
229 * @param ackflag Nonzero if we want an ack.
230 * @param size Size of the packet.
231 * @param va Virtual address of start of packet.
232 * @param handle Packet handle.
233 * @param csum0 Shim checksum header.
234 * @param csum1 Checksum seed.
236 #define __netio_fastio_send_pkt_ck(fastio_index, ackflag, size, va, handle, \
238 __netio_fastio6((fastio_index) + NETIO_FASTIO_SEND_PKT_CK, ackflag, \
239 size, va, handle, csum0, csum1)
242 /** Format for the "csum0" argument to the __netio_fastio_send routines
243 * and LEPP. Note that this is currently exactly identical to the
244 * ShimProtocolOffloadHeader.
250 unsigned int start_byte
:7; /**< The first byte to be checksummed */
251 unsigned int count
:14; /**< Number of bytes to be checksummed. */
252 unsigned int destination_byte
:7; /**< The byte to write the checksum to. */
253 unsigned int reserved
:4; /**< Reserved. */
254 } bits
; /**< Decomposed method of access. */
255 unsigned int word
; /**< To send out the IDN. */
256 } __netio_checksum_header_t
;
259 /** Sendv packet with 1 or 2 segments.
260 * @param fastio_index Fast I/O index.
261 * @param flags Ack/csum/notify flags in low 3 bits; number of segments minus
262 * 1 in next 2 bits; expected checksum in high 16 bits.
263 * @param confno Confirmation number to request, if notify flag set.
264 * @param csum0 Checksum descriptor; if zero, no checksum.
265 * @param va_F Virtual address of first segment.
266 * @param va_L Virtual address of last segment, if 2 segments.
267 * @param len_F_L Length of first segment in low 16 bits; length of last
268 * segment, if 2 segments, in high 16 bits.
270 #define __netio_fastio_sendv_pkt_1_2(fastio_index, flags, confno, csum0, \
271 va_F, va_L, len_F_L) \
272 __netio_fastio6((fastio_index) + NETIO_FASTIO_SENDV_PKT, flags, confno, \
273 csum0, va_F, va_L, len_F_L)
275 /** Send packet on PCIe interface.
276 * @param fastio_index Fast I/O index.
277 * @param flags Ack/csum/notify flags in low 3 bits.
278 * @param confno Confirmation number to request, if notify flag set.
279 * @param csum0 Checksum descriptor; Hard wired 0, not needed for PCIe.
280 * @param va_F Virtual address of the packet buffer.
281 * @param va_L Virtual address of last segment, if 2 segments. Hard wired 0.
282 * @param len_F_L Length of the packet buffer in low 16 bits.
284 #define __netio_fastio_send_pcie_pkt(fastio_index, flags, confno, csum0, \
285 va_F, va_L, len_F_L) \
286 __netio_fastio6((fastio_index) + PCIE_FASTIO_SENDV_PKT, flags, confno, \
287 csum0, va_F, va_L, len_F_L)
289 /** Sendv packet with 3 or 4 segments.
290 * @param fastio_index Fast I/O index.
291 * @param flags Ack/csum/notify flags in low 3 bits; number of segments minus
292 * 1 in next 2 bits; expected checksum in high 16 bits.
293 * @param confno Confirmation number to request, if notify flag set.
294 * @param csum0 Checksum descriptor; if zero, no checksum.
295 * @param va_F Virtual address of first segment.
296 * @param va_L Virtual address of last segment (third segment if 3 segments,
297 * fourth segment if 4 segments).
298 * @param len_F_L Length of first segment in low 16 bits; length of last
299 * segment in high 16 bits.
300 * @param va_M0 Virtual address of "middle 0" segment; this segment is sent
301 * second when there are three segments, and third if there are four.
302 * @param va_M1 Virtual address of "middle 1" segment; this segment is sent
303 * second when there are four segments.
304 * @param len_M0_M1 Length of middle 0 segment in low 16 bits; length of middle
305 * 1 segment, if 4 segments, in high 16 bits.
307 #define __netio_fastio_sendv_pkt_3_4(fastio_index, flags, confno, csum0, va_F, \
308 va_L, len_F_L, va_M0, va_M1, len_M0_M1) \
309 __netio_fastio9((fastio_index) + NETIO_FASTIO_SENDV_PKT, flags, confno, \
310 csum0, va_F, va_L, len_F_L, va_M0, va_M1, len_M0_M1)
312 /** Send vector of packets.
313 * @param fastio_index Fast I/O index.
314 * @param seqno Number of packets transmitted so far on this interface;
315 * used to decide which packets should be acknowledged.
316 * @param nentries Number of entries in vector.
317 * @param va Virtual address of start of vector entry array.
318 * @return 3-word netio_fastio_rv3_t structure. The structure's err member
319 * is an error code, or zero if no error. The val0 member is the
320 * updated value of seqno; it has been incremented by 1 for each
321 * packet sent. That increment may be less than nentries if an
322 * error occurred, or if some of the entries in the vector contain
323 * handles equal to NETIO_PKT_HANDLE_NONE. The val1 member is the
324 * updated value of nentries; it has been decremented by 1 for each
325 * vector entry processed. Again, that decrement may be less than
326 * nentries (leaving the returned value positive) if an error
329 #define __netio_fastio_send_pkt_vec(fastio_index, seqno, nentries, va) \
330 __netio_fastio3_rv3((fastio_index) + NETIO_FASTIO_SEND_PKT_VEC, seqno, \
334 /** An egress DMA command for LEPP. */
337 /** Is this a TSO transfer?
339 * NOTE: This field is always 0, to distinguish it from
340 * lepp_tso_cmd_t. It must come first!
344 /** Unused padding bits. */
347 /** Should this packet be sent directly from caches instead of DRAM,
348 * using hash-for-home to locate the packet data?
350 uint8_t hash_for_home
: 1;
352 /** Should we compute a checksum? */
353 uint8_t compute_checksum
: 1;
355 /** Is this the final buffer for this packet?
357 * A single packet can be split over several input buffers (a "gather"
358 * operation). This flag indicates that this is the last buffer
361 uint8_t end_of_packet
: 1;
363 /** Should LEPP advance 'comp_busy' when this DMA is fully finished? */
364 uint8_t send_completion
: 1;
366 /** High bits of Client Physical Address of the start of the buffer
369 * NOTE: Only 6 bits are actually needed here, as CPAs are
370 * currently 38 bits. So two bits could be scavenged from this.
374 /** The number of bytes to be egressed. */
377 /** Low 32 bits of Client Physical Address of the start of the buffer
382 /** Checksum information (only used if 'compute_checksum'). */
383 __netio_checksum_header_t checksum_data
;
388 /** A chunk of physical memory for a TSO egress. */
391 /** The low bits of the CPA. */
393 /** The high bits of the CPA. */
394 uint16_t cpa_hi
: 15;
395 /** Should this packet be sent directly from caches instead of DRAM,
396 * using hash-for-home to locate the packet data?
398 uint16_t hash_for_home
: 1;
399 /** The length in bytes. */
404 /** An LEPP command that handles TSO. */
407 /** Is this a TSO transfer?
409 * NOTE: This field is always 1, to distinguish it from
410 * lepp_cmd_t. It must come first!
414 /** Unused padding bits. */
417 /** Size of the header[] array in bytes. It must be in the range
418 * [40, 127], which are the smallest header for a TCP packet over
419 * Ethernet and the maximum possible prepend size supported by
420 * hardware, respectively. Note that the array storage must be
421 * padded out to a multiple of four bytes so that the following
422 * LEPP command is aligned properly.
426 /** Byte offset of the IP header in header[]. */
429 /** Byte offset of the TCP header in header[]. */
432 /** The number of bytes to use for the payload of each packet,
433 * except of course the last one, which may not have enough bytes.
434 * This means that each Ethernet packet except the last will have a
435 * size of header_size + payload_size.
437 uint16_t payload_size
;
439 /** The length of the 'frags' array that follows this struct. */
442 /** The actual frags. */
443 lepp_frag_t frags
[0 /* Variable-sized; num_frags entries. */];
446 * The packet header template logically follows frags[],
447 * but you can't declare that in C.
449 * uint32_t header[header_size_in_words_rounded_up];
455 /** An LEPP completion ring entry. */
456 typedef void* lepp_comp_t
;
459 /** Maximum number of frags for one TSO command. This is adapted from
460 * linux's "MAX_SKB_FRAGS", and presumably over-estimates by one, for
461 * our page size of exactly 65536. We add one for a "body" fragment.
463 #define LEPP_MAX_FRAGS (65536 / HV_DEFAULT_PAGE_SIZE_SMALL + 2 + 1)
465 /** Total number of bytes needed for an lepp_tso_cmd_t. */
466 #define LEPP_TSO_CMD_SIZE(num_frags, header_size) \
467 (sizeof(lepp_tso_cmd_t) + \
468 (num_frags) * sizeof(lepp_frag_t) + \
469 (((header_size) + 3) & -4))
471 /** The size of the lepp "cmd" queue. */
472 #define LEPP_CMD_QUEUE_BYTES \
473 (((CHIP_L2_CACHE_SIZE() - 2 * CHIP_L2_LINE_SIZE()) / \
474 (sizeof(lepp_cmd_t) + sizeof(lepp_comp_t))) * sizeof(lepp_cmd_t))
476 /** The largest possible command that can go in lepp_queue_t::cmds[]. */
477 #define LEPP_MAX_CMD_SIZE LEPP_TSO_CMD_SIZE(LEPP_MAX_FRAGS, 128)
479 /** The largest possible value of lepp_queue_t::cmd_{head, tail} (inclusive).
481 #define LEPP_CMD_LIMIT \
482 (LEPP_CMD_QUEUE_BYTES - LEPP_MAX_CMD_SIZE)
484 /** The maximum number of completions in an LEPP queue. */
485 #define LEPP_COMP_QUEUE_SIZE \
486 ((LEPP_CMD_LIMIT + sizeof(lepp_cmd_t) - 1) / sizeof(lepp_cmd_t))
488 /** Increment an index modulo the queue size. */
489 #define LEPP_QINC(var) \
490 (var = __insn_mnz(var - (LEPP_COMP_QUEUE_SIZE - 1), var + 1))
492 /** A queue used to convey egress commands from the client to LEPP. */
495 /** Index of first completion not yet processed by user code.
496 * If this is equal to comp_busy, there are no such completions.
498 * NOTE: This is only read/written by the user.
500 unsigned int comp_head
;
502 /** Index of first completion record not yet completed.
503 * If this is equal to comp_tail, there are no such completions.
504 * This index gets advanced (modulo LEPP_QUEUE_SIZE) whenever
505 * a command with the 'completion' bit set is finished.
507 * NOTE: This is only written by LEPP, only read by the user.
509 volatile unsigned int comp_busy
;
511 /** Index of the first empty slot in the completion ring.
512 * Entries from this up to but not including comp_head (in ring order)
513 * can be filled in with completion data.
515 * NOTE: This is only read/written by the user.
517 unsigned int comp_tail
;
519 /** Byte index of first command enqueued for LEPP but not yet processed.
521 * This is always divisible by sizeof(void*) and always <= LEPP_CMD_LIMIT.
523 * NOTE: LEPP advances this counter as soon as it no longer needs
524 * the cmds[] storage for this entry, but the transfer is not actually
525 * complete (i.e. the buffer pointed to by the command is no longer
526 * needed) until comp_busy advances.
528 * If this is equal to cmd_tail, the ring is empty.
530 * NOTE: This is only written by LEPP, only read by the user.
532 volatile unsigned int cmd_head
;
534 /** Byte index of first empty slot in the command ring. This field can
535 * be incremented up to but not equal to cmd_head (because that would
536 * mean the ring is empty).
538 * This is always divisible by sizeof(void*) and always <= LEPP_CMD_LIMIT.
540 * NOTE: This is read/written by the user, only read by LEPP.
542 volatile unsigned int cmd_tail
;
544 /** A ring of variable-sized egress DMA commands.
546 * NOTE: Only written by the user, only read by LEPP.
548 char cmds
[LEPP_CMD_QUEUE_BYTES
]
549 __attribute__((aligned(CHIP_L2_LINE_SIZE())));
551 /** A ring of user completion data.
552 * NOTE: Only read/written by the user.
554 lepp_comp_t comps
[LEPP_COMP_QUEUE_SIZE
]
555 __attribute__((aligned(CHIP_L2_LINE_SIZE())));
559 /** An internal helper function for determining the number of entries
560 * available in a ring buffer, given that there is one sentinel.
562 static inline unsigned int
563 _lepp_num_free_slots(unsigned int head
, unsigned int tail
)
566 * One entry is reserved for use as a sentinel, to distinguish
567 * "empty" from "full". So we compute
568 * (head - tail - 1) % LEPP_QUEUE_SIZE, but without using a slow % operation.
570 return (head
- tail
- 1) + ((head
<= tail
) ? LEPP_COMP_QUEUE_SIZE
: 0);
574 /** Returns how many new comp entries can be enqueued. */
575 static inline unsigned int
576 lepp_num_free_comp_slots(const lepp_queue_t
* q
)
578 return _lepp_num_free_slots(q
->comp_head
, q
->comp_tail
);
582 lepp_qsub(int v1
, int v2
)
585 return delta
+ ((delta
>> 31) & LEPP_COMP_QUEUE_SIZE
);
589 /** FIXME: Check this from linux, via a new "pwrite()" call. */
590 #define LIPP_VERSION 1
593 /** We use exactly two bytes of alignment padding. */
594 #define LIPP_PACKET_PADDING 2
596 /** The minimum size of a "small" buffer (including the padding). */
597 #define LIPP_SMALL_PACKET_SIZE 128
600 * NOTE: The following two values should total to less than around
601 * 13582, to keep the total size used for "lipp_state_t" below 64K.
604 /** The maximum number of "small" buffers.
605 * This is enough for 53 network cpus with 128 credits. Note that
606 * if these are exhausted, we will fall back to using large buffers.
608 #define LIPP_SMALL_BUFFERS 6785
610 /** The maximum number of "large" buffers.
611 * This is enough for 53 network cpus with 128 credits.
613 #define LIPP_LARGE_BUFFERS 6785
615 #endif /* __DRV_XGBE_INTF_H__ */