1 #define _GNU_SOURCE 1
2 #include <stdio.h>
3 #include <stdlib.h>
4 #include <sys/types.h>
5 #include <sys/sysmacros.h>
6 #include <sys/stat.h>
7 #include <fcntl.h>
8 #include <sys/ioctl.h>
9 #include <unistd.h>
10 #include <dirent.h>
11 #include <rte_version.h>
12 #include <rte_ethdev.h>
13 #include "dpdk_iface_common.h"
14 /*--------------------------------------------------------------------------*/
15 //#define DEBUG 1
16 #define SYSFS_PCI_DRIVER_PATH "/sys/bus/pci/drivers/"
17 #define SYSFS_PCI_IGB_UIO SYSFS_PCI_DRIVER_PATH"igb_uio"
18 #define SYSFS_PCI_VFIO_PCI SYSFS_PCI_DRIVER_PATH"vfio-pci"
19 #define SYSFS_PCI_UIOPCIGEN SYSFS_PCI_DRIVER_PATH"uio_pci_generic"
20 #define RTE_ARGC_MAX (RTE_MAX_ETHPORTS << 1) + 7
21 /*--------------------------------------------------------------------------*/
22 typedef struct {
23 PciDevice pd;
24 struct rte_eth_dev_info dev_details;
25 struct ether_addr ports_eth_addr;
26 } DevInfo;
27
28 static DevInfo di[RTE_MAX_ETHPORTS];
29 /*--------------------------------------------------------------------------*/
30 /**
31 * Really crappy version for detecting pci entries..
32 * but it should work.
33 */
34 int
IsPciEnt(const struct dirent * entry)35 IsPciEnt(const struct dirent *entry)
36 {
37 if (entry->d_type == DT_LNK &&
38 strstr(entry->d_name, ":") != NULL)
39 return 1;
40
41 return 0;
42 }
43 /*--------------------------------------------------------------------------*/
44 /**
45 * Similar to strverscmp(), but sorts in hexadecimal context
46 */
47 int
localversionsort(const void * elem1,const void * elem2)48 localversionsort(const void *elem1, const void *elem2)
49 {
50 uint16_t domain1, domain2;
51 uint8_t bus1, bus2, device1, device2, function1, function2;
52 DevInfo *d1 = (DevInfo *)elem1;
53 DevInfo *d2 = (DevInfo *)elem2;
54
55 domain1 = d1->pd.pa.domain;
56 domain2 = d2->pd.pa.domain;
57 bus1 = d1->pd.pa.bus;
58 bus2 = d2->pd.pa.bus;
59 device1 = d1->pd.pa.device;
60 device2 = d2->pd.pa.device;
61 function1 = d1->pd.pa.function;
62 function2 = d2->pd.pa.function;
63
64 if (domain1 < domain2) return -1;
65 if (domain2 < domain1) return 1;
66
67 if (bus1 < bus2) return -1;
68 if (bus2 < bus1) return 1;
69
70 if (device1 < device2) return -1;
71 if (device2 < device1) return 1;
72
73 if (function1 < function2)
74 return -1;
75 if (function2 < function1)
76 return 1;
77
78 return 0;
79 }
80 /*--------------------------------------------------------------------------*/
81 int
probe_all_rte_devices(char ** argv,int * argc)82 probe_all_rte_devices(char **argv, int *argc)
83 {
84 struct dirent **dirlist;
85 int pci_index, total_files, i, j;
86
87 /* reset pci_index */
88 pci_index = 0;
89
90 for (j = 0; j < 3; j++) {
91 switch (j) {
92 case 0:
93 /* scan igb_uio first */
94 total_files = scandir(SYSFS_PCI_IGB_UIO, &dirlist,
95 IsPciEnt, versionsort);
96 break;
97 case 1:
98 /* scan vfio_pci next */
99 total_files = scandir(SYSFS_PCI_VFIO_PCI, &dirlist,
100 IsPciEnt, versionsort);
101 break;
102 case 2:
103 /* finally scan uio_pci_generic */
104 total_files = scandir(SYSFS_PCI_UIOPCIGEN, &dirlist,
105 IsPciEnt, versionsort);
106 break;
107 default:
108 fprintf(stderr, "Control can never come here!\n");
109 goto panic_err;
110 }
111
112 for (i = 0; i < total_files; i++, pci_index++) {
113 argv[*argc] = strdup("-w");
114 argv[*argc + 1] = strdup(dirlist[i]->d_name);
115 if (argv[*argc] == NULL ||
116 argv[*argc + 1] == NULL)
117 goto alloc_err;
118 *argc += 2;
119 if (sscanf(dirlist[i]->d_name, PCI_DOM":"PCI_BUS":"
120 PCI_DEVICE"."PCI_FUNC,
121 &di[pci_index].pd.pa.domain,
122 &di[pci_index].pd.pa.bus,
123 &di[pci_index].pd.pa.device,
124 &di[pci_index].pd.pa.function) != 4)
125 goto sscanf_err;
126 free(dirlist[i]);
127 }
128
129 //free(dirlist);
130 }
131
132 /* now sort all recorded entries */
133 qsort(di, pci_index, sizeof(DevInfo), localversionsort);
134 return pci_index;
135 sscanf_err:
136 fprintf(stderr, "Unable to retrieve pci address!\n");
137 exit(EXIT_FAILURE);
138 alloc_err:
139 fprintf(stderr, "Can't allocate memory for argv items!\n");
140 exit(EXIT_FAILURE);
141 panic_err:
142 fprintf(stderr, "Could not open the directory!\n");
143 exit(EXIT_FAILURE);
144 }
145 /*--------------------------------------------------------------------------*/
146 int
fetch_major_no()147 fetch_major_no()
148 {
149 FILE *f;
150 int major_no;
151 char *line;
152 size_t len;
153 char dummy[512];
154
155 major_no = -1;
156 len = 0;
157 line = NULL;
158
159 f = fopen(DEV_PROC_PATH, "r");
160 if (f == NULL) {
161 fprintf(stderr, "Can't open %s file\n", DEV_PROC_PATH);
162 return -1;
163 }
164
165 while (getline(&line, &len, f) != -1) {
166 if (strstr(line, DEV_NAME) != NULL) {
167 if (sscanf(line, "%d %s", &major_no, dummy) == 2) {
168 free(line);
169 break;
170 }
171 }
172 free(line);
173 line = NULL;
174 len = 0;
175 }
176
177 /* close the file descriptor */
178 fclose(f);
179
180 return major_no;
181 }
182 /*--------------------------------------------------------------------------*/
183 int
main(int argc,char ** argv)184 main(int argc, char **argv)
185 {
186 int ret, fd, num_devices, i;
187 dev_t dev;
188 char *cpumaskbuf = "0x1";
189 char *mem_channels = "4";
190 char *rte_argv[RTE_ARGC_MAX] = {"",
191 "-c",
192 cpumaskbuf,
193 "-n",
194 mem_channels,
195 "--proc-type=auto"
196 };
197 int rte_argc = 6;
198
199 ret = probe_all_rte_devices(rte_argv, &rte_argc);
200
201 #if DEBUG
202 for (i = 0; i < ret; i++) {
203 fprintf(stderr, "Pci Address: %04hX:%02hhX:%02hhX.%01hhX\n",
204 di[i].pd.pa.domain,
205 di[i].pd.pa.bus,
206 di[i].pd.pa.device,
207 di[i].pd.pa.function);
208 }
209 #endif
210
211 if (geteuid()) {
212 fprintf(stderr, "[CAUTION] Run the app as root!\n");
213 exit(EXIT_FAILURE);
214 }
215
216 /* remove previously created dpdk-iface device node file */
217 fprintf(stderr, "Removing existing device node entry...");
218 ret = remove(DEV_PATH);
219 fprintf(stderr, (ret == 0) ? "\033[32m done. \033[0m \n" :
220 "\033[32m not present. \033[0m \n");
221
222 /* create dpdk-iface device node entry */
223 #if 0
224 dev = makedev(MAJOR_NO, 0);
225 #else
226 dev = makedev(fetch_major_no(), 0);
227 #endif
228 ret = mknod(DEV_PATH, S_IFCHR | O_RDWR, dev);
229 if (ret == 0)
230 fprintf(stderr, "Creating device node entry...");
231 else {
232 fprintf(stderr, "Failed to create device node entry\n");
233 return EXIT_FAILURE;
234 }
235
236 fprintf(stderr, "\033[32m done. \033[0m \n");
237
238 /* setting permissions on the device node entry */
239 ret = chmod(DEV_PATH,
240 S_IRGRP | S_IROTH | S_IRUSR |
241 S_IWGRP | S_IWOTH | S_IWUSR);
242
243 if (ret == 0)
244 fprintf(stderr, "Setting permissions on the device node entry...");
245 else {
246 fprintf(stderr, "Failed to set permissions on the device node entry\n");
247 return EXIT_FAILURE;
248 }
249
250 fprintf(stderr, "\033[32m done. \033[0m \n");
251
252 #if RTE_VERSION < RTE_VERSION_NUM(17, 05, 0, 16)
253 rte_set_log_level(RTE_LOG_EMERG);
254 #else
255 rte_log_set_global_level(RTE_LOG_EMERG);
256 #endif
257
258 fprintf(stderr, "Scanning the system for dpdk-compatible devices...");
259 /* initialize the rte env first */
260 ret = rte_eal_init(rte_argc, rte_argv);
261
262 /* get total count of detected ethernet ports */
263 num_devices = rte_eth_dev_count();
264 if (num_devices == 0) {
265 fprintf(stderr, "No Ethernet port detected!\n");
266 exit(EXIT_FAILURE);
267 }
268
269 for (ret = 0; ret < num_devices; ret++) {
270 di[ret].pd.ports_eth_addr = &di[ret].ports_eth_addr.addr_bytes[0];
271 /* get mac addr entries of detected dpdk ports */
272 rte_eth_macaddr_get(ret, &di[ret].ports_eth_addr);
273 /* check port capabailties/info */
274 rte_eth_dev_info_get(ret, &di[ret].dev_details);
275 /* get numa socket location for future socket-mem field */
276 if ((di[ret].pd.numa_socket=rte_eth_dev_socket_id(ret)) == -1) {
277 fprintf(stderr, "Can't determine socket ID!\n");
278 exit(EXIT_FAILURE);
279 }
280 }
281
282 fprintf(stderr, "\033[32m done. \033[0m \n");
283
284 /* open the device node first */
285 fd = open(DEV_PATH, O_RDWR);
286 if (fd == -1) {
287 fprintf(stderr, "Failed to open %s for port detection!\n",
288 DEV_PATH);
289 exit(EXIT_FAILURE);
290 }
291
292 /* clear all previous entries */
293 fprintf(stderr, "Clearing previous entries\n");
294 ret = ioctl(fd, CLEAR_IFACE, di[0].ports_eth_addr.addr_bytes);
295 if (ret == -1) {
296 fprintf(stderr, "ioctl call failed!\n");
297 return EXIT_FAILURE;
298 }
299
300 /* register the newly detected dpdk ports */
301 for (ret = 0; ret < num_devices; ret++) {
302 if (strcmp(di[ret].dev_details.driver_name, "net_mlx4") &&
303 strcmp(di[ret].dev_details.driver_name, "net_mlx5")) {
304 fprintf(stderr, "Registering port %d (%02X:%02X:%02X:%02X:%02X:%02X) to mTCP stack",
305 ret,
306 di[ret].ports_eth_addr.addr_bytes[0],
307 di[ret].ports_eth_addr.addr_bytes[1],
308 di[ret].ports_eth_addr.addr_bytes[2],
309 di[ret].ports_eth_addr.addr_bytes[3],
310 di[ret].ports_eth_addr.addr_bytes[4],
311 di[ret].ports_eth_addr.addr_bytes[5]);
312 di[ret].pd.ports_eth_addr = di[ret].ports_eth_addr.addr_bytes;
313
314 if (ioctl(fd, CREATE_IFACE, &di[ret].pd) == -1) {
315 fprintf(stderr, "ioctl call failed!\n");
316 }
317 fprintf(stderr, " (%s).\n",
318 di[ret].pd.ifname);
319 }
320 }
321
322 /* close the fd */
323 close(fd);
324
325 #if 0
326 /*
327 * XXX: It seems that there is a bug in the RTE SDK.
328 * The dynamically allocated rte_argv params are left
329 * as dangling pointers. Freeing them causes program
330 * to crash.
331 */
332
333 /* free up all resources */
334 for (; rte_argc >= 6; rte_argc--) {
335 if (rte_argv[rte_argc] != NULL) {
336 fprintf(stderr, "Cleaning up rte_argv[%d]: %s (%p)\n",
337 rte_argc, rte_argv[rte_argc], rte_argv[rte_argc]);
338 free(rte_argv[rte_argc]);
339 rte_argv[rte_argc] = NULL;
340 }
341 }
342 #endif
343 return EXIT_SUCCESS;
344 }
345 /*--------------------------------------------------------------------------*/
346