1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2010 Alexander Motin <[email protected]>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer,
12 * without modification, immediately at the beginning of the file.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/module.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/ata.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/endian.h>
38 #include <sys/malloc.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <vm/uma.h>
42 #include <machine/stdarg.h>
43 #include <machine/resource.h>
44 #include <machine/bus.h>
45 #include <sys/rman.h>
46 #include <dev/pci/pcivar.h>
47 #include "mvs.h"
48
49 #include <cam/cam.h>
50 #include <cam/cam_ccb.h>
51 #include <cam/cam_sim.h>
52 #include <cam/cam_xpt_sim.h>
53 #include <cam/cam_debug.h>
54
55 /* local prototypes */
56 static int mvs_ch_init(device_t dev);
57 static int mvs_ch_deinit(device_t dev);
58 static int mvs_ch_suspend(device_t dev);
59 static int mvs_ch_resume(device_t dev);
60 static void mvs_dmainit(device_t dev);
61 static void mvs_dmasetupc_cb(void *xsc,
62 bus_dma_segment_t *segs, int nsegs, int error);
63 static void mvs_dmafini(device_t dev);
64 static void mvs_slotsalloc(device_t dev);
65 static void mvs_slotsfree(device_t dev);
66 static void mvs_setup_edma_queues(device_t dev);
67 static void mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode);
68 static void mvs_ch_pm(void *arg);
69 static void mvs_ch_intr_locked(void *data);
70 static void mvs_ch_intr(void *data);
71 static void mvs_reset(device_t dev);
72 static void mvs_softreset(device_t dev, union ccb *ccb);
73
74 static int mvs_sata_connect(struct mvs_channel *ch);
75 static int mvs_sata_phy_reset(device_t dev);
76 static int mvs_wait(device_t dev, u_int s, u_int c, int t);
77 static void mvs_tfd_read(device_t dev, union ccb *ccb);
78 static void mvs_tfd_write(device_t dev, union ccb *ccb);
79 static void mvs_legacy_intr(device_t dev, int poll);
80 static void mvs_crbq_intr(device_t dev);
81 static void mvs_begin_transaction(device_t dev, union ccb *ccb);
82 static void mvs_legacy_execute_transaction(struct mvs_slot *slot);
83 static void mvs_timeout(void *arg);
84 static void mvs_dmasetprd(void *arg,
85 bus_dma_segment_t *segs, int nsegs, int error);
86 static void mvs_requeue_frozen(device_t dev);
87 static void mvs_execute_transaction(struct mvs_slot *slot);
88 static void mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et);
89
90 static void mvs_issue_recovery(device_t dev);
91 static void mvs_process_read_log(device_t dev, union ccb *ccb);
92 static void mvs_process_request_sense(device_t dev, union ccb *ccb);
93
94 static void mvsaction(struct cam_sim *sim, union ccb *ccb);
95 static void mvspoll(struct cam_sim *sim);
96
97 static MALLOC_DEFINE(M_MVS, "MVS driver", "MVS driver data buffers");
98
99 #define recovery_type spriv_field0
100 #define RECOVERY_NONE 0
101 #define RECOVERY_READ_LOG 1
102 #define RECOVERY_REQUEST_SENSE 2
103 #define recovery_slot spriv_field1
104
105 static int
mvs_ch_probe(device_t dev)106 mvs_ch_probe(device_t dev)
107 {
108
109 device_set_desc(dev, "Marvell SATA channel");
110 return (BUS_PROBE_DEFAULT);
111 }
112
113 static int
mvs_ch_attach(device_t dev)114 mvs_ch_attach(device_t dev)
115 {
116 struct mvs_controller *ctlr = device_get_softc(device_get_parent(dev));
117 struct mvs_channel *ch = device_get_softc(dev);
118 struct cam_devq *devq;
119 int rid, error, i, sata_rev = 0;
120
121 ch->dev = dev;
122 ch->unit = (intptr_t)device_get_ivars(dev);
123 ch->quirks = ctlr->quirks;
124 mtx_init(&ch->mtx, "MVS channel lock", NULL, MTX_DEF);
125 ch->pm_level = 0;
126 resource_int_value(device_get_name(dev),
127 device_get_unit(dev), "pm_level", &ch->pm_level);
128 if (ch->pm_level > 3)
129 callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
130 callout_init_mtx(&ch->reset_timer, &ch->mtx, 0);
131 resource_int_value(device_get_name(dev),
132 device_get_unit(dev), "sata_rev", &sata_rev);
133 for (i = 0; i < 16; i++) {
134 ch->user[i].revision = sata_rev;
135 ch->user[i].mode = 0;
136 ch->user[i].bytecount = (ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048;
137 ch->user[i].tags = MVS_MAX_SLOTS;
138 ch->curr[i] = ch->user[i];
139 if (ch->pm_level) {
140 ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
141 CTS_SATA_CAPS_H_APST |
142 CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST;
143 }
144 ch->user[i].caps |= CTS_SATA_CAPS_H_AN;
145 }
146 rid = ch->unit;
147 if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
148 &rid, RF_ACTIVE)))
149 return (ENXIO);
150 mvs_dmainit(dev);
151 mvs_slotsalloc(dev);
152 mvs_ch_init(dev);
153 mtx_lock(&ch->mtx);
154 rid = ATA_IRQ_RID;
155 if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
156 &rid, RF_SHAREABLE | RF_ACTIVE))) {
157 device_printf(dev, "Unable to map interrupt\n");
158 error = ENXIO;
159 goto err0;
160 }
161 if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
162 mvs_ch_intr_locked, dev, &ch->ih))) {
163 device_printf(dev, "Unable to setup interrupt\n");
164 error = ENXIO;
165 goto err1;
166 }
167 /* Create the device queue for our SIM. */
168 devq = cam_simq_alloc(MVS_MAX_SLOTS - 1);
169 if (devq == NULL) {
170 device_printf(dev, "Unable to allocate simq\n");
171 error = ENOMEM;
172 goto err1;
173 }
174 /* Construct SIM entry */
175 ch->sim = cam_sim_alloc(mvsaction, mvspoll, "mvsch", ch,
176 device_get_unit(dev), &ch->mtx,
177 2, (ch->quirks & MVS_Q_GENI) ? 0 : MVS_MAX_SLOTS - 1,
178 devq);
179 if (ch->sim == NULL) {
180 cam_simq_free(devq);
181 device_printf(dev, "unable to allocate sim\n");
182 error = ENOMEM;
183 goto err1;
184 }
185 if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
186 device_printf(dev, "unable to register xpt bus\n");
187 error = ENXIO;
188 goto err2;
189 }
190 if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
191 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
192 device_printf(dev, "unable to create path\n");
193 error = ENXIO;
194 goto err3;
195 }
196 if (ch->pm_level > 3) {
197 callout_reset(&ch->pm_timer,
198 (ch->pm_level == 4) ? hz / 1000 : hz / 8,
199 mvs_ch_pm, dev);
200 }
201 mtx_unlock(&ch->mtx);
202 return (0);
203
204 err3:
205 xpt_bus_deregister(cam_sim_path(ch->sim));
206 err2:
207 cam_sim_free(ch->sim, /*free_devq*/TRUE);
208 err1:
209 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
210 err0:
211 bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
212 mtx_unlock(&ch->mtx);
213 mtx_destroy(&ch->mtx);
214 return (error);
215 }
216
217 static int
mvs_ch_detach(device_t dev)218 mvs_ch_detach(device_t dev)
219 {
220 struct mvs_channel *ch = device_get_softc(dev);
221
222 mtx_lock(&ch->mtx);
223 xpt_async(AC_LOST_DEVICE, ch->path, NULL);
224 /* Forget about reset. */
225 if (ch->resetting) {
226 ch->resetting = 0;
227 xpt_release_simq(ch->sim, TRUE);
228 }
229 xpt_free_path(ch->path);
230 xpt_bus_deregister(cam_sim_path(ch->sim));
231 cam_sim_free(ch->sim, /*free_devq*/TRUE);
232 mtx_unlock(&ch->mtx);
233
234 if (ch->pm_level > 3)
235 callout_drain(&ch->pm_timer);
236 callout_drain(&ch->reset_timer);
237 bus_teardown_intr(dev, ch->r_irq, ch->ih);
238 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
239
240 mvs_ch_deinit(dev);
241 mvs_slotsfree(dev);
242 mvs_dmafini(dev);
243
244 bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
245 mtx_destroy(&ch->mtx);
246 return (0);
247 }
248
249 static int
mvs_ch_init(device_t dev)250 mvs_ch_init(device_t dev)
251 {
252 struct mvs_channel *ch = device_get_softc(dev);
253 uint32_t reg;
254
255 /* Disable port interrupts */
256 ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
257 /* Stop EDMA */
258 ch->curr_mode = MVS_EDMA_UNKNOWN;
259 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
260 /* Clear and configure FIS interrupts. */
261 ATA_OUTL(ch->r_mem, SATA_FISIC, 0);
262 reg = ATA_INL(ch->r_mem, SATA_FISC);
263 reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
264 ATA_OUTL(ch->r_mem, SATA_FISC, reg);
265 reg = ATA_INL(ch->r_mem, SATA_FISIM);
266 reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
267 ATA_OUTL(ch->r_mem, SATA_FISC, reg);
268 /* Clear SATA error register. */
269 ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
270 /* Clear any outstanding error interrupts. */
271 ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
272 /* Unmask all error interrupts */
273 ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
274 return (0);
275 }
276
277 static int
mvs_ch_deinit(device_t dev)278 mvs_ch_deinit(device_t dev)
279 {
280 struct mvs_channel *ch = device_get_softc(dev);
281
282 /* Stop EDMA */
283 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
284 /* Disable port interrupts. */
285 ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
286 return (0);
287 }
288
289 static int
mvs_ch_suspend(device_t dev)290 mvs_ch_suspend(device_t dev)
291 {
292 struct mvs_channel *ch = device_get_softc(dev);
293
294 mtx_lock(&ch->mtx);
295 xpt_freeze_simq(ch->sim, 1);
296 while (ch->oslots)
297 msleep(ch, &ch->mtx, PRIBIO, "mvssusp", hz/100);
298 /* Forget about reset. */
299 if (ch->resetting) {
300 ch->resetting = 0;
301 callout_stop(&ch->reset_timer);
302 xpt_release_simq(ch->sim, TRUE);
303 }
304 mvs_ch_deinit(dev);
305 mtx_unlock(&ch->mtx);
306 return (0);
307 }
308
309 static int
mvs_ch_resume(device_t dev)310 mvs_ch_resume(device_t dev)
311 {
312 struct mvs_channel *ch = device_get_softc(dev);
313
314 mtx_lock(&ch->mtx);
315 mvs_ch_init(dev);
316 mvs_reset(dev);
317 xpt_release_simq(ch->sim, TRUE);
318 mtx_unlock(&ch->mtx);
319 return (0);
320 }
321
322 struct mvs_dc_cb_args {
323 bus_addr_t maddr;
324 int error;
325 };
326
327 static void
mvs_dmainit(device_t dev)328 mvs_dmainit(device_t dev)
329 {
330 struct mvs_channel *ch = device_get_softc(dev);
331 struct mvs_dc_cb_args dcba;
332
333 /* EDMA command request area. */
334 if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
335 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
336 NULL, NULL, MVS_WORKRQ_SIZE, 1, MVS_WORKRQ_SIZE,
337 0, NULL, NULL, &ch->dma.workrq_tag))
338 goto error;
339 if (bus_dmamem_alloc(ch->dma.workrq_tag, (void **)&ch->dma.workrq, 0,
340 &ch->dma.workrq_map))
341 goto error;
342 if (bus_dmamap_load(ch->dma.workrq_tag, ch->dma.workrq_map,
343 ch->dma.workrq, MVS_WORKRQ_SIZE, mvs_dmasetupc_cb, &dcba, 0) ||
344 dcba.error) {
345 bus_dmamem_free(ch->dma.workrq_tag,
346 ch->dma.workrq, ch->dma.workrq_map);
347 goto error;
348 }
349 ch->dma.workrq_bus = dcba.maddr;
350 /* EDMA command response area. */
351 if (bus_dma_tag_create(bus_get_dma_tag(dev), 256, 0,
352 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
353 NULL, NULL, MVS_WORKRP_SIZE, 1, MVS_WORKRP_SIZE,
354 0, NULL, NULL, &ch->dma.workrp_tag))
355 goto error;
356 if (bus_dmamem_alloc(ch->dma.workrp_tag, (void **)&ch->dma.workrp, 0,
357 &ch->dma.workrp_map))
358 goto error;
359 if (bus_dmamap_load(ch->dma.workrp_tag, ch->dma.workrp_map,
360 ch->dma.workrp, MVS_WORKRP_SIZE, mvs_dmasetupc_cb, &dcba, 0) ||
361 dcba.error) {
362 bus_dmamem_free(ch->dma.workrp_tag,
363 ch->dma.workrp, ch->dma.workrp_map);
364 goto error;
365 }
366 ch->dma.workrp_bus = dcba.maddr;
367 /* Data area. */
368 if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, MVS_EPRD_MAX,
369 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
370 NULL, NULL,
371 MVS_SG_ENTRIES * PAGE_SIZE, MVS_SG_ENTRIES, MVS_EPRD_MAX,
372 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
373 goto error;
374 }
375 return;
376
377 error:
378 device_printf(dev, "WARNING - DMA initialization failed\n");
379 mvs_dmafini(dev);
380 }
381
382 static void
mvs_dmasetupc_cb(void * xsc,bus_dma_segment_t * segs,int nsegs,int error)383 mvs_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
384 {
385 struct mvs_dc_cb_args *dcba = (struct mvs_dc_cb_args *)xsc;
386
387 if (!(dcba->error = error))
388 dcba->maddr = segs[0].ds_addr;
389 }
390
391 static void
mvs_dmafini(device_t dev)392 mvs_dmafini(device_t dev)
393 {
394 struct mvs_channel *ch = device_get_softc(dev);
395
396 if (ch->dma.data_tag) {
397 bus_dma_tag_destroy(ch->dma.data_tag);
398 ch->dma.data_tag = NULL;
399 }
400 if (ch->dma.workrp_bus) {
401 bus_dmamap_unload(ch->dma.workrp_tag, ch->dma.workrp_map);
402 bus_dmamem_free(ch->dma.workrp_tag,
403 ch->dma.workrp, ch->dma.workrp_map);
404 ch->dma.workrp_bus = 0;
405 ch->dma.workrp = NULL;
406 }
407 if (ch->dma.workrp_tag) {
408 bus_dma_tag_destroy(ch->dma.workrp_tag);
409 ch->dma.workrp_tag = NULL;
410 }
411 if (ch->dma.workrq_bus) {
412 bus_dmamap_unload(ch->dma.workrq_tag, ch->dma.workrq_map);
413 bus_dmamem_free(ch->dma.workrq_tag,
414 ch->dma.workrq, ch->dma.workrq_map);
415 ch->dma.workrq_bus = 0;
416 ch->dma.workrq = NULL;
417 }
418 if (ch->dma.workrq_tag) {
419 bus_dma_tag_destroy(ch->dma.workrq_tag);
420 ch->dma.workrq_tag = NULL;
421 }
422 }
423
424 static void
mvs_slotsalloc(device_t dev)425 mvs_slotsalloc(device_t dev)
426 {
427 struct mvs_channel *ch = device_get_softc(dev);
428 int i;
429
430 /* Alloc and setup command/dma slots */
431 bzero(ch->slot, sizeof(ch->slot));
432 for (i = 0; i < MVS_MAX_SLOTS; i++) {
433 struct mvs_slot *slot = &ch->slot[i];
434
435 slot->dev = dev;
436 slot->slot = i;
437 slot->state = MVS_SLOT_EMPTY;
438 slot->eprd_offset = MVS_EPRD_OFFSET + MVS_EPRD_SIZE * i;
439 slot->ccb = NULL;
440 callout_init_mtx(&slot->timeout, &ch->mtx, 0);
441
442 if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
443 device_printf(ch->dev, "FAILURE - create data_map\n");
444 }
445 }
446
447 static void
mvs_slotsfree(device_t dev)448 mvs_slotsfree(device_t dev)
449 {
450 struct mvs_channel *ch = device_get_softc(dev);
451 int i;
452
453 /* Free all dma slots */
454 for (i = 0; i < MVS_MAX_SLOTS; i++) {
455 struct mvs_slot *slot = &ch->slot[i];
456
457 callout_drain(&slot->timeout);
458 if (slot->dma.data_map) {
459 bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
460 slot->dma.data_map = NULL;
461 }
462 }
463 }
464
465 static void
mvs_setup_edma_queues(device_t dev)466 mvs_setup_edma_queues(device_t dev)
467 {
468 struct mvs_channel *ch = device_get_softc(dev);
469 uint64_t work;
470
471 /* Requests queue. */
472 work = ch->dma.workrq_bus;
473 ATA_OUTL(ch->r_mem, EDMA_REQQBAH, work >> 32);
474 ATA_OUTL(ch->r_mem, EDMA_REQQIP, work & 0xffffffff);
475 ATA_OUTL(ch->r_mem, EDMA_REQQOP, work & 0xffffffff);
476 bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
477 BUS_DMASYNC_PREWRITE);
478 /* Responses queue. */
479 memset(ch->dma.workrp, 0xff, MVS_WORKRP_SIZE);
480 work = ch->dma.workrp_bus;
481 ATA_OUTL(ch->r_mem, EDMA_RESQBAH, work >> 32);
482 ATA_OUTL(ch->r_mem, EDMA_RESQIP, work & 0xffffffff);
483 ATA_OUTL(ch->r_mem, EDMA_RESQOP, work & 0xffffffff);
484 bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
485 BUS_DMASYNC_PREREAD);
486 ch->out_idx = 0;
487 ch->in_idx = 0;
488 }
489
490 static void
mvs_set_edma_mode(device_t dev,enum mvs_edma_mode mode)491 mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode)
492 {
493 struct mvs_channel *ch = device_get_softc(dev);
494 int timeout;
495 uint32_t ecfg, fcfg, hc, ltm, unkn;
496
497 if (mode == ch->curr_mode)
498 return;
499 /* If we are running, we should stop first. */
500 if (ch->curr_mode != MVS_EDMA_OFF) {
501 ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EDSEDMA);
502 timeout = 0;
503 while (ATA_INL(ch->r_mem, EDMA_CMD) & EDMA_CMD_EENEDMA) {
504 DELAY(1000);
505 if (timeout++ > 1000) {
506 device_printf(dev, "stopping EDMA engine failed\n");
507 break;
508 }
509 }
510 }
511 ch->curr_mode = mode;
512 ch->fbs_enabled = 0;
513 ch->fake_busy = 0;
514 /* Report mode to controller. Needed for correct CCC operation. */
515 MVS_EDMA(device_get_parent(dev), dev, mode);
516 /* Configure new mode. */
517 ecfg = EDMA_CFG_RESERVED | EDMA_CFG_RESERVED2 | EDMA_CFG_EHOSTQUEUECACHEEN;
518 if (ch->pm_present) {
519 ecfg |= EDMA_CFG_EMASKRXPM;
520 if (ch->quirks & MVS_Q_GENIIE) {
521 ecfg |= EDMA_CFG_EEDMAFBS;
522 ch->fbs_enabled = 1;
523 }
524 }
525 if (ch->quirks & MVS_Q_GENI)
526 ecfg |= EDMA_CFG_ERDBSZ;
527 else if (ch->quirks & MVS_Q_GENII)
528 ecfg |= EDMA_CFG_ERDBSZEXT | EDMA_CFG_EWRBUFFERLEN;
529 if (ch->quirks & MVS_Q_CT)
530 ecfg |= EDMA_CFG_ECUTTHROUGHEN;
531 if (mode != MVS_EDMA_OFF)
532 ecfg |= EDMA_CFG_EEARLYCOMPLETIONEN;
533 if (mode == MVS_EDMA_QUEUED)
534 ecfg |= EDMA_CFG_EQUE;
535 else if (mode == MVS_EDMA_NCQ)
536 ecfg |= EDMA_CFG_ESATANATVCMDQUE;
537 ATA_OUTL(ch->r_mem, EDMA_CFG, ecfg);
538 mvs_setup_edma_queues(dev);
539 if (ch->quirks & MVS_Q_GENIIE) {
540 /* Configure FBS-related registers */
541 fcfg = ATA_INL(ch->r_mem, SATA_FISC);
542 ltm = ATA_INL(ch->r_mem, SATA_LTM);
543 hc = ATA_INL(ch->r_mem, EDMA_HC);
544 if (ch->fbs_enabled) {
545 fcfg |= SATA_FISC_FISDMAACTIVATESYNCRESP;
546 if (mode == MVS_EDMA_NCQ) {
547 fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
548 hc &= ~EDMA_IE_EDEVERR;
549 } else {
550 fcfg |= SATA_FISC_FISWAIT4HOSTRDYEN_B0;
551 hc |= EDMA_IE_EDEVERR;
552 }
553 ltm |= (1 << 8);
554 } else {
555 fcfg &= ~SATA_FISC_FISDMAACTIVATESYNCRESP;
556 fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
557 hc |= EDMA_IE_EDEVERR;
558 ltm &= ~(1 << 8);
559 }
560 ATA_OUTL(ch->r_mem, SATA_FISC, fcfg);
561 ATA_OUTL(ch->r_mem, SATA_LTM, ltm);
562 ATA_OUTL(ch->r_mem, EDMA_HC, hc);
563 /* This is some magic, required to handle several DRQs
564 * with basic DMA. */
565 unkn = ATA_INL(ch->r_mem, EDMA_UNKN_RESD);
566 if (mode == MVS_EDMA_OFF)
567 unkn |= 1;
568 else
569 unkn &= ~1;
570 ATA_OUTL(ch->r_mem, EDMA_UNKN_RESD, unkn);
571 }
572 /* Run EDMA. */
573 if (mode != MVS_EDMA_OFF)
574 ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EENEDMA);
575 }
576
577 static device_method_t mvsch_methods[] = {
578 DEVMETHOD(device_probe, mvs_ch_probe),
579 DEVMETHOD(device_attach, mvs_ch_attach),
580 DEVMETHOD(device_detach, mvs_ch_detach),
581 DEVMETHOD(device_suspend, mvs_ch_suspend),
582 DEVMETHOD(device_resume, mvs_ch_resume),
583 { 0, 0 }
584 };
585 static driver_t mvsch_driver = {
586 "mvsch",
587 mvsch_methods,
588 sizeof(struct mvs_channel)
589 };
590 DRIVER_MODULE(mvsch, mvs, mvsch_driver, 0, 0);
591 DRIVER_MODULE(mvsch, sata, mvsch_driver, 0, 0);
592
593 static void
mvs_phy_check_events(device_t dev,u_int32_t serr)594 mvs_phy_check_events(device_t dev, u_int32_t serr)
595 {
596 struct mvs_channel *ch = device_get_softc(dev);
597
598 if (ch->pm_level == 0) {
599 u_int32_t status = ATA_INL(ch->r_mem, SATA_SS);
600 union ccb *ccb;
601
602 if (bootverbose) {
603 if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
604 ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
605 ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE)) {
606 device_printf(dev, "CONNECT requested\n");
607 } else
608 device_printf(dev, "DISCONNECT requested\n");
609 }
610 mvs_reset(dev);
611 if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
612 return;
613 if (xpt_create_path(&ccb->ccb_h.path, NULL,
614 cam_sim_path(ch->sim),
615 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
616 xpt_free_ccb(ccb);
617 return;
618 }
619 xpt_rescan(ccb);
620 }
621 }
622
623 static void
mvs_notify_events(device_t dev)624 mvs_notify_events(device_t dev)
625 {
626 struct mvs_channel *ch = device_get_softc(dev);
627 struct cam_path *dpath;
628 uint32_t fis;
629 int d;
630
631 /* Try to read PMP field from SDB FIS. Present only for Gen-IIe. */
632 fis = ATA_INL(ch->r_mem, SATA_FISDW0);
633 if ((fis & 0x80ff) == 0x80a1)
634 d = (fis & 0x0f00) >> 8;
635 else
636 d = ch->pm_present ? 15 : 0;
637 if (bootverbose)
638 device_printf(dev, "SNTF %d\n", d);
639 if (xpt_create_path(&dpath, NULL,
640 xpt_path_path_id(ch->path), d, 0) == CAM_REQ_CMP) {
641 xpt_async(AC_SCSI_AEN, dpath, NULL);
642 xpt_free_path(dpath);
643 }
644 }
645
646 static void
mvs_ch_intr_locked(void * data)647 mvs_ch_intr_locked(void *data)
648 {
649 struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
650 device_t dev = (device_t)arg->arg;
651 struct mvs_channel *ch = device_get_softc(dev);
652
653 mtx_lock(&ch->mtx);
654 mvs_ch_intr(data);
655 mtx_unlock(&ch->mtx);
656 }
657
658 static void
mvs_ch_pm(void * arg)659 mvs_ch_pm(void *arg)
660 {
661 device_t dev = (device_t)arg;
662 struct mvs_channel *ch = device_get_softc(dev);
663 uint32_t work;
664
665 if (ch->numrslots != 0)
666 return;
667 /* If we are idle - request power state transition. */
668 work = ATA_INL(ch->r_mem, SATA_SC);
669 work &= ~SATA_SC_SPM_MASK;
670 if (ch->pm_level == 4)
671 work |= SATA_SC_SPM_PARTIAL;
672 else
673 work |= SATA_SC_SPM_SLUMBER;
674 ATA_OUTL(ch->r_mem, SATA_SC, work);
675 }
676
677 static void
mvs_ch_pm_wake(device_t dev)678 mvs_ch_pm_wake(device_t dev)
679 {
680 struct mvs_channel *ch = device_get_softc(dev);
681 uint32_t work;
682 int timeout = 0;
683
684 work = ATA_INL(ch->r_mem, SATA_SS);
685 if (work & SATA_SS_IPM_ACTIVE)
686 return;
687 /* If we are not in active state - request power state transition. */
688 work = ATA_INL(ch->r_mem, SATA_SC);
689 work &= ~SATA_SC_SPM_MASK;
690 work |= SATA_SC_SPM_ACTIVE;
691 ATA_OUTL(ch->r_mem, SATA_SC, work);
692 /* Wait for transition to happen. */
693 while ((ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_IPM_ACTIVE) == 0 &&
694 timeout++ < 100) {
695 DELAY(100);
696 }
697 }
698
699 static void
mvs_ch_intr(void * data)700 mvs_ch_intr(void *data)
701 {
702 struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
703 device_t dev = (device_t)arg->arg;
704 struct mvs_channel *ch = device_get_softc(dev);
705 uint32_t iec, serr = 0, fisic = 0;
706 enum mvs_err_type et;
707 int i, ccs, port = -1, selfdis = 0;
708 int edma = (ch->numtslots != 0 || ch->numdslots != 0);
709
710 /* New item in response queue. */
711 if ((arg->cause & 2) && edma)
712 mvs_crbq_intr(dev);
713 /* Some error or special event. */
714 if (arg->cause & 1) {
715 iec = ATA_INL(ch->r_mem, EDMA_IEC);
716 if (iec & EDMA_IE_SERRINT) {
717 serr = ATA_INL(ch->r_mem, SATA_SE);
718 ATA_OUTL(ch->r_mem, SATA_SE, serr);
719 }
720 /* EDMA self-disabled due to error. */
721 if (iec & EDMA_IE_ESELFDIS)
722 selfdis = 1;
723 /* Transport interrupt. */
724 if (iec & EDMA_IE_ETRANSINT) {
725 /* For Gen-I this bit means self-disable. */
726 if (ch->quirks & MVS_Q_GENI)
727 selfdis = 1;
728 /* For Gen-II this bit means SDB-N. */
729 else if (ch->quirks & MVS_Q_GENII)
730 fisic = SATA_FISC_FISWAIT4HOSTRDYEN_B1;
731 else /* For Gen-IIe - read FIS interrupt cause. */
732 fisic = ATA_INL(ch->r_mem, SATA_FISIC);
733 }
734 if (selfdis)
735 ch->curr_mode = MVS_EDMA_UNKNOWN;
736 ATA_OUTL(ch->r_mem, EDMA_IEC, ~iec);
737 /* Interface errors or Device error. */
738 if (iec & (0xfc1e9000 | EDMA_IE_EDEVERR)) {
739 port = -1;
740 if (ch->numpslots != 0) {
741 ccs = 0;
742 } else {
743 if (ch->quirks & MVS_Q_GENIIE)
744 ccs = EDMA_S_EIOID(ATA_INL(ch->r_mem, EDMA_S));
745 else
746 ccs = EDMA_S_EDEVQUETAG(ATA_INL(ch->r_mem, EDMA_S));
747 /* Check if error is one-PMP-port-specific, */
748 if (ch->fbs_enabled) {
749 /* Which ports were active. */
750 for (i = 0; i < 16; i++) {
751 if (ch->numrslotspd[i] == 0)
752 continue;
753 if (port == -1)
754 port = i;
755 else if (port != i) {
756 port = -2;
757 break;
758 }
759 }
760 /* If several ports were active and EDMA still enabled -
761 * other ports are probably unaffected and may continue.
762 */
763 if (port == -2 && !selfdis) {
764 uint16_t p = ATA_INL(ch->r_mem, SATA_SATAITC) >> 16;
765 port = ffs(p) - 1;
766 if (port != (fls(p) - 1))
767 port = -2;
768 }
769 }
770 }
771 mvs_requeue_frozen(dev);
772 for (i = 0; i < MVS_MAX_SLOTS; i++) {
773 /* XXX: requests in loading state. */
774 if (((ch->rslots >> i) & 1) == 0)
775 continue;
776 if (port >= 0 &&
777 ch->slot[i].ccb->ccb_h.target_id != port)
778 continue;
779 if (iec & EDMA_IE_EDEVERR) { /* Device error. */
780 if (port != -2) {
781 if (ch->numtslots == 0) {
782 /* Untagged operation. */
783 if (i == ccs)
784 et = MVS_ERR_TFE;
785 else
786 et = MVS_ERR_INNOCENT;
787 } else {
788 /* Tagged operation. */
789 et = MVS_ERR_NCQ;
790 }
791 } else {
792 et = MVS_ERR_TFE;
793 ch->fatalerr = 1;
794 }
795 } else if (iec & 0xfc1e9000) {
796 if (ch->numtslots == 0 &&
797 i != ccs && port != -2)
798 et = MVS_ERR_INNOCENT;
799 else
800 et = MVS_ERR_SATA;
801 } else
802 et = MVS_ERR_INVALID;
803 mvs_end_transaction(&ch->slot[i], et);
804 }
805 }
806 /* Process SDB-N. */
807 if (fisic & SATA_FISC_FISWAIT4HOSTRDYEN_B1)
808 mvs_notify_events(dev);
809 if (fisic)
810 ATA_OUTL(ch->r_mem, SATA_FISIC, ~fisic);
811 /* Process hot-plug. */
812 if ((iec & (EDMA_IE_EDEVDIS | EDMA_IE_EDEVCON)) ||
813 (serr & SATA_SE_PHY_CHANGED))
814 mvs_phy_check_events(dev, serr);
815 }
816 /* Legacy mode device interrupt. */
817 if ((arg->cause & 2) && !edma)
818 mvs_legacy_intr(dev, arg->cause & 4);
819 }
820
821 static uint8_t
mvs_getstatus(device_t dev,int clear)822 mvs_getstatus(device_t dev, int clear)
823 {
824 struct mvs_channel *ch = device_get_softc(dev);
825 uint8_t status = ATA_INB(ch->r_mem, clear ? ATA_STATUS : ATA_ALTSTAT);
826
827 if (ch->fake_busy) {
828 if (status & (ATA_S_BUSY | ATA_S_DRQ | ATA_S_ERROR))
829 ch->fake_busy = 0;
830 else
831 status |= ATA_S_BUSY;
832 }
833 return (status);
834 }
835
836 static void
mvs_legacy_intr(device_t dev,int poll)837 mvs_legacy_intr(device_t dev, int poll)
838 {
839 struct mvs_channel *ch = device_get_softc(dev);
840 struct mvs_slot *slot = &ch->slot[0]; /* PIO is always in slot 0. */
841 union ccb *ccb = slot->ccb;
842 enum mvs_err_type et = MVS_ERR_NONE;
843 u_int length, resid, size;
844 uint8_t buf[2];
845 uint8_t status, ireason;
846
847 /* Clear interrupt and get status. */
848 status = mvs_getstatus(dev, 1);
849 if (slot->state < MVS_SLOT_RUNNING)
850 return;
851 /* Wait a bit for late !BUSY status update. */
852 if (status & ATA_S_BUSY) {
853 if (poll)
854 return;
855 DELAY(100);
856 if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY) {
857 DELAY(1000);
858 if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY)
859 return;
860 }
861 }
862 /* If we got an error, we are done. */
863 if (status & ATA_S_ERROR) {
864 et = MVS_ERR_TFE;
865 goto end_finished;
866 }
867 if (ccb->ccb_h.func_code == XPT_ATA_IO) { /* ATA PIO */
868 ccb->ataio.res.status = status;
869 /* Are we moving data? */
870 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
871 /* If data read command - get them. */
872 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
873 if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
874 device_printf(dev, "timeout waiting for read DRQ\n");
875 et = MVS_ERR_TIMEOUT;
876 xpt_freeze_simq(ch->sim, 1);
877 ch->toslots |= (1 << slot->slot);
878 goto end_finished;
879 }
880 ATA_INSW_STRM(ch->r_mem, ATA_DATA,
881 (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
882 ch->transfersize / 2);
883 }
884 /* Update how far we've gotten. */
885 ch->donecount += ch->transfersize;
886 /* Do we need more? */
887 if (ccb->ataio.dxfer_len > ch->donecount) {
888 /* Set this transfer size according to HW capabilities */
889 ch->transfersize = min(ccb->ataio.dxfer_len - ch->donecount,
890 ch->transfersize);
891 /* If data write command - put them */
892 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
893 if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
894 device_printf(dev,
895 "timeout waiting for write DRQ\n");
896 et = MVS_ERR_TIMEOUT;
897 xpt_freeze_simq(ch->sim, 1);
898 ch->toslots |= (1 << slot->slot);
899 goto end_finished;
900 }
901 ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
902 (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
903 ch->transfersize / 2);
904 return;
905 }
906 /* If data read command, return & wait for interrupt */
907 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
908 return;
909 }
910 }
911 } else if (ch->basic_dma) { /* ATAPI DMA */
912 if (status & ATA_S_DWF)
913 et = MVS_ERR_TFE;
914 else if (ATA_INL(ch->r_mem, DMA_S) & DMA_S_ERR)
915 et = MVS_ERR_TFE;
916 /* Stop basic DMA. */
917 ATA_OUTL(ch->r_mem, DMA_C, 0);
918 goto end_finished;
919 } else { /* ATAPI PIO */
920 length = ATA_INB(ch->r_mem,ATA_CYL_LSB) |
921 (ATA_INB(ch->r_mem,ATA_CYL_MSB) << 8);
922 size = min(ch->transfersize, length);
923 ireason = ATA_INB(ch->r_mem,ATA_IREASON);
924 switch ((ireason & (ATA_I_CMD | ATA_I_IN)) |
925 (status & ATA_S_DRQ)) {
926 case ATAPI_P_CMDOUT:
927 device_printf(dev, "ATAPI CMDOUT\n");
928 /* Return wait for interrupt */
929 return;
930
931 case ATAPI_P_WRITE:
932 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
933 device_printf(dev, "trying to write on read buffer\n");
934 et = MVS_ERR_TFE;
935 goto end_finished;
936 break;
937 }
938 ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
939 (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
940 (size + 1) / 2);
941 for (resid = ch->transfersize + (size & 1);
942 resid < length; resid += sizeof(int16_t))
943 ATA_OUTW(ch->r_mem, ATA_DATA, 0);
944 ch->donecount += length;
945 /* Set next transfer size according to HW capabilities */
946 ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
947 ch->curr[ccb->ccb_h.target_id].bytecount);
948 /* Return wait for interrupt */
949 return;
950
951 case ATAPI_P_READ:
952 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
953 device_printf(dev, "trying to read on write buffer\n");
954 et = MVS_ERR_TFE;
955 goto end_finished;
956 }
957 if (size >= 2) {
958 ATA_INSW_STRM(ch->r_mem, ATA_DATA,
959 (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
960 size / 2);
961 }
962 if (size & 1) {
963 ATA_INSW_STRM(ch->r_mem, ATA_DATA, (void*)buf, 1);
964 ((uint8_t *)ccb->csio.data_ptr + ch->donecount +
965 (size & ~1))[0] = buf[0];
966 }
967 for (resid = ch->transfersize + (size & 1);
968 resid < length; resid += sizeof(int16_t))
969 ATA_INW(ch->r_mem, ATA_DATA);
970 ch->donecount += length;
971 /* Set next transfer size according to HW capabilities */
972 ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
973 ch->curr[ccb->ccb_h.target_id].bytecount);
974 /* Return wait for interrupt */
975 return;
976
977 case ATAPI_P_DONEDRQ:
978 device_printf(dev,
979 "WARNING - DONEDRQ non conformant device\n");
980 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
981 ATA_INSW_STRM(ch->r_mem, ATA_DATA,
982 (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
983 length / 2);
984 ch->donecount += length;
985 }
986 else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
987 ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
988 (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
989 length / 2);
990 ch->donecount += length;
991 }
992 else
993 et = MVS_ERR_TFE;
994 /* FALLTHROUGH */
995
996 case ATAPI_P_ABORT:
997 case ATAPI_P_DONE:
998 if (status & (ATA_S_ERROR | ATA_S_DWF))
999 et = MVS_ERR_TFE;
1000 goto end_finished;
1001
1002 default:
1003 device_printf(dev, "unknown transfer phase"
1004 " (status %02x, ireason %02x)\n",
1005 status, ireason);
1006 et = MVS_ERR_TFE;
1007 }
1008 }
1009
1010 end_finished:
1011 mvs_end_transaction(slot, et);
1012 }
1013
1014 static void
mvs_crbq_intr(device_t dev)1015 mvs_crbq_intr(device_t dev)
1016 {
1017 struct mvs_channel *ch = device_get_softc(dev);
1018 struct mvs_crpb *crpb;
1019 union ccb *ccb;
1020 int in_idx, fin_idx, cin_idx, slot;
1021 uint32_t val;
1022 uint16_t flags;
1023
1024 val = ATA_INL(ch->r_mem, EDMA_RESQIP);
1025 if (val == 0)
1026 val = ATA_INL(ch->r_mem, EDMA_RESQIP);
1027 in_idx = (val & EDMA_RESQP_ERPQP_MASK) >>
1028 EDMA_RESQP_ERPQP_SHIFT;
1029 bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1030 BUS_DMASYNC_POSTREAD);
1031 fin_idx = cin_idx = ch->in_idx;
1032 ch->in_idx = in_idx;
1033 while (in_idx != cin_idx) {
1034 crpb = (struct mvs_crpb *)
1035 (ch->dma.workrp + MVS_CRPB_OFFSET +
1036 (MVS_CRPB_SIZE * cin_idx));
1037 slot = le16toh(crpb->id) & MVS_CRPB_TAG_MASK;
1038 flags = le16toh(crpb->rspflg);
1039 /*
1040 * Handle only successful completions here.
1041 * Errors will be handled by main intr handler.
1042 */
1043 #if defined(__i386__) || defined(__amd64__)
1044 if (crpb->id == 0xffff && crpb->rspflg == 0xffff) {
1045 device_printf(dev, "Unfilled CRPB "
1046 "%d (%d->%d) tag %d flags %04x rs %08x\n",
1047 cin_idx, fin_idx, in_idx, slot, flags, ch->rslots);
1048 } else
1049 #endif
1050 if (ch->numtslots != 0 ||
1051 (flags & EDMA_IE_EDEVERR) == 0) {
1052 #if defined(__i386__) || defined(__amd64__)
1053 crpb->id = 0xffff;
1054 crpb->rspflg = 0xffff;
1055 #endif
1056 if (ch->slot[slot].state >= MVS_SLOT_RUNNING) {
1057 ccb = ch->slot[slot].ccb;
1058 ccb->ataio.res.status =
1059 (flags & MVS_CRPB_ATASTS_MASK) >>
1060 MVS_CRPB_ATASTS_SHIFT;
1061 mvs_end_transaction(&ch->slot[slot], MVS_ERR_NONE);
1062 } else {
1063 device_printf(dev, "Unused tag in CRPB "
1064 "%d (%d->%d) tag %d flags %04x rs %08x\n",
1065 cin_idx, fin_idx, in_idx, slot, flags,
1066 ch->rslots);
1067 }
1068 } else {
1069 device_printf(dev,
1070 "CRPB with error %d tag %d flags %04x\n",
1071 cin_idx, slot, flags);
1072 }
1073 cin_idx = (cin_idx + 1) & (MVS_MAX_SLOTS - 1);
1074 }
1075 bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1076 BUS_DMASYNC_PREREAD);
1077 if (cin_idx == ch->in_idx) {
1078 ATA_OUTL(ch->r_mem, EDMA_RESQOP,
1079 ch->dma.workrp_bus | (cin_idx << EDMA_RESQP_ERPQP_SHIFT));
1080 }
1081 }
1082
1083 /* Must be called with channel locked. */
1084 static int
mvs_check_collision(device_t dev,union ccb * ccb)1085 mvs_check_collision(device_t dev, union ccb *ccb)
1086 {
1087 struct mvs_channel *ch = device_get_softc(dev);
1088
1089 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1090 /* NCQ DMA */
1091 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1092 /* Can't mix NCQ and non-NCQ DMA commands. */
1093 if (ch->numdslots != 0)
1094 return (1);
1095 /* Can't mix NCQ and PIO commands. */
1096 if (ch->numpslots != 0)
1097 return (1);
1098 /* If we have no FBS */
1099 if (!ch->fbs_enabled) {
1100 /* Tagged command while tagged to other target is active. */
1101 if (ch->numtslots != 0 &&
1102 ch->taggedtarget != ccb->ccb_h.target_id)
1103 return (1);
1104 }
1105 /* Non-NCQ DMA */
1106 } else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1107 /* Can't mix non-NCQ DMA and NCQ commands. */
1108 if (ch->numtslots != 0)
1109 return (1);
1110 /* Can't mix non-NCQ DMA and PIO commands. */
1111 if (ch->numpslots != 0)
1112 return (1);
1113 /* PIO */
1114 } else {
1115 /* Can't mix PIO with anything. */
1116 if (ch->numrslots != 0)
1117 return (1);
1118 }
1119 if (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
1120 /* Atomic command while anything active. */
1121 if (ch->numrslots != 0)
1122 return (1);
1123 }
1124 } else { /* ATAPI */
1125 /* ATAPI goes without EDMA, so can't mix it with anything. */
1126 if (ch->numrslots != 0)
1127 return (1);
1128 }
1129 /* We have some atomic command running. */
1130 if (ch->aslots != 0)
1131 return (1);
1132 return (0);
1133 }
1134
1135 static void
mvs_tfd_read(device_t dev,union ccb * ccb)1136 mvs_tfd_read(device_t dev, union ccb *ccb)
1137 {
1138 struct mvs_channel *ch = device_get_softc(dev);
1139 struct ata_res *res = &ccb->ataio.res;
1140
1141 res->status = ATA_INB(ch->r_mem, ATA_ALTSTAT);
1142 res->error = ATA_INB(ch->r_mem, ATA_ERROR);
1143 res->device = ATA_INB(ch->r_mem, ATA_DRIVE);
1144 ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_HOB);
1145 res->sector_count_exp = ATA_INB(ch->r_mem, ATA_COUNT);
1146 res->lba_low_exp = ATA_INB(ch->r_mem, ATA_SECTOR);
1147 res->lba_mid_exp = ATA_INB(ch->r_mem, ATA_CYL_LSB);
1148 res->lba_high_exp = ATA_INB(ch->r_mem, ATA_CYL_MSB);
1149 ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
1150 res->sector_count = ATA_INB(ch->r_mem, ATA_COUNT);
1151 res->lba_low = ATA_INB(ch->r_mem, ATA_SECTOR);
1152 res->lba_mid = ATA_INB(ch->r_mem, ATA_CYL_LSB);
1153 res->lba_high = ATA_INB(ch->r_mem, ATA_CYL_MSB);
1154 }
1155
1156 static void
mvs_tfd_write(device_t dev,union ccb * ccb)1157 mvs_tfd_write(device_t dev, union ccb *ccb)
1158 {
1159 struct mvs_channel *ch = device_get_softc(dev);
1160 struct ata_cmd *cmd = &ccb->ataio.cmd;
1161
1162 ATA_OUTB(ch->r_mem, ATA_DRIVE, cmd->device);
1163 ATA_OUTB(ch->r_mem, ATA_CONTROL, cmd->control);
1164 ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features_exp);
1165 ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features);
1166 ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count_exp);
1167 ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count);
1168 ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low_exp);
1169 ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low);
1170 ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid_exp);
1171 ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid);
1172 ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high_exp);
1173 ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high);
1174 ATA_OUTB(ch->r_mem, ATA_COMMAND, cmd->command);
1175 }
1176
1177 /* Must be called with channel locked. */
1178 static void
mvs_begin_transaction(device_t dev,union ccb * ccb)1179 mvs_begin_transaction(device_t dev, union ccb *ccb)
1180 {
1181 struct mvs_channel *ch = device_get_softc(dev);
1182 struct mvs_slot *slot;
1183 int slotn, tag;
1184
1185 if (ch->pm_level > 0)
1186 mvs_ch_pm_wake(dev);
1187 /* Softreset is a special case. */
1188 if (ccb->ccb_h.func_code == XPT_ATA_IO &&
1189 (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
1190 mvs_softreset(dev, ccb);
1191 return;
1192 }
1193 /* Choose empty slot. */
1194 slotn = ffs(~ch->oslots) - 1;
1195 if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
1196 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
1197 if (ch->quirks & MVS_Q_GENIIE)
1198 tag = ffs(~ch->otagspd[ccb->ccb_h.target_id]) - 1;
1199 else
1200 tag = slotn;
1201 } else
1202 tag = 0;
1203 /* Occupy chosen slot. */
1204 slot = &ch->slot[slotn];
1205 slot->ccb = ccb;
1206 slot->tag = tag;
1207 /* Stop PM timer. */
1208 if (ch->numrslots == 0 && ch->pm_level > 3)
1209 callout_stop(&ch->pm_timer);
1210 /* Update channel stats. */
1211 ch->oslots |= (1 << slot->slot);
1212 ch->numrslots++;
1213 ch->numrslotspd[ccb->ccb_h.target_id]++;
1214 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1215 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1216 ch->otagspd[ccb->ccb_h.target_id] |= (1 << slot->tag);
1217 ch->numtslots++;
1218 ch->numtslotspd[ccb->ccb_h.target_id]++;
1219 ch->taggedtarget = ccb->ccb_h.target_id;
1220 mvs_set_edma_mode(dev, MVS_EDMA_NCQ);
1221 } else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1222 ch->numdslots++;
1223 mvs_set_edma_mode(dev, MVS_EDMA_ON);
1224 } else {
1225 ch->numpslots++;
1226 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1227 }
1228 if (ccb->ataio.cmd.flags &
1229 (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
1230 ch->aslots |= (1 << slot->slot);
1231 }
1232 } else {
1233 uint8_t *cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1234 ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
1235 ch->numpslots++;
1236 /* Use ATAPI DMA only for commands without under-/overruns. */
1237 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1238 ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA &&
1239 (ch->quirks & MVS_Q_SOC) == 0 &&
1240 (cdb[0] == 0x08 ||
1241 cdb[0] == 0x0a ||
1242 cdb[0] == 0x28 ||
1243 cdb[0] == 0x2a ||
1244 cdb[0] == 0x88 ||
1245 cdb[0] == 0x8a ||
1246 cdb[0] == 0xa8 ||
1247 cdb[0] == 0xaa ||
1248 cdb[0] == 0xbe)) {
1249 ch->basic_dma = 1;
1250 }
1251 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1252 }
1253 if (ch->numpslots == 0 || ch->basic_dma) {
1254 slot->state = MVS_SLOT_LOADING;
1255 bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map,
1256 ccb, mvs_dmasetprd, slot, 0);
1257 } else
1258 mvs_legacy_execute_transaction(slot);
1259 }
1260
1261 /* Locked by busdma engine. */
1262 static void
mvs_dmasetprd(void * arg,bus_dma_segment_t * segs,int nsegs,int error)1263 mvs_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1264 {
1265 struct mvs_slot *slot = arg;
1266 struct mvs_channel *ch = device_get_softc(slot->dev);
1267 struct mvs_eprd *eprd;
1268 int i;
1269
1270 if (error) {
1271 device_printf(slot->dev, "DMA load error\n");
1272 mvs_end_transaction(slot, MVS_ERR_INVALID);
1273 return;
1274 }
1275 KASSERT(nsegs <= MVS_SG_ENTRIES, ("too many DMA segment entries\n"));
1276 /* If there is only one segment - no need to use S/G table on Gen-IIe. */
1277 if (nsegs == 1 && ch->basic_dma == 0 && (ch->quirks & MVS_Q_GENIIE)) {
1278 slot->dma.addr = segs[0].ds_addr;
1279 slot->dma.len = segs[0].ds_len;
1280 } else {
1281 slot->dma.addr = 0;
1282 /* Get a piece of the workspace for this EPRD */
1283 eprd = (struct mvs_eprd *)(ch->dma.workrq + slot->eprd_offset);
1284 /* Fill S/G table */
1285 for (i = 0; i < nsegs; i++) {
1286 eprd[i].prdbal = htole32(segs[i].ds_addr);
1287 eprd[i].bytecount = htole32(segs[i].ds_len & MVS_EPRD_MASK);
1288 eprd[i].prdbah = htole32((segs[i].ds_addr >> 16) >> 16);
1289 }
1290 eprd[i - 1].bytecount |= htole32(MVS_EPRD_EOF);
1291 }
1292 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1293 ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
1294 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
1295 if (ch->basic_dma)
1296 mvs_legacy_execute_transaction(slot);
1297 else
1298 mvs_execute_transaction(slot);
1299 }
1300
1301 static void
mvs_legacy_execute_transaction(struct mvs_slot * slot)1302 mvs_legacy_execute_transaction(struct mvs_slot *slot)
1303 {
1304 device_t dev = slot->dev;
1305 struct mvs_channel *ch = device_get_softc(dev);
1306 bus_addr_t eprd;
1307 union ccb *ccb = slot->ccb;
1308 int port = ccb->ccb_h.target_id & 0x0f;
1309 int timeout;
1310
1311 slot->state = MVS_SLOT_RUNNING;
1312 ch->rslots |= (1 << slot->slot);
1313 ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
1314 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1315 mvs_tfd_write(dev, ccb);
1316 /* Device reset doesn't interrupt. */
1317 if (ccb->ataio.cmd.command == ATA_DEVICE_RESET) {
1318 int timeout = 1000000;
1319 do {
1320 DELAY(10);
1321 ccb->ataio.res.status = ATA_INB(ch->r_mem, ATA_STATUS);
1322 } while (ccb->ataio.res.status & ATA_S_BUSY && timeout--);
1323 mvs_legacy_intr(dev, 1);
1324 return;
1325 }
1326 ch->donecount = 0;
1327 if (ccb->ataio.cmd.command == ATA_READ_MUL ||
1328 ccb->ataio.cmd.command == ATA_READ_MUL48 ||
1329 ccb->ataio.cmd.command == ATA_WRITE_MUL ||
1330 ccb->ataio.cmd.command == ATA_WRITE_MUL48) {
1331 ch->transfersize = min(ccb->ataio.dxfer_len,
1332 ch->curr[port].bytecount);
1333 } else
1334 ch->transfersize = min(ccb->ataio.dxfer_len, 512);
1335 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE)
1336 ch->fake_busy = 1;
1337 /* If data write command - output the data */
1338 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
1339 if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
1340 device_printf(dev,
1341 "timeout waiting for write DRQ\n");
1342 xpt_freeze_simq(ch->sim, 1);
1343 ch->toslots |= (1 << slot->slot);
1344 mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1345 return;
1346 }
1347 ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
1348 (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
1349 ch->transfersize / 2);
1350 }
1351 } else {
1352 ch->donecount = 0;
1353 ch->transfersize = min(ccb->csio.dxfer_len,
1354 ch->curr[port].bytecount);
1355 /* Write ATA PACKET command. */
1356 if (ch->basic_dma) {
1357 ATA_OUTB(ch->r_mem, ATA_FEATURE, ATA_F_DMA);
1358 ATA_OUTB(ch->r_mem, ATA_CYL_LSB, 0);
1359 ATA_OUTB(ch->r_mem, ATA_CYL_MSB, 0);
1360 } else {
1361 ATA_OUTB(ch->r_mem, ATA_FEATURE, 0);
1362 ATA_OUTB(ch->r_mem, ATA_CYL_LSB, ch->transfersize);
1363 ATA_OUTB(ch->r_mem, ATA_CYL_MSB, ch->transfersize >> 8);
1364 }
1365 ATA_OUTB(ch->r_mem, ATA_COMMAND, ATA_PACKET_CMD);
1366 ch->fake_busy = 1;
1367 /* Wait for ready to write ATAPI command block */
1368 if (mvs_wait(dev, 0, ATA_S_BUSY, 1000) < 0) {
1369 device_printf(dev, "timeout waiting for ATAPI !BUSY\n");
1370 xpt_freeze_simq(ch->sim, 1);
1371 ch->toslots |= (1 << slot->slot);
1372 mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1373 return;
1374 }
1375 timeout = 5000;
1376 while (timeout--) {
1377 int reason = ATA_INB(ch->r_mem, ATA_IREASON);
1378 int status = ATA_INB(ch->r_mem, ATA_STATUS);
1379
1380 if (((reason & (ATA_I_CMD | ATA_I_IN)) |
1381 (status & (ATA_S_DRQ | ATA_S_BUSY))) == ATAPI_P_CMDOUT)
1382 break;
1383 DELAY(20);
1384 }
1385 if (timeout <= 0) {
1386 device_printf(dev,
1387 "timeout waiting for ATAPI command ready\n");
1388 xpt_freeze_simq(ch->sim, 1);
1389 ch->toslots |= (1 << slot->slot);
1390 mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
1391 return;
1392 }
1393 /* Write ATAPI command. */
1394 ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
1395 (uint16_t *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
1396 ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes),
1397 ch->curr[port].atapi / 2);
1398 DELAY(10);
1399 if (ch->basic_dma) {
1400 /* Start basic DMA. */
1401 eprd = ch->dma.workrq_bus + slot->eprd_offset;
1402 ATA_OUTL(ch->r_mem, DMA_DTLBA, eprd);
1403 ATA_OUTL(ch->r_mem, DMA_DTHBA, (eprd >> 16) >> 16);
1404 ATA_OUTL(ch->r_mem, DMA_C, DMA_C_START |
1405 (((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) ?
1406 DMA_C_READ : 0));
1407 }
1408 }
1409 /* Start command execution timeout */
1410 callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout, 0,
1411 mvs_timeout, slot, 0);
1412 }
1413
1414 /* Must be called with channel locked. */
1415 static void
mvs_execute_transaction(struct mvs_slot * slot)1416 mvs_execute_transaction(struct mvs_slot *slot)
1417 {
1418 device_t dev = slot->dev;
1419 struct mvs_channel *ch = device_get_softc(dev);
1420 bus_addr_t eprd;
1421 struct mvs_crqb *crqb;
1422 struct mvs_crqb_gen2e *crqb2e;
1423 union ccb *ccb = slot->ccb;
1424 int port = ccb->ccb_h.target_id & 0x0f;
1425 int i;
1426
1427 /* Get address of the prepared EPRD */
1428 eprd = ch->dma.workrq_bus + slot->eprd_offset;
1429 /* Prepare CRQB. Gen IIe uses different CRQB format. */
1430 if (ch->quirks & MVS_Q_GENIIE) {
1431 crqb2e = (struct mvs_crqb_gen2e *)
1432 (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1433 crqb2e->ctrlflg = htole32(
1434 ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB2E_READ : 0) |
1435 (slot->tag << MVS_CRQB2E_DTAG_SHIFT) |
1436 (port << MVS_CRQB2E_PMP_SHIFT) |
1437 (slot->slot << MVS_CRQB2E_HTAG_SHIFT));
1438 /* If there is only one segment - no need to use S/G table. */
1439 if (slot->dma.addr != 0) {
1440 eprd = slot->dma.addr;
1441 crqb2e->ctrlflg |= htole32(MVS_CRQB2E_CPRD);
1442 crqb2e->drbc = slot->dma.len;
1443 }
1444 crqb2e->cprdbl = htole32(eprd);
1445 crqb2e->cprdbh = htole32((eprd >> 16) >> 16);
1446 crqb2e->cmd[0] = 0;
1447 crqb2e->cmd[1] = 0;
1448 crqb2e->cmd[2] = ccb->ataio.cmd.command;
1449 crqb2e->cmd[3] = ccb->ataio.cmd.features;
1450 crqb2e->cmd[4] = ccb->ataio.cmd.lba_low;
1451 crqb2e->cmd[5] = ccb->ataio.cmd.lba_mid;
1452 crqb2e->cmd[6] = ccb->ataio.cmd.lba_high;
1453 crqb2e->cmd[7] = ccb->ataio.cmd.device;
1454 crqb2e->cmd[8] = ccb->ataio.cmd.lba_low_exp;
1455 crqb2e->cmd[9] = ccb->ataio.cmd.lba_mid_exp;
1456 crqb2e->cmd[10] = ccb->ataio.cmd.lba_high_exp;
1457 crqb2e->cmd[11] = ccb->ataio.cmd.features_exp;
1458 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1459 crqb2e->cmd[12] = slot->tag << 3;
1460 crqb2e->cmd[13] = 0;
1461 } else {
1462 crqb2e->cmd[12] = ccb->ataio.cmd.sector_count;
1463 crqb2e->cmd[13] = ccb->ataio.cmd.sector_count_exp;
1464 }
1465 crqb2e->cmd[14] = 0;
1466 crqb2e->cmd[15] = 0;
1467 } else {
1468 crqb = (struct mvs_crqb *)
1469 (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1470 crqb->cprdbl = htole32(eprd);
1471 crqb->cprdbh = htole32((eprd >> 16) >> 16);
1472 crqb->ctrlflg = htole16(
1473 ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB_READ : 0) |
1474 (slot->slot << MVS_CRQB_TAG_SHIFT) |
1475 (port << MVS_CRQB_PMP_SHIFT));
1476 i = 0;
1477 /*
1478 * Controller can handle only 11 of 12 ATA registers,
1479 * so we have to choose which one to skip.
1480 */
1481 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1482 crqb->cmd[i++] = ccb->ataio.cmd.features_exp;
1483 crqb->cmd[i++] = 0x11;
1484 }
1485 crqb->cmd[i++] = ccb->ataio.cmd.features;
1486 crqb->cmd[i++] = 0x11;
1487 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1488 crqb->cmd[i++] = (slot->tag << 3) |
1489 (ccb->ataio.cmd.sector_count & 0x07);
1490 crqb->cmd[i++] = 0x12;
1491 } else {
1492 crqb->cmd[i++] = ccb->ataio.cmd.sector_count_exp;
1493 crqb->cmd[i++] = 0x12;
1494 crqb->cmd[i++] = ccb->ataio.cmd.sector_count;
1495 crqb->cmd[i++] = 0x12;
1496 }
1497 crqb->cmd[i++] = ccb->ataio.cmd.lba_low_exp;
1498 crqb->cmd[i++] = 0x13;
1499 crqb->cmd[i++] = ccb->ataio.cmd.lba_low;
1500 crqb->cmd[i++] = 0x13;
1501 crqb->cmd[i++] = ccb->ataio.cmd.lba_mid_exp;
1502 crqb->cmd[i++] = 0x14;
1503 crqb->cmd[i++] = ccb->ataio.cmd.lba_mid;
1504 crqb->cmd[i++] = 0x14;
1505 crqb->cmd[i++] = ccb->ataio.cmd.lba_high_exp;
1506 crqb->cmd[i++] = 0x15;
1507 crqb->cmd[i++] = ccb->ataio.cmd.lba_high;
1508 crqb->cmd[i++] = 0x15;
1509 crqb->cmd[i++] = ccb->ataio.cmd.device;
1510 crqb->cmd[i++] = 0x16;
1511 crqb->cmd[i++] = ccb->ataio.cmd.command;
1512 crqb->cmd[i++] = 0x97;
1513 }
1514 bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
1515 BUS_DMASYNC_PREWRITE);
1516 bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
1517 BUS_DMASYNC_PREREAD);
1518 slot->state = MVS_SLOT_RUNNING;
1519 ch->rslots |= (1 << slot->slot);
1520 /* Issue command to the controller. */
1521 ch->out_idx = (ch->out_idx + 1) & (MVS_MAX_SLOTS - 1);
1522 ATA_OUTL(ch->r_mem, EDMA_REQQIP,
1523 ch->dma.workrq_bus + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
1524 /* Start command execution timeout */
1525 callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout, 0,
1526 mvs_timeout, slot, 0);
1527 return;
1528 }
1529
1530 /* Must be called with channel locked. */
1531 static void
mvs_process_timeout(device_t dev)1532 mvs_process_timeout(device_t dev)
1533 {
1534 struct mvs_channel *ch = device_get_softc(dev);
1535 int i;
1536
1537 mtx_assert(&ch->mtx, MA_OWNED);
1538 /* Handle the rest of commands. */
1539 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1540 /* Do we have a running request on slot? */
1541 if (ch->slot[i].state < MVS_SLOT_RUNNING)
1542 continue;
1543 mvs_end_transaction(&ch->slot[i], MVS_ERR_TIMEOUT);
1544 }
1545 }
1546
1547 /* Must be called with channel locked. */
1548 static void
mvs_rearm_timeout(device_t dev)1549 mvs_rearm_timeout(device_t dev)
1550 {
1551 struct mvs_channel *ch = device_get_softc(dev);
1552 int i;
1553
1554 mtx_assert(&ch->mtx, MA_OWNED);
1555 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1556 struct mvs_slot *slot = &ch->slot[i];
1557
1558 /* Do we have a running request on slot? */
1559 if (slot->state < MVS_SLOT_RUNNING)
1560 continue;
1561 if ((ch->toslots & (1 << i)) == 0)
1562 continue;
1563 callout_reset_sbt(&slot->timeout,
1564 SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0,
1565 mvs_timeout, slot, 0);
1566 }
1567 }
1568
1569 /* Locked by callout mechanism. */
1570 static void
mvs_timeout(void * arg)1571 mvs_timeout(void *arg)
1572 {
1573 struct mvs_slot *slot = arg;
1574 device_t dev = slot->dev;
1575 struct mvs_channel *ch = device_get_softc(dev);
1576
1577 /* Check for stale timeout. */
1578 if (slot->state < MVS_SLOT_RUNNING)
1579 return;
1580 device_printf(dev, "Timeout on slot %d\n", slot->slot);
1581 device_printf(dev, "iec %08x sstat %08x serr %08x edma_s %08x "
1582 "dma_c %08x dma_s %08x rs %08x status %02x\n",
1583 ATA_INL(ch->r_mem, EDMA_IEC),
1584 ATA_INL(ch->r_mem, SATA_SS), ATA_INL(ch->r_mem, SATA_SE),
1585 ATA_INL(ch->r_mem, EDMA_S), ATA_INL(ch->r_mem, DMA_C),
1586 ATA_INL(ch->r_mem, DMA_S), ch->rslots,
1587 ATA_INB(ch->r_mem, ATA_ALTSTAT));
1588 /* Handle frozen command. */
1589 mvs_requeue_frozen(dev);
1590 /* We wait for other commands timeout and pray. */
1591 if (ch->toslots == 0)
1592 xpt_freeze_simq(ch->sim, 1);
1593 ch->toslots |= (1 << slot->slot);
1594 if ((ch->rslots & ~ch->toslots) == 0)
1595 mvs_process_timeout(dev);
1596 else
1597 device_printf(dev, " ... waiting for slots %08x\n",
1598 ch->rslots & ~ch->toslots);
1599 }
1600
1601 /* Must be called with channel locked. */
1602 static void
mvs_end_transaction(struct mvs_slot * slot,enum mvs_err_type et)1603 mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et)
1604 {
1605 device_t dev = slot->dev;
1606 struct mvs_channel *ch = device_get_softc(dev);
1607 union ccb *ccb = slot->ccb;
1608 int lastto;
1609
1610 bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
1611 BUS_DMASYNC_POSTWRITE);
1612 /* Read result registers to the result struct
1613 * May be incorrect if several commands finished same time,
1614 * so read only when sure or have to.
1615 */
1616 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1617 struct ata_res *res = &ccb->ataio.res;
1618
1619 if ((et == MVS_ERR_TFE) ||
1620 (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
1621 mvs_tfd_read(dev, ccb);
1622 } else
1623 bzero(res, sizeof(*res));
1624 } else {
1625 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
1626 ch->basic_dma == 0)
1627 ccb->csio.resid = ccb->csio.dxfer_len - ch->donecount;
1628 }
1629 if (ch->numpslots == 0 || ch->basic_dma) {
1630 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
1631 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
1632 (ccb->ccb_h.flags & CAM_DIR_IN) ?
1633 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
1634 bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
1635 }
1636 }
1637 if (et != MVS_ERR_NONE)
1638 ch->eslots |= (1 << slot->slot);
1639 /* In case of error, freeze device for proper recovery. */
1640 if ((et != MVS_ERR_NONE) && (!ch->recoverycmd) &&
1641 !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
1642 xpt_freeze_devq(ccb->ccb_h.path, 1);
1643 ccb->ccb_h.status |= CAM_DEV_QFRZN;
1644 }
1645 /* Set proper result status. */
1646 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1647 switch (et) {
1648 case MVS_ERR_NONE:
1649 ccb->ccb_h.status |= CAM_REQ_CMP;
1650 if (ccb->ccb_h.func_code == XPT_SCSI_IO)
1651 ccb->csio.scsi_status = SCSI_STATUS_OK;
1652 break;
1653 case MVS_ERR_INVALID:
1654 ch->fatalerr = 1;
1655 ccb->ccb_h.status |= CAM_REQ_INVALID;
1656 break;
1657 case MVS_ERR_INNOCENT:
1658 ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1659 break;
1660 case MVS_ERR_TFE:
1661 case MVS_ERR_NCQ:
1662 if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
1663 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
1664 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
1665 } else {
1666 ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
1667 }
1668 break;
1669 case MVS_ERR_SATA:
1670 ch->fatalerr = 1;
1671 if (!ch->recoverycmd) {
1672 xpt_freeze_simq(ch->sim, 1);
1673 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1674 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1675 }
1676 ccb->ccb_h.status |= CAM_UNCOR_PARITY;
1677 break;
1678 case MVS_ERR_TIMEOUT:
1679 if (!ch->recoverycmd) {
1680 xpt_freeze_simq(ch->sim, 1);
1681 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
1682 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
1683 }
1684 ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
1685 break;
1686 default:
1687 ch->fatalerr = 1;
1688 ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
1689 }
1690 /* Free slot. */
1691 ch->oslots &= ~(1 << slot->slot);
1692 ch->rslots &= ~(1 << slot->slot);
1693 ch->aslots &= ~(1 << slot->slot);
1694 slot->state = MVS_SLOT_EMPTY;
1695 slot->ccb = NULL;
1696 /* Update channel stats. */
1697 ch->numrslots--;
1698 ch->numrslotspd[ccb->ccb_h.target_id]--;
1699 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1700 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
1701 ch->otagspd[ccb->ccb_h.target_id] &= ~(1 << slot->tag);
1702 ch->numtslots--;
1703 ch->numtslotspd[ccb->ccb_h.target_id]--;
1704 } else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
1705 ch->numdslots--;
1706 } else {
1707 ch->numpslots--;
1708 }
1709 } else {
1710 ch->numpslots--;
1711 ch->basic_dma = 0;
1712 }
1713 /* Cancel timeout state if request completed normally. */
1714 if (et != MVS_ERR_TIMEOUT) {
1715 lastto = (ch->toslots == (1 << slot->slot));
1716 ch->toslots &= ~(1 << slot->slot);
1717 if (lastto)
1718 xpt_release_simq(ch->sim, TRUE);
1719 }
1720 /* If it was our READ LOG command - process it. */
1721 if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
1722 mvs_process_read_log(dev, ccb);
1723 /* If it was our REQUEST SENSE command - process it. */
1724 } else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
1725 mvs_process_request_sense(dev, ccb);
1726 /* If it was NCQ or ATAPI command error, put result on hold. */
1727 } else if (et == MVS_ERR_NCQ ||
1728 ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
1729 (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) {
1730 ch->hold[slot->slot] = ccb;
1731 ch->holdtag[slot->slot] = slot->tag;
1732 ch->numhslots++;
1733 } else
1734 xpt_done(ccb);
1735 /* If we have no other active commands, ... */
1736 if (ch->rslots == 0) {
1737 /* if there was fatal error - reset port. */
1738 if (ch->toslots != 0 || ch->fatalerr) {
1739 mvs_reset(dev);
1740 } else {
1741 /* if we have slots in error, we can reinit port. */
1742 if (ch->eslots != 0) {
1743 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
1744 ch->eslots = 0;
1745 }
1746 /* if there commands on hold, we can do READ LOG. */
1747 if (!ch->recoverycmd && ch->numhslots)
1748 mvs_issue_recovery(dev);
1749 }
1750 /* If all the rest of commands are in timeout - give them chance. */
1751 } else if ((ch->rslots & ~ch->toslots) == 0 &&
1752 et != MVS_ERR_TIMEOUT)
1753 mvs_rearm_timeout(dev);
1754 /* Unfreeze frozen command. */
1755 if (ch->frozen && !mvs_check_collision(dev, ch->frozen)) {
1756 union ccb *fccb = ch->frozen;
1757 ch->frozen = NULL;
1758 mvs_begin_transaction(dev, fccb);
1759 xpt_release_simq(ch->sim, TRUE);
1760 }
1761 /* Start PM timer. */
1762 if (ch->numrslots == 0 && ch->pm_level > 3 &&
1763 (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
1764 callout_schedule(&ch->pm_timer,
1765 (ch->pm_level == 4) ? hz / 1000 : hz / 8);
1766 }
1767 }
1768
1769 static void
mvs_issue_recovery(device_t dev)1770 mvs_issue_recovery(device_t dev)
1771 {
1772 struct mvs_channel *ch = device_get_softc(dev);
1773 union ccb *ccb;
1774 struct ccb_ataio *ataio;
1775 struct ccb_scsiio *csio;
1776 int i;
1777
1778 /* Find some held command. */
1779 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1780 if (ch->hold[i])
1781 break;
1782 }
1783 ccb = xpt_alloc_ccb_nowait();
1784 if (ccb == NULL) {
1785 device_printf(dev, "Unable to allocate recovery command\n");
1786 completeall:
1787 /* We can't do anything -- complete held commands. */
1788 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1789 if (ch->hold[i] == NULL)
1790 continue;
1791 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1792 ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL;
1793 xpt_done(ch->hold[i]);
1794 ch->hold[i] = NULL;
1795 ch->numhslots--;
1796 }
1797 mvs_reset(dev);
1798 return;
1799 }
1800 xpt_setup_ccb(&ccb->ccb_h, ch->hold[i]->ccb_h.path,
1801 ch->hold[i]->ccb_h.pinfo.priority);
1802 if (ccb->ccb_h.func_code == XPT_ATA_IO) {
1803 /* READ LOG */
1804 ccb->ccb_h.recovery_type = RECOVERY_READ_LOG;
1805 ccb->ccb_h.func_code = XPT_ATA_IO;
1806 ccb->ccb_h.flags = CAM_DIR_IN;
1807 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */
1808 ataio = &ccb->ataio;
1809 ataio->data_ptr = malloc(512, M_MVS, M_NOWAIT);
1810 if (ataio->data_ptr == NULL) {
1811 xpt_free_ccb(ccb);
1812 device_printf(dev,
1813 "Unable to allocate memory for READ LOG command\n");
1814 goto completeall;
1815 }
1816 ataio->dxfer_len = 512;
1817 bzero(&ataio->cmd, sizeof(ataio->cmd));
1818 ataio->cmd.flags = CAM_ATAIO_48BIT;
1819 ataio->cmd.command = 0x2F; /* READ LOG EXT */
1820 ataio->cmd.sector_count = 1;
1821 ataio->cmd.sector_count_exp = 0;
1822 ataio->cmd.lba_low = 0x10;
1823 ataio->cmd.lba_mid = 0;
1824 ataio->cmd.lba_mid_exp = 0;
1825 } else {
1826 /* REQUEST SENSE */
1827 ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE;
1828 ccb->ccb_h.recovery_slot = i;
1829 ccb->ccb_h.func_code = XPT_SCSI_IO;
1830 ccb->ccb_h.flags = CAM_DIR_IN;
1831 ccb->ccb_h.status = 0;
1832 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */
1833 csio = &ccb->csio;
1834 csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data;
1835 csio->dxfer_len = ch->hold[i]->csio.sense_len;
1836 csio->cdb_len = 6;
1837 bzero(&csio->cdb_io, sizeof(csio->cdb_io));
1838 csio->cdb_io.cdb_bytes[0] = 0x03;
1839 csio->cdb_io.cdb_bytes[4] = csio->dxfer_len;
1840 }
1841 /* Freeze SIM while doing recovery. */
1842 ch->recoverycmd = 1;
1843 xpt_freeze_simq(ch->sim, 1);
1844 mvs_begin_transaction(dev, ccb);
1845 }
1846
1847 static void
mvs_process_read_log(device_t dev,union ccb * ccb)1848 mvs_process_read_log(device_t dev, union ccb *ccb)
1849 {
1850 struct mvs_channel *ch = device_get_softc(dev);
1851 uint8_t *data;
1852 struct ata_res *res;
1853 int i;
1854
1855 ch->recoverycmd = 0;
1856
1857 data = ccb->ataio.data_ptr;
1858 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
1859 (data[0] & 0x80) == 0) {
1860 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1861 if (!ch->hold[i])
1862 continue;
1863 if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
1864 continue;
1865 if ((data[0] & 0x1F) == ch->holdtag[i]) {
1866 res = &ch->hold[i]->ataio.res;
1867 res->status = data[2];
1868 res->error = data[3];
1869 res->lba_low = data[4];
1870 res->lba_mid = data[5];
1871 res->lba_high = data[6];
1872 res->device = data[7];
1873 res->lba_low_exp = data[8];
1874 res->lba_mid_exp = data[9];
1875 res->lba_high_exp = data[10];
1876 res->sector_count = data[12];
1877 res->sector_count_exp = data[13];
1878 } else {
1879 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1880 ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
1881 }
1882 xpt_done(ch->hold[i]);
1883 ch->hold[i] = NULL;
1884 ch->numhslots--;
1885 }
1886 } else {
1887 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
1888 device_printf(dev, "Error while READ LOG EXT\n");
1889 else if ((data[0] & 0x80) == 0) {
1890 device_printf(dev,
1891 "Non-queued command error in READ LOG EXT\n");
1892 }
1893 for (i = 0; i < MVS_MAX_SLOTS; i++) {
1894 if (!ch->hold[i])
1895 continue;
1896 if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
1897 continue;
1898 xpt_done(ch->hold[i]);
1899 ch->hold[i] = NULL;
1900 ch->numhslots--;
1901 }
1902 }
1903 free(ccb->ataio.data_ptr, M_MVS);
1904 xpt_free_ccb(ccb);
1905 xpt_release_simq(ch->sim, TRUE);
1906 }
1907
1908 static void
mvs_process_request_sense(device_t dev,union ccb * ccb)1909 mvs_process_request_sense(device_t dev, union ccb *ccb)
1910 {
1911 struct mvs_channel *ch = device_get_softc(dev);
1912 int i;
1913
1914 ch->recoverycmd = 0;
1915
1916 i = ccb->ccb_h.recovery_slot;
1917 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
1918 ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID;
1919 } else {
1920 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
1921 ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL;
1922 }
1923 xpt_done(ch->hold[i]);
1924 ch->hold[i] = NULL;
1925 ch->numhslots--;
1926 xpt_free_ccb(ccb);
1927 xpt_release_simq(ch->sim, TRUE);
1928 }
1929
1930 static int
mvs_wait(device_t dev,u_int s,u_int c,int t)1931 mvs_wait(device_t dev, u_int s, u_int c, int t)
1932 {
1933 int timeout = 0;
1934 uint8_t st;
1935
1936 while (((st = mvs_getstatus(dev, 0)) & (s | c)) != s) {
1937 if (timeout >= t) {
1938 if (t != 0)
1939 device_printf(dev, "Wait status %02x\n", st);
1940 return (-1);
1941 }
1942 DELAY(1000);
1943 timeout++;
1944 }
1945 return (timeout);
1946 }
1947
1948 static void
mvs_requeue_frozen(device_t dev)1949 mvs_requeue_frozen(device_t dev)
1950 {
1951 struct mvs_channel *ch = device_get_softc(dev);
1952 union ccb *fccb = ch->frozen;
1953
1954 if (fccb) {
1955 ch->frozen = NULL;
1956 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
1957 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
1958 xpt_freeze_devq(fccb->ccb_h.path, 1);
1959 fccb->ccb_h.status |= CAM_DEV_QFRZN;
1960 }
1961 xpt_done(fccb);
1962 }
1963 }
1964
1965 static void
mvs_reset_to(void * arg)1966 mvs_reset_to(void *arg)
1967 {
1968 device_t dev = arg;
1969 struct mvs_channel *ch = device_get_softc(dev);
1970 int t;
1971
1972 if (ch->resetting == 0)
1973 return;
1974 ch->resetting--;
1975 if ((t = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, 0)) >= 0) {
1976 if (bootverbose) {
1977 device_printf(dev,
1978 "MVS reset: device ready after %dms\n",
1979 (310 - ch->resetting) * 100);
1980 }
1981 ch->resetting = 0;
1982 xpt_release_simq(ch->sim, TRUE);
1983 return;
1984 }
1985 if (ch->resetting == 0) {
1986 device_printf(dev,
1987 "MVS reset: device not ready after 31000ms\n");
1988 xpt_release_simq(ch->sim, TRUE);
1989 return;
1990 }
1991 callout_schedule(&ch->reset_timer, hz / 10);
1992 }
1993
1994 static void
mvs_errata(device_t dev)1995 mvs_errata(device_t dev)
1996 {
1997 struct mvs_channel *ch = device_get_softc(dev);
1998 uint32_t val;
1999
2000 if (ch->quirks & MVS_Q_SOC65) {
2001 val = ATA_INL(ch->r_mem, SATA_PHYM3);
2002 val &= ~(0x3 << 27); /* SELMUPF = 1 */
2003 val |= (0x1 << 27);
2004 val &= ~(0x3 << 29); /* SELMUPI = 1 */
2005 val |= (0x1 << 29);
2006 ATA_OUTL(ch->r_mem, SATA_PHYM3, val);
2007
2008 val = ATA_INL(ch->r_mem, SATA_PHYM4);
2009 val &= ~0x1; /* SATU_OD8 = 0 */
2010 val |= (0x1 << 16); /* reserved bit 16 = 1 */
2011 ATA_OUTL(ch->r_mem, SATA_PHYM4, val);
2012
2013 val = ATA_INL(ch->r_mem, SATA_PHYM9_GEN2);
2014 val &= ~0xf; /* TXAMP[3:0] = 8 */
2015 val |= 0x8;
2016 val &= ~(0x1 << 14); /* TXAMP[4] = 0 */
2017 ATA_OUTL(ch->r_mem, SATA_PHYM9_GEN2, val);
2018
2019 val = ATA_INL(ch->r_mem, SATA_PHYM9_GEN1);
2020 val &= ~0xf; /* TXAMP[3:0] = 8 */
2021 val |= 0x8;
2022 val &= ~(0x1 << 14); /* TXAMP[4] = 0 */
2023 ATA_OUTL(ch->r_mem, SATA_PHYM9_GEN1, val);
2024 }
2025 }
2026
2027 static void
mvs_reset(device_t dev)2028 mvs_reset(device_t dev)
2029 {
2030 struct mvs_channel *ch = device_get_softc(dev);
2031 int i;
2032
2033 xpt_freeze_simq(ch->sim, 1);
2034 if (bootverbose)
2035 device_printf(dev, "MVS reset...\n");
2036 /* Forget about previous reset. */
2037 if (ch->resetting) {
2038 ch->resetting = 0;
2039 callout_stop(&ch->reset_timer);
2040 xpt_release_simq(ch->sim, TRUE);
2041 }
2042 /* Requeue freezed command. */
2043 mvs_requeue_frozen(dev);
2044 /* Kill the engine and requeue all running commands. */
2045 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
2046 ATA_OUTL(ch->r_mem, DMA_C, 0);
2047 for (i = 0; i < MVS_MAX_SLOTS; i++) {
2048 /* Do we have a running request on slot? */
2049 if (ch->slot[i].state < MVS_SLOT_RUNNING)
2050 continue;
2051 /* XXX; Commands in loading state. */
2052 mvs_end_transaction(&ch->slot[i], MVS_ERR_INNOCENT);
2053 }
2054 for (i = 0; i < MVS_MAX_SLOTS; i++) {
2055 if (!ch->hold[i])
2056 continue;
2057 xpt_done(ch->hold[i]);
2058 ch->hold[i] = NULL;
2059 ch->numhslots--;
2060 }
2061 if (ch->toslots != 0)
2062 xpt_release_simq(ch->sim, TRUE);
2063 ch->eslots = 0;
2064 ch->toslots = 0;
2065 ch->fatalerr = 0;
2066 ch->fake_busy = 0;
2067 /* Tell the XPT about the event */
2068 xpt_async(AC_BUS_RESET, ch->path, NULL);
2069 ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
2070 ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EATARST);
2071 DELAY(25);
2072 ATA_OUTL(ch->r_mem, EDMA_CMD, 0);
2073 mvs_errata(dev);
2074 /* Reset and reconnect PHY, */
2075 if (!mvs_sata_phy_reset(dev)) {
2076 if (bootverbose)
2077 device_printf(dev, "MVS reset: device not found\n");
2078 ch->devices = 0;
2079 ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2080 ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
2081 ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
2082 xpt_release_simq(ch->sim, TRUE);
2083 return;
2084 }
2085 if (bootverbose)
2086 device_printf(dev, "MVS reset: device found\n");
2087 /* Wait for clearing busy status. */
2088 if ((i = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ,
2089 dumping ? 31000 : 0)) < 0) {
2090 if (dumping) {
2091 device_printf(dev,
2092 "MVS reset: device not ready after 31000ms\n");
2093 } else
2094 ch->resetting = 310;
2095 } else if (bootverbose)
2096 device_printf(dev, "MVS reset: device ready after %dms\n", i);
2097 ch->devices = 1;
2098 ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2099 ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
2100 ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
2101 if (ch->resetting)
2102 callout_reset(&ch->reset_timer, hz / 10, mvs_reset_to, dev);
2103 else
2104 xpt_release_simq(ch->sim, TRUE);
2105 }
2106
2107 static void
mvs_softreset(device_t dev,union ccb * ccb)2108 mvs_softreset(device_t dev, union ccb *ccb)
2109 {
2110 struct mvs_channel *ch = device_get_softc(dev);
2111 int port = ccb->ccb_h.target_id & 0x0f;
2112 int i, stuck;
2113 uint8_t status;
2114
2115 mvs_set_edma_mode(dev, MVS_EDMA_OFF);
2116 ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
2117 ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_RESET);
2118 DELAY(10000);
2119 ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
2120 ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2121 /* Wait for clearing busy status. */
2122 if ((i = mvs_wait(dev, 0, ATA_S_BUSY, ccb->ccb_h.timeout)) < 0) {
2123 ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
2124 stuck = 1;
2125 } else {
2126 status = mvs_getstatus(dev, 0);
2127 if (status & ATA_S_ERROR)
2128 ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
2129 else
2130 ccb->ccb_h.status |= CAM_REQ_CMP;
2131 if (status & ATA_S_DRQ)
2132 stuck = 1;
2133 else
2134 stuck = 0;
2135 }
2136 mvs_tfd_read(dev, ccb);
2137
2138 /*
2139 * XXX: If some device on PMP failed to soft-reset,
2140 * try to recover by sending dummy soft-reset to PMP.
2141 */
2142 if (stuck && ch->pm_present && port != 15) {
2143 ATA_OUTB(ch->r_mem, SATA_SATAICTL,
2144 15 << SATA_SATAICTL_PMPTX_SHIFT);
2145 ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_RESET);
2146 DELAY(10000);
2147 ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
2148 mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, ccb->ccb_h.timeout);
2149 }
2150
2151 xpt_done(ccb);
2152 }
2153
2154 static int
mvs_sata_connect(struct mvs_channel * ch)2155 mvs_sata_connect(struct mvs_channel *ch)
2156 {
2157 u_int32_t status;
2158 int timeout, found = 0;
2159
2160 /* Wait up to 100ms for "connect well" */
2161 for (timeout = 0; timeout < 1000 ; timeout++) {
2162 status = ATA_INL(ch->r_mem, SATA_SS);
2163 if ((status & SATA_SS_DET_MASK) != SATA_SS_DET_NO_DEVICE)
2164 found = 1;
2165 if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
2166 ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
2167 ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE))
2168 break;
2169 if ((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_OFFLINE) {
2170 if (bootverbose) {
2171 device_printf(ch->dev, "SATA offline status=%08x\n",
2172 status);
2173 }
2174 return (0);
2175 }
2176 if (found == 0 && timeout >= 100)
2177 break;
2178 DELAY(100);
2179 }
2180 if (timeout >= 1000 || !found) {
2181 if (bootverbose) {
2182 device_printf(ch->dev,
2183 "SATA connect timeout time=%dus status=%08x\n",
2184 timeout * 100, status);
2185 }
2186 return (0);
2187 }
2188 if (bootverbose) {
2189 device_printf(ch->dev, "SATA connect time=%dus status=%08x\n",
2190 timeout * 100, status);
2191 }
2192 /* Clear SATA error register */
2193 ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
2194 return (1);
2195 }
2196
2197 static int
mvs_sata_phy_reset(device_t dev)2198 mvs_sata_phy_reset(device_t dev)
2199 {
2200 struct mvs_channel *ch = device_get_softc(dev);
2201 int sata_rev;
2202 uint32_t val;
2203
2204 sata_rev = ch->user[ch->pm_present ? 15 : 0].revision;
2205 if (sata_rev == 1)
2206 val = SATA_SC_SPD_SPEED_GEN1;
2207 else if (sata_rev == 2)
2208 val = SATA_SC_SPD_SPEED_GEN2;
2209 else if (sata_rev == 3)
2210 val = SATA_SC_SPD_SPEED_GEN3;
2211 else
2212 val = 0;
2213 ATA_OUTL(ch->r_mem, SATA_SC,
2214 SATA_SC_DET_RESET | val |
2215 SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER);
2216 DELAY(1000);
2217 ATA_OUTL(ch->r_mem, SATA_SC,
2218 SATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 :
2219 (SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER)));
2220 if (!mvs_sata_connect(ch)) {
2221 if (ch->pm_level > 0)
2222 ATA_OUTL(ch->r_mem, SATA_SC, SATA_SC_DET_DISABLE);
2223 return (0);
2224 }
2225 return (1);
2226 }
2227
2228 static int
mvs_check_ids(device_t dev,union ccb * ccb)2229 mvs_check_ids(device_t dev, union ccb *ccb)
2230 {
2231 struct mvs_channel *ch = device_get_softc(dev);
2232
2233 if (ccb->ccb_h.target_id > ((ch->quirks & MVS_Q_GENI) ? 0 : 15)) {
2234 ccb->ccb_h.status = CAM_TID_INVALID;
2235 xpt_done(ccb);
2236 return (-1);
2237 }
2238 if (ccb->ccb_h.target_lun != 0) {
2239 ccb->ccb_h.status = CAM_LUN_INVALID;
2240 xpt_done(ccb);
2241 return (-1);
2242 }
2243 /*
2244 * It's a programming error to see AUXILIARY register requests.
2245 */
2246 KASSERT(ccb->ccb_h.func_code != XPT_ATA_IO ||
2247 ((ccb->ataio.ata_flags & ATA_FLAG_AUX) == 0),
2248 ("AUX register unsupported"));
2249 return (0);
2250 }
2251
2252 static void
mvsaction(struct cam_sim * sim,union ccb * ccb)2253 mvsaction(struct cam_sim *sim, union ccb *ccb)
2254 {
2255 device_t dev, parent;
2256 struct mvs_channel *ch;
2257
2258 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("mvsaction func_code=%x\n",
2259 ccb->ccb_h.func_code));
2260
2261 ch = (struct mvs_channel *)cam_sim_softc(sim);
2262 dev = ch->dev;
2263 switch (ccb->ccb_h.func_code) {
2264 /* Common cases first */
2265 case XPT_ATA_IO: /* Execute the requested I/O operation */
2266 case XPT_SCSI_IO:
2267 if (mvs_check_ids(dev, ccb))
2268 return;
2269 if (ch->devices == 0 ||
2270 (ch->pm_present == 0 &&
2271 ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
2272 ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2273 break;
2274 }
2275 ccb->ccb_h.recovery_type = RECOVERY_NONE;
2276 /* Check for command collision. */
2277 if (mvs_check_collision(dev, ccb)) {
2278 /* Freeze command. */
2279 ch->frozen = ccb;
2280 /* We have only one frozen slot, so freeze simq also. */
2281 xpt_freeze_simq(ch->sim, 1);
2282 return;
2283 }
2284 mvs_begin_transaction(dev, ccb);
2285 return;
2286 case XPT_ABORT: /* Abort the specified CCB */
2287 /* XXX Implement */
2288 ccb->ccb_h.status = CAM_REQ_INVALID;
2289 break;
2290 case XPT_SET_TRAN_SETTINGS:
2291 {
2292 struct ccb_trans_settings *cts = &ccb->cts;
2293 struct mvs_device *d;
2294
2295 if (mvs_check_ids(dev, ccb))
2296 return;
2297 if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2298 d = &ch->curr[ccb->ccb_h.target_id];
2299 else
2300 d = &ch->user[ccb->ccb_h.target_id];
2301 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
2302 d->revision = cts->xport_specific.sata.revision;
2303 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
2304 d->mode = cts->xport_specific.sata.mode;
2305 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) {
2306 d->bytecount = min((ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048,
2307 cts->xport_specific.sata.bytecount);
2308 }
2309 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
2310 d->tags = min(MVS_MAX_SLOTS, cts->xport_specific.sata.tags);
2311 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
2312 ch->pm_present = cts->xport_specific.sata.pm_present;
2313 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
2314 d->atapi = cts->xport_specific.sata.atapi;
2315 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
2316 d->caps = cts->xport_specific.sata.caps;
2317 ccb->ccb_h.status = CAM_REQ_CMP;
2318 break;
2319 }
2320 case XPT_GET_TRAN_SETTINGS:
2321 /* Get default/user set transfer settings for the target */
2322 {
2323 struct ccb_trans_settings *cts = &ccb->cts;
2324 struct mvs_device *d;
2325 uint32_t status;
2326
2327 if (mvs_check_ids(dev, ccb))
2328 return;
2329 if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
2330 d = &ch->curr[ccb->ccb_h.target_id];
2331 else
2332 d = &ch->user[ccb->ccb_h.target_id];
2333 cts->protocol = PROTO_UNSPECIFIED;
2334 cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
2335 cts->transport = XPORT_SATA;
2336 cts->transport_version = XPORT_VERSION_UNSPECIFIED;
2337 cts->proto_specific.valid = 0;
2338 cts->xport_specific.sata.valid = 0;
2339 if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
2340 (ccb->ccb_h.target_id == 15 ||
2341 (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
2342 status = ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_SPD_MASK;
2343 if (status & 0x0f0) {
2344 cts->xport_specific.sata.revision =
2345 (status & 0x0f0) >> 4;
2346 cts->xport_specific.sata.valid |=
2347 CTS_SATA_VALID_REVISION;
2348 }
2349 cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
2350 // if (ch->pm_level)
2351 // cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
2352 cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN;
2353 cts->xport_specific.sata.caps &=
2354 ch->user[ccb->ccb_h.target_id].caps;
2355 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
2356 } else {
2357 cts->xport_specific.sata.revision = d->revision;
2358 cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
2359 cts->xport_specific.sata.caps = d->caps;
2360 if (cts->type == CTS_TYPE_CURRENT_SETTINGS/* &&
2361 (ch->quirks & MVS_Q_GENIIE) == 0*/)
2362 cts->xport_specific.sata.caps &= ~CTS_SATA_CAPS_H_AN;
2363 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
2364 }
2365 cts->xport_specific.sata.mode = d->mode;
2366 cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
2367 cts->xport_specific.sata.bytecount = d->bytecount;
2368 cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
2369 cts->xport_specific.sata.pm_present = ch->pm_present;
2370 cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
2371 cts->xport_specific.sata.tags = d->tags;
2372 cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
2373 cts->xport_specific.sata.atapi = d->atapi;
2374 cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
2375 ccb->ccb_h.status = CAM_REQ_CMP;
2376 break;
2377 }
2378 case XPT_RESET_BUS: /* Reset the specified SCSI bus */
2379 case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */
2380 mvs_reset(dev);
2381 ccb->ccb_h.status = CAM_REQ_CMP;
2382 break;
2383 case XPT_TERM_IO: /* Terminate the I/O process */
2384 /* XXX Implement */
2385 ccb->ccb_h.status = CAM_REQ_INVALID;
2386 break;
2387 case XPT_PATH_INQ: /* Path routing inquiry */
2388 {
2389 struct ccb_pathinq *cpi = &ccb->cpi;
2390
2391 parent = device_get_parent(dev);
2392 cpi->version_num = 1; /* XXX??? */
2393 cpi->hba_inquiry = PI_SDTR_ABLE;
2394 if (!(ch->quirks & MVS_Q_GENI)) {
2395 cpi->hba_inquiry |= PI_SATAPM;
2396 /* Gen-II is extremely slow with NCQ on PMP. */
2397 if ((ch->quirks & MVS_Q_GENIIE) || ch->pm_present == 0)
2398 cpi->hba_inquiry |= PI_TAG_ABLE;
2399 }
2400 cpi->target_sprt = 0;
2401 cpi->hba_misc = PIM_SEQSCAN;
2402 cpi->hba_eng_cnt = 0;
2403 if (!(ch->quirks & MVS_Q_GENI))
2404 cpi->max_target = 15;
2405 else
2406 cpi->max_target = 0;
2407 cpi->max_lun = 0;
2408 cpi->initiator_id = 0;
2409 cpi->bus_id = cam_sim_bus(sim);
2410 cpi->base_transfer_speed = 150000;
2411 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
2412 strlcpy(cpi->hba_vid, "Marvell", HBA_IDLEN);
2413 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
2414 cpi->unit_number = cam_sim_unit(sim);
2415 cpi->transport = XPORT_SATA;
2416 cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
2417 cpi->protocol = PROTO_ATA;
2418 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
2419 cpi->maxio = maxphys;
2420 if ((ch->quirks & MVS_Q_SOC) == 0) {
2421 cpi->hba_vendor = pci_get_vendor(parent);
2422 cpi->hba_device = pci_get_device(parent);
2423 cpi->hba_subvendor = pci_get_subvendor(parent);
2424 cpi->hba_subdevice = pci_get_subdevice(parent);
2425 }
2426 cpi->ccb_h.status = CAM_REQ_CMP;
2427 break;
2428 }
2429 default:
2430 ccb->ccb_h.status = CAM_REQ_INVALID;
2431 break;
2432 }
2433 xpt_done(ccb);
2434 }
2435
2436 static void
mvspoll(struct cam_sim * sim)2437 mvspoll(struct cam_sim *sim)
2438 {
2439 struct mvs_channel *ch = (struct mvs_channel *)cam_sim_softc(sim);
2440 struct mvs_intr_arg arg;
2441
2442 arg.arg = ch->dev;
2443 arg.cause = 2 | 4; /* XXX */
2444 mvs_ch_intr(&arg);
2445 if (ch->resetting != 0 &&
2446 (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) {
2447 ch->resetpolldiv = 1000;
2448 mvs_reset_to(ch->dev);
2449 }
2450 }
2451