git.hungrycats.org Git

1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB

2 /*

5 */

6

7 #include <linux/skbuff.h>

8

9 #include "rxe.h"

10 #include "rxe_loc.h"

11 #include "rxe_queue.h"

12

13 enum resp_states {

14 RESPST_NONE,

15 RESPST_GET_REQ,

16 RESPST_CHK_PSN,

17 RESPST_CHK_OP_SEQ,

18 RESPST_CHK_OP_VALID,

19 RESPST_CHK_RESOURCE,

20 RESPST_CHK_LENGTH,

21 RESPST_CHK_RKEY,

22 RESPST_EXECUTE,

23 RESPST_READ_REPLY,

24 RESPST_COMPLETE,

25 RESPST_ACKNOWLEDGE,

26 RESPST_CLEANUP,

27 RESPST_DUPLICATE_REQUEST,

28 RESPST_ERR_MALFORMED_WQE,

29 RESPST_ERR_UNSUPPORTED_OPCODE,

30 RESPST_ERR_MISALIGNED_ATOMIC,

31 RESPST_ERR_PSN_OUT_OF_SEQ,

32 RESPST_ERR_MISSING_OPCODE_FIRST,

33 RESPST_ERR_MISSING_OPCODE_LAST_C,

34 RESPST_ERR_MISSING_OPCODE_LAST_D1E,

35 RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,

36 RESPST_ERR_RNR,

37 RESPST_ERR_RKEY_VIOLATION,

38 RESPST_ERR_LENGTH,

39 RESPST_ERR_CQ_OVERFLOW,

40 RESPST_ERROR,

41 RESPST_RESET,

42 RESPST_DONE,

43 RESPST_EXIT,

44 };

45

46 static char *resp_state_name[] = {

47 [RESPST_NONE] = "NONE",

48 [RESPST_GET_REQ] = "GET_REQ",

49 [RESPST_CHK_PSN] = "CHK_PSN",

50 [RESPST_CHK_OP_SEQ] = "CHK_OP_SEQ",

51 [RESPST_CHK_OP_VALID] = "CHK_OP_VALID",

52 [RESPST_CHK_RESOURCE] = "CHK_RESOURCE",

53 [RESPST_CHK_LENGTH] = "CHK_LENGTH",

54 [RESPST_CHK_RKEY] = "CHK_RKEY",

55 [RESPST_EXECUTE] = "EXECUTE",

56 [RESPST_READ_REPLY] = "READ_REPLY",

57 [RESPST_COMPLETE] = "COMPLETE",

58 [RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",

59 [RESPST_CLEANUP] = "CLEANUP",

60 [RESPST_DUPLICATE_REQUEST] = "DUPLICATE_REQUEST",

61 [RESPST_ERR_MALFORMED_WQE] = "ERR_MALFORMED_WQE",

62 [RESPST_ERR_UNSUPPORTED_OPCODE] = "ERR_UNSUPPORTED_OPCODE",

63 [RESPST_ERR_MISALIGNED_ATOMIC] = "ERR_MISALIGNED_ATOMIC",

64 [RESPST_ERR_PSN_OUT_OF_SEQ] = "ERR_PSN_OUT_OF_SEQ",

65 [RESPST_ERR_MISSING_OPCODE_FIRST] = "ERR_MISSING_OPCODE_FIRST",

66 [RESPST_ERR_MISSING_OPCODE_LAST_C] = "ERR_MISSING_OPCODE_LAST_C",

67 [RESPST_ERR_MISSING_OPCODE_LAST_D1E] = "ERR_MISSING_OPCODE_LAST_D1E",

68 [RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ",

69 [RESPST_ERR_RNR] = "ERR_RNR",

70 [RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION",

71 [RESPST_ERR_LENGTH] = "ERR_LENGTH",

72 [RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",

73 [RESPST_ERROR] = "ERROR",

74 [RESPST_RESET] = "RESET",

75 [RESPST_DONE] = "DONE",

76 [RESPST_EXIT] = "EXIT",

77 };

78

79 /* rxe_recv calls here to add a request packet to the input queue */

80 void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)

81 {

82 int must_sched;

83 struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);

84

85 skb_queue_tail(&qp->req_pkts, skb);

86

87 must_sched = (pkt->opcode == IB_OPCODE_RC_RDMA_READ_REQUEST) ||

88 (skb_queue_len(&qp->req_pkts) > 1);

89

90 rxe_run_task(&qp->resp.task, must_sched);

91 }

92

93 static inline enum resp_states get_req(struct rxe_qp *qp,

94 struct rxe_pkt_info **pkt_p)

95 {

96 struct sk_buff *skb;

97

98 if (qp->resp.state == QP_STATE_ERROR) {

99 while ((skb = skb_dequeue(&qp->req_pkts))) {

100 rxe_drop_ref(qp);

101 kfree_skb(skb);

102 }

103

104 /* go drain recv wr queue */

105 return RESPST_CHK_RESOURCE;

106 }

107

108 skb = skb_peek(&qp->req_pkts);

109 if (!skb)

110 return RESPST_EXIT;

111

112 *pkt_p = SKB_TO_PKT(skb);

113

114 return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN;

115 }

116

117 static enum resp_states check_psn(struct rxe_qp *qp,

118 struct rxe_pkt_info *pkt)

119 {

120 int diff = psn_compare(pkt->psn, qp->resp.psn);

121 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

122

123 switch (qp_type(qp)) {

124 case IB_QPT_RC:

125 if (diff > 0) {

126 if (qp->resp.sent_psn_nak)

127 return RESPST_CLEANUP;

128

129 qp->resp.sent_psn_nak = 1;

130 rxe_counter_inc(rxe, RXE_CNT_OUT_OF_SEQ_REQ);

131 return RESPST_ERR_PSN_OUT_OF_SEQ;

132

133 } else if (diff < 0) {

134 rxe_counter_inc(rxe, RXE_CNT_DUP_REQ);

135 return RESPST_DUPLICATE_REQUEST;

136 }

137

138 if (qp->resp.sent_psn_nak)

139 qp->resp.sent_psn_nak = 0;

140

141 break;

142

143 case IB_QPT_UC:

144 if (qp->resp.drop_msg || diff != 0) {

145 if (pkt->mask & RXE_START_MASK) {

146 qp->resp.drop_msg = 0;

147 return RESPST_CHK_OP_SEQ;

148 }

149

150 qp->resp.drop_msg = 1;

151 return RESPST_CLEANUP;

152 }

153 break;

154 default:

155 break;

156 }

157

158 return RESPST_CHK_OP_SEQ;

159 }

160

161 static enum resp_states check_op_seq(struct rxe_qp *qp,

162 struct rxe_pkt_info *pkt)

163 {

164 switch (qp_type(qp)) {

165 case IB_QPT_RC:

166 switch (qp->resp.opcode) {

167 case IB_OPCODE_RC_SEND_FIRST:

168 case IB_OPCODE_RC_SEND_MIDDLE:

169 switch (pkt->opcode) {

170 case IB_OPCODE_RC_SEND_MIDDLE:

171 case IB_OPCODE_RC_SEND_LAST:

172 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:

173 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:

174 return RESPST_CHK_OP_VALID;

175 default:

176 return RESPST_ERR_MISSING_OPCODE_LAST_C;

177 }

178

179 case IB_OPCODE_RC_RDMA_WRITE_FIRST:

180 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:

181 switch (pkt->opcode) {

182 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:

183 case IB_OPCODE_RC_RDMA_WRITE_LAST:

184 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:

185 return RESPST_CHK_OP_VALID;

186 default:

187 return RESPST_ERR_MISSING_OPCODE_LAST_C;

188 }

189

190 default:

191 switch (pkt->opcode) {

192 case IB_OPCODE_RC_SEND_MIDDLE:

193 case IB_OPCODE_RC_SEND_LAST:

194 case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:

195 case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:

196 case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:

197 case IB_OPCODE_RC_RDMA_WRITE_LAST:

198 case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:

199 return RESPST_ERR_MISSING_OPCODE_FIRST;

200 default:

201 return RESPST_CHK_OP_VALID;

202 }

203 }

204 break;

205

206 case IB_QPT_UC:

207 switch (qp->resp.opcode) {

208 case IB_OPCODE_UC_SEND_FIRST:

209 case IB_OPCODE_UC_SEND_MIDDLE:

210 switch (pkt->opcode) {

211 case IB_OPCODE_UC_SEND_MIDDLE:

212 case IB_OPCODE_UC_SEND_LAST:

213 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:

214 return RESPST_CHK_OP_VALID;

215 default:

216 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;

217 }

218

219 case IB_OPCODE_UC_RDMA_WRITE_FIRST:

220 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:

221 switch (pkt->opcode) {

222 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:

223 case IB_OPCODE_UC_RDMA_WRITE_LAST:

224 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:

225 return RESPST_CHK_OP_VALID;

226 default:

227 return RESPST_ERR_MISSING_OPCODE_LAST_D1E;

228 }

229

230 default:

231 switch (pkt->opcode) {

232 case IB_OPCODE_UC_SEND_MIDDLE:

233 case IB_OPCODE_UC_SEND_LAST:

234 case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:

235 case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:

236 case IB_OPCODE_UC_RDMA_WRITE_LAST:

237 case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:

238 qp->resp.drop_msg = 1;

239 return RESPST_CLEANUP;

240 default:

241 return RESPST_CHK_OP_VALID;

242 }

243 }

244 break;

245

246 default:

247 return RESPST_CHK_OP_VALID;

248 }

249 }

250

251 static enum resp_states check_op_valid(struct rxe_qp *qp,

252 struct rxe_pkt_info *pkt)

253 {

254 switch (qp_type(qp)) {

255 case IB_QPT_RC:

256 if (((pkt->mask & RXE_READ_MASK) &&

257 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) ||

258 ((pkt->mask & RXE_WRITE_MASK) &&

259 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) ||

260 ((pkt->mask & RXE_ATOMIC_MASK) &&

261 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) {

262 return RESPST_ERR_UNSUPPORTED_OPCODE;

263 }

264

265 break;

266

267 case IB_QPT_UC:

268 if ((pkt->mask & RXE_WRITE_MASK) &&

269 !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) {

270 qp->resp.drop_msg = 1;

271 return RESPST_CLEANUP;

272 }

273

274 break;

275

276 case IB_QPT_UD:

277 case IB_QPT_SMI:

278 case IB_QPT_GSI:

279 break;

280

281 default:

282 WARN_ON_ONCE(1);

283 break;

284 }

285

286 return RESPST_CHK_RESOURCE;

287 }

288

289 static enum resp_states get_srq_wqe(struct rxe_qp *qp)

290 {

291 struct rxe_srq *srq = qp->srq;

292 struct rxe_queue *q = srq->rq.queue;

293 struct rxe_recv_wqe *wqe;

294 struct ib_event ev;

295 size_t size;

296

297 if (srq->error)

298 return RESPST_ERR_RNR;

299

300 spin_lock_bh(&srq->rq.consumer_lock);

301

302 wqe = queue_head(q);

303 if (!wqe) {

304 spin_unlock_bh(&srq->rq.consumer_lock);

305 return RESPST_ERR_RNR;

306 }

307

308 /* don't trust user space data */

309 if (unlikely(wqe->dma.num_sge > srq->rq.max_sge)) {

310 spin_unlock_bh(&srq->rq.consumer_lock);

311 pr_warn("%s: invalid num_sge in SRQ entry\n", __func__);

312 return RESPST_ERR_MALFORMED_WQE;

313 }

314 size = sizeof(wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);

315 memcpy(&qp->resp.srq_wqe, wqe, size);

316

317 qp->resp.wqe = &qp->resp.srq_wqe.wqe;

318 advance_consumer(q);

319

320 if (srq->limit && srq->ibsrq.event_handler &&

321 (queue_count(q) < srq->limit)) {

322 srq->limit = 0;

323 goto event;

324 }

325

326 spin_unlock_bh(&srq->rq.consumer_lock);

327 return RESPST_CHK_LENGTH;

328

329 event:

330 spin_unlock_bh(&srq->rq.consumer_lock);

331 ev.device = qp->ibqp.device;

332 ev.element.srq = qp->ibqp.srq;

333 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;

334 srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context);

335 return RESPST_CHK_LENGTH;

336 }

337

338 static enum resp_states check_resource(struct rxe_qp *qp,

339 struct rxe_pkt_info *pkt)

340 {

341 struct rxe_srq *srq = qp->srq;

342

343 if (qp->resp.state == QP_STATE_ERROR) {

344 if (qp->resp.wqe) {

345 qp->resp.status = IB_WC_WR_FLUSH_ERR;

346 return RESPST_COMPLETE;

347 } else if (!srq) {

348 qp->resp.wqe = queue_head(qp->rq.queue);

349 if (qp->resp.wqe) {

350 qp->resp.status = IB_WC_WR_FLUSH_ERR;

351 return RESPST_COMPLETE;

352 } else {

353 return RESPST_EXIT;

354 }

355 } else {

356 return RESPST_EXIT;

357 }

358 }

359

360 if (pkt->mask & RXE_READ_OR_ATOMIC) {

361 /* it is the requesters job to not send

362 * too many read/atomic ops, we just

363 * recycle the responder resource queue

364 */

365 if (likely(qp->attr.max_dest_rd_atomic > 0))

366 return RESPST_CHK_LENGTH;

367 else

368 return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ;

369 }

370

371 if (pkt->mask & RXE_RWR_MASK) {

372 if (srq)

373 return get_srq_wqe(qp);

374

375 qp->resp.wqe = queue_head(qp->rq.queue);

376 return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR;

377 }

378

379 return RESPST_CHK_LENGTH;

380 }

381

382 static enum resp_states check_length(struct rxe_qp *qp,

383 struct rxe_pkt_info *pkt)

384 {

385 switch (qp_type(qp)) {

386 case IB_QPT_RC:

387 return RESPST_CHK_RKEY;

388

389 case IB_QPT_UC:

390 return RESPST_CHK_RKEY;

391

392 default:

393 return RESPST_CHK_RKEY;

394 }

395 }

396

397 static enum resp_states check_rkey(struct rxe_qp *qp,

398 struct rxe_pkt_info *pkt)

399 {

400 struct rxe_mem *mem = NULL;

401 u64 va;

402 u32 rkey;

403 u32 resid;

404 u32 pktlen;

405 int mtu = qp->mtu;

406 enum resp_states state;

407 int access;

408

409 if (pkt->mask & (RXE_READ_MASK | RXE_WRITE_MASK)) {

410 if (pkt->mask & RXE_RETH_MASK) {

411 qp->resp.va = reth_va(pkt);

412 qp->resp.rkey = reth_rkey(pkt);

413 qp->resp.resid = reth_len(pkt);

414 qp->resp.length = reth_len(pkt);

415 }

416 access = (pkt->mask & RXE_READ_MASK) ? IB_ACCESS_REMOTE_READ

417 : IB_ACCESS_REMOTE_WRITE;

418 } else if (pkt->mask & RXE_ATOMIC_MASK) {

419 qp->resp.va = atmeth_va(pkt);

420 qp->resp.rkey = atmeth_rkey(pkt);

421 qp->resp.resid = sizeof(u64);

422 access = IB_ACCESS_REMOTE_ATOMIC;

423 } else {

424 return RESPST_EXECUTE;

425 }

426

427 /* A zero-byte op is not required to set an addr or rkey. */

428 if ((pkt->mask & (RXE_READ_MASK | RXE_WRITE_OR_SEND)) &&

429 (pkt->mask & RXE_RETH_MASK) &&

430 reth_len(pkt) == 0) {

431 return RESPST_EXECUTE;

432 }

433

434 va = qp->resp.va;

435 rkey = qp->resp.rkey;

436 resid = qp->resp.resid;

437 pktlen = payload_size(pkt);

438

439 mem = lookup_mem(qp->pd, access, rkey, lookup_remote);

440 if (!mem) {

441 state = RESPST_ERR_RKEY_VIOLATION;

442 goto err;

443 }

444

445 if (unlikely(mem->state == RXE_MEM_STATE_FREE)) {

446 state = RESPST_ERR_RKEY_VIOLATION;

447 goto err;

448 }

449

450 if (mem_check_range(mem, va, resid)) {

451 state = RESPST_ERR_RKEY_VIOLATION;

452 goto err;

453 }

454

455 if (pkt->mask & RXE_WRITE_MASK) {

456 if (resid > mtu) {

457 if (pktlen != mtu || bth_pad(pkt)) {

458 state = RESPST_ERR_LENGTH;

459 goto err;

460 }

461 } else {

462 if (pktlen != resid) {

463 state = RESPST_ERR_LENGTH;

464 goto err;

465 }

466 if ((bth_pad(pkt) != (0x3 & (-resid)))) {

467 /* This case may not be exactly that

468 * but nothing else fits.

469 */

470 state = RESPST_ERR_LENGTH;

471 goto err;

472 }

473 }

474 }

475

476 WARN_ON_ONCE(qp->resp.mr);

477

478 qp->resp.mr = mem;

479 return RESPST_EXECUTE;

480

481 err:

482 if (mem)

483 rxe_drop_ref(mem);

484 return state;

485 }

486

487 static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,

488 int data_len)

489 {

490 int err;

491

492 err = copy_data(qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,

493 data_addr, data_len, to_mem_obj, NULL);

494 if (unlikely(err))

495 return (err == -ENOSPC) ? RESPST_ERR_LENGTH

496 : RESPST_ERR_MALFORMED_WQE;

497

498 return RESPST_NONE;

499 }

500

501 static enum resp_states write_data_in(struct rxe_qp *qp,

502 struct rxe_pkt_info *pkt)

503 {

504 enum resp_states rc = RESPST_NONE;

505 int err;

506 int data_len = payload_size(pkt);

507

508 err = rxe_mem_copy(qp->resp.mr, qp->resp.va, payload_addr(pkt),

509 data_len, to_mem_obj, NULL);

510 if (err) {

511 rc = RESPST_ERR_RKEY_VIOLATION;

512 goto out;

513 }

514

515 qp->resp.va += data_len;

516 qp->resp.resid -= data_len;

517

518 out:

519 return rc;

520 }

521

522 /* Guarantee atomicity of atomic operations at the machine level. */

523 static DEFINE_SPINLOCK(atomic_ops_lock);

524

525 static enum resp_states process_atomic(struct rxe_qp *qp,

526 struct rxe_pkt_info *pkt)

527 {

528 u64 iova = atmeth_va(pkt);

529 u64 *vaddr;

530 enum resp_states ret;

531 struct rxe_mem *mr = qp->resp.mr;

532

533 if (mr->state != RXE_MEM_STATE_VALID) {

534 ret = RESPST_ERR_RKEY_VIOLATION;

535 goto out;

536 }

537

538 vaddr = iova_to_vaddr(mr, iova, sizeof(u64));

539

540 /* check vaddr is 8 bytes aligned. */

541 if (!vaddr || (uintptr_t)vaddr & 7) {

542 ret = RESPST_ERR_MISALIGNED_ATOMIC;

543 goto out;

544 }

545

546 spin_lock_bh(&atomic_ops_lock);

547

548 qp->resp.atomic_orig = *vaddr;

549

550 if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP ||

551 pkt->opcode == IB_OPCODE_RD_COMPARE_SWAP) {

552 if (*vaddr == atmeth_comp(pkt))

553 *vaddr = atmeth_swap_add(pkt);

554 } else {

555 *vaddr += atmeth_swap_add(pkt);

556 }

557

558 spin_unlock_bh(&atomic_ops_lock);

559

560 ret = RESPST_NONE;

561 out:

562 return ret;

563 }

564

565 static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,

566 struct rxe_pkt_info *pkt,

567 struct rxe_pkt_info *ack,

568 int opcode,

569 int payload,

570 u32 psn,

571 u8 syndrome,

572 u32 *crcp)

573 {

574 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

575 struct sk_buff *skb;

576 u32 crc = 0;

577 u32 *p;

578 int paylen;

579 int pad;

580 int err;

581

582 /*

583 * allocate packet

584 */

585 pad = (-payload) & 0x3;

586 paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;

587

588 skb = rxe_init_packet(rxe, &qp->pri_av, paylen, ack);

589 if (!skb)

590 return NULL;

591

592 ack->qp = qp;

593 ack->opcode = opcode;

594 ack->mask = rxe_opcode[opcode].mask;

595 ack->offset = pkt->offset;

596 ack->paylen = paylen;

597

598 /* fill in bth using the request packet headers */

599 memcpy(ack->hdr, pkt->hdr, pkt->offset + RXE_BTH_BYTES);

600

601 bth_set_opcode(ack, opcode);

602 bth_set_qpn(ack, qp->attr.dest_qp_num);

603 bth_set_pad(ack, pad);

604 bth_set_se(ack, 0);

605 bth_set_psn(ack, psn);

606 bth_set_ack(ack, 0);

607 ack->psn = psn;

608

609 if (ack->mask & RXE_AETH_MASK) {

610 aeth_set_syn(ack, syndrome);

611 aeth_set_msn(ack, qp->resp.msn);

612 }

613

614 if (ack->mask & RXE_ATMACK_MASK)

615 atmack_set_orig(ack, qp->resp.atomic_orig);

616

617 err = rxe_prepare(ack, skb, &crc);

618 if (err) {

619 kfree_skb(skb);

620 return NULL;

621 }

622

623 if (crcp) {

624 /* CRC computation will be continued by the caller */

625 *crcp = crc;

626 } else {

627 p = payload_addr(ack) + payload + bth_pad(ack);

628 *p = ~crc;

629 }

630

631 return skb;

632 }

633

634 /* RDMA read response. If res is not NULL, then we have a current RDMA request

635 * being processed or replayed.

636 */

637 static enum resp_states read_reply(struct rxe_qp *qp,

638 struct rxe_pkt_info *req_pkt)

639 {

640 struct rxe_pkt_info ack_pkt;

641 struct sk_buff *skb;

642 int mtu = qp->mtu;

643 enum resp_states state;

644 int payload;

645 int opcode;

646 int err;

647 struct resp_res *res = qp->resp.res;

648 u32 icrc;

649 u32 *p;

650

651 if (!res) {

652 /* This is the first time we process that request. Get a

653 * resource

654 */

655 res = &qp->resp.resources[qp->resp.res_head];

656

657 free_rd_atomic_resource(qp, res);

658 rxe_advance_resp_resource(qp);

659

660 res->type = RXE_READ_MASK;

661 res->replay = 0;

662

663 res->read.va = qp->resp.va;

664 res->read.va_org = qp->resp.va;

665

666 res->first_psn = req_pkt->psn;

667

668 if (reth_len(req_pkt)) {

669 res->last_psn = (req_pkt->psn +

670 (reth_len(req_pkt) + mtu - 1) /

671 mtu - 1) & BTH_PSN_MASK;

672 } else {

673 res->last_psn = res->first_psn;

674 }

675 res->cur_psn = req_pkt->psn;

676

677 res->read.resid = qp->resp.resid;

678 res->read.length = qp->resp.resid;

679 res->read.rkey = qp->resp.rkey;

680

681 /* note res inherits the reference to mr from qp */

682 res->read.mr = qp->resp.mr;

683 qp->resp.mr = NULL;

684

685 qp->resp.res = res;

686 res->state = rdatm_res_state_new;

687 }

688

689 if (res->state == rdatm_res_state_new) {

690 if (res->read.resid <= mtu)

691 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY;

692 else

693 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST;

694 } else {

695 if (res->read.resid > mtu)

696 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE;

697 else

698 opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;

699 }

700

701 res->state = rdatm_res_state_next;

702

703 payload = min_t(int, res->read.resid, mtu);

704

705 skb = prepare_ack_packet(qp, req_pkt, &ack_pkt, opcode, payload,

706 res->cur_psn, AETH_ACK_UNLIMITED, &icrc);

707 if (!skb)

708 return RESPST_ERR_RNR;

709

710 err = rxe_mem_copy(res->read.mr, res->read.va, payload_addr(&ack_pkt),

711 payload, from_mem_obj, &icrc);

712 if (err)

713 pr_err("Failed copying memory\n");

714

715 if (bth_pad(&ack_pkt)) {

716 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

717 u8 *pad = payload_addr(&ack_pkt) + payload;

718

719 memset(pad, 0, bth_pad(&ack_pkt));

720 icrc = rxe_crc32(rxe, icrc, pad, bth_pad(&ack_pkt));

721 }

722 p = payload_addr(&ack_pkt) + payload + bth_pad(&ack_pkt);

723 *p = ~icrc;

724

725 err = rxe_xmit_packet(qp, &ack_pkt, skb);

726 if (err) {

727 pr_err("Failed sending RDMA reply.\n");

728 return RESPST_ERR_RNR;

729 }

730

731 res->read.va += payload;

732 res->read.resid -= payload;

733 res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK;

734

735 if (res->read.resid > 0) {

736 state = RESPST_DONE;

737 } else {

738 qp->resp.res = NULL;

739 if (!res->replay)

740 qp->resp.opcode = -1;

741 if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)

742 qp->resp.psn = res->cur_psn;

743 state = RESPST_CLEANUP;

744 }

745

746 return state;

747 }

748

749 static void build_rdma_network_hdr(union rdma_network_hdr *hdr,

750 struct rxe_pkt_info *pkt)

751 {

752 struct sk_buff *skb = PKT_TO_SKB(pkt);

753

754 memset(hdr, 0, sizeof(*hdr));

755 if (skb->protocol == htons(ETH_P_IP))

756 memcpy(&hdr->roce4grh, ip_hdr(skb), sizeof(hdr->roce4grh));

757 else if (skb->protocol == htons(ETH_P_IPV6))

758 memcpy(&hdr->ibgrh, ipv6_hdr(skb), sizeof(hdr->ibgrh));

759 }

760

761 /* Executes a new request. A retried request never reach that function (send

762 * and writes are discarded, and reads and atomics are retried elsewhere.

763 */

764 static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)

765 {

766 enum resp_states err;

767

768 if (pkt->mask & RXE_SEND_MASK) {

769 if (qp_type(qp) == IB_QPT_UD ||

770 qp_type(qp) == IB_QPT_SMI ||

771 qp_type(qp) == IB_QPT_GSI) {

772 union rdma_network_hdr hdr;

773

774 build_rdma_network_hdr(&hdr, pkt);

775

776 err = send_data_in(qp, &hdr, sizeof(hdr));

777 if (err)

778 return err;

779 }

780 err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));

781 if (err)

782 return err;

783 } else if (pkt->mask & RXE_WRITE_MASK) {

784 err = write_data_in(qp, pkt);

785 if (err)

786 return err;

787 } else if (pkt->mask & RXE_READ_MASK) {

788 /* For RDMA Read we can increment the msn now. See C9-148. */

789 qp->resp.msn++;

790 return RESPST_READ_REPLY;

791 } else if (pkt->mask & RXE_ATOMIC_MASK) {

792 err = process_atomic(qp, pkt);

793 if (err)

794 return err;

795 } else {

796 /* Unreachable */

797 WARN_ON_ONCE(1);

798 }

799

800 /* next expected psn, read handles this separately */

801 qp->resp.psn = (pkt->psn + 1) & BTH_PSN_MASK;

802 qp->resp.ack_psn = qp->resp.psn;

803

804 qp->resp.opcode = pkt->opcode;

805 qp->resp.status = IB_WC_SUCCESS;

806

807 if (pkt->mask & RXE_COMP_MASK) {

808 /* We successfully processed this new request. */

809 qp->resp.msn++;

810 return RESPST_COMPLETE;

811 } else if (qp_type(qp) == IB_QPT_RC)

812 return RESPST_ACKNOWLEDGE;

813 else

814 return RESPST_CLEANUP;

815 }

816

817 static enum resp_states do_complete(struct rxe_qp *qp,

818 struct rxe_pkt_info *pkt)

819 {

820 struct rxe_cqe cqe;

821 struct ib_wc *wc = &cqe.ibwc;

822 struct ib_uverbs_wc *uwc = &cqe.uibwc;

823 struct rxe_recv_wqe *wqe = qp->resp.wqe;

824 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

825

826 if (unlikely(!wqe))

827 return RESPST_CLEANUP;

828

829 memset(&cqe, 0, sizeof(cqe));

830

831 if (qp->rcq->is_user) {

832 uwc->status = qp->resp.status;

833 uwc->qp_num = qp->ibqp.qp_num;

834 uwc->wr_id = wqe->wr_id;

835 } else {

836 wc->status = qp->resp.status;

837 wc->qp = &qp->ibqp;

838 wc->wr_id = wqe->wr_id;

839 }

840

841 if (wc->status == IB_WC_SUCCESS) {

842 rxe_counter_inc(rxe, RXE_CNT_RDMA_RECV);

843 wc->opcode = (pkt->mask & RXE_IMMDT_MASK &&

844 pkt->mask & RXE_WRITE_MASK) ?

845 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;

846 wc->vendor_err = 0;

847 wc->byte_len = (pkt->mask & RXE_IMMDT_MASK &&

848 pkt->mask & RXE_WRITE_MASK) ?

849 qp->resp.length : wqe->dma.length - wqe->dma.resid;

850

851 /* fields after byte_len are different between kernel and user

852 * space

853 */

854 if (qp->rcq->is_user) {

855 uwc->wc_flags = IB_WC_GRH;

856

857 if (pkt->mask & RXE_IMMDT_MASK) {

858 uwc->wc_flags |= IB_WC_WITH_IMM;

859 uwc->ex.imm_data = immdt_imm(pkt);

860 }

861

862 if (pkt->mask & RXE_IETH_MASK) {

863 uwc->wc_flags |= IB_WC_WITH_INVALIDATE;

864 uwc->ex.invalidate_rkey = ieth_rkey(pkt);

865 }

866

867 uwc->qp_num = qp->ibqp.qp_num;

868

869 if (pkt->mask & RXE_DETH_MASK)

870 uwc->src_qp = deth_sqp(pkt);

871

872 uwc->port_num = qp->attr.port_num;

873 } else {

874 struct sk_buff *skb = PKT_TO_SKB(pkt);

875

876 wc->wc_flags = IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE;

877 if (skb->protocol == htons(ETH_P_IP))

878 wc->network_hdr_type = RDMA_NETWORK_IPV4;

879 else

880 wc->network_hdr_type = RDMA_NETWORK_IPV6;

881

882 if (is_vlan_dev(skb->dev)) {

883 wc->wc_flags |= IB_WC_WITH_VLAN;

884 wc->vlan_id = vlan_dev_vlan_id(skb->dev);

885 }

886

887 if (pkt->mask & RXE_IMMDT_MASK) {

888 wc->wc_flags |= IB_WC_WITH_IMM;

889 wc->ex.imm_data = immdt_imm(pkt);

890 }

891

892 if (pkt->mask & RXE_IETH_MASK) {

893 struct rxe_mem *rmr;

894

895 wc->wc_flags |= IB_WC_WITH_INVALIDATE;

896 wc->ex.invalidate_rkey = ieth_rkey(pkt);

897

898 rmr = rxe_pool_get_index(&rxe->mr_pool,

899 wc->ex.invalidate_rkey >> 8);

900 if (unlikely(!rmr)) {

901 pr_err("Bad rkey %#x invalidation\n",

902 wc->ex.invalidate_rkey);

903 return RESPST_ERROR;

904 }

905 rmr->state = RXE_MEM_STATE_FREE;

906 rxe_drop_ref(rmr);

907 }

908

909 wc->qp = &qp->ibqp;

910

911 if (pkt->mask & RXE_DETH_MASK)

912 wc->src_qp = deth_sqp(pkt);

913

914 wc->port_num = qp->attr.port_num;

915 }

916 }

917

918 /* have copy for srq and reference for !srq */

919 if (!qp->srq)

920 advance_consumer(qp->rq.queue);

921

922 qp->resp.wqe = NULL;

923

924 if (rxe_cq_post(qp->rcq, &cqe, pkt ? bth_se(pkt) : 1))

925 return RESPST_ERR_CQ_OVERFLOW;

926

927 if (qp->resp.state == QP_STATE_ERROR)

928 return RESPST_CHK_RESOURCE;

929

930 if (!pkt)

931 return RESPST_DONE;

932 else if (qp_type(qp) == IB_QPT_RC)

933 return RESPST_ACKNOWLEDGE;

934 else

935 return RESPST_CLEANUP;

936 }

937

938 static int send_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,

939 u8 syndrome, u32 psn)

940 {

941 int err = 0;

942 struct rxe_pkt_info ack_pkt;

943 struct sk_buff *skb;

944

945 skb = prepare_ack_packet(qp, pkt, &ack_pkt, IB_OPCODE_RC_ACKNOWLEDGE,

946 0, psn, syndrome, NULL);

947 if (!skb) {

948 err = -ENOMEM;

949 goto err1;

950 }

951

952 err = rxe_xmit_packet(qp, &ack_pkt, skb);

953 if (err)

954 pr_err_ratelimited("Failed sending ack\n");

955

956 err1:

957 return err;

958 }

959

960 static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,

961 u8 syndrome)

962 {

963 int rc = 0;

964 struct rxe_pkt_info ack_pkt;

965 struct sk_buff *skb;

966 struct resp_res *res;

967

968 skb = prepare_ack_packet(qp, pkt, &ack_pkt,

969 IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE, 0, pkt->psn,

970 syndrome, NULL);

971 if (!skb) {

972 rc = -ENOMEM;

973 goto out;

974 }

975

976 res = &qp->resp.resources[qp->resp.res_head];

977 free_rd_atomic_resource(qp, res);

978 rxe_advance_resp_resource(qp);

979

980 memcpy(SKB_TO_PKT(skb), &ack_pkt, sizeof(ack_pkt));

981 memset((unsigned char *)SKB_TO_PKT(skb) + sizeof(ack_pkt), 0,

982 sizeof(skb->cb) - sizeof(ack_pkt));

983

984 skb_get(skb);

985 res->type = RXE_ATOMIC_MASK;

986 res->atomic.skb = skb;

987 res->first_psn = ack_pkt.psn;

988 res->last_psn = ack_pkt.psn;

989 res->cur_psn = ack_pkt.psn;

990

991 rc = rxe_xmit_packet(qp, &ack_pkt, skb);

992 if (rc) {

993 pr_err_ratelimited("Failed sending ack\n");

994 rxe_drop_ref(qp);

995 }

996 out:

997 return rc;

998 }

999

1000 static enum resp_states acknowledge(struct rxe_qp *qp,

1001 struct rxe_pkt_info *pkt)

1002 {

1003 if (qp_type(qp) != IB_QPT_RC)

1004 return RESPST_CLEANUP;

1005

1006 if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED)

1007 send_ack(qp, pkt, qp->resp.aeth_syndrome, pkt->psn);

1008 else if (pkt->mask & RXE_ATOMIC_MASK)

1009 send_atomic_ack(qp, pkt, AETH_ACK_UNLIMITED);

1010 else if (bth_ack(pkt))

1011 send_ack(qp, pkt, AETH_ACK_UNLIMITED, pkt->psn);

1012

1013 return RESPST_CLEANUP;

1014 }

1015

1016 static enum resp_states cleanup(struct rxe_qp *qp,

1017 struct rxe_pkt_info *pkt)

1018 {

1019 struct sk_buff *skb;

1020

1021 if (pkt) {

1022 skb = skb_dequeue(&qp->req_pkts);

1023 rxe_drop_ref(qp);

1024 kfree_skb(skb);

1025 }

1026

1027 if (qp->resp.mr) {

1028 rxe_drop_ref(qp->resp.mr);

1029 qp->resp.mr = NULL;

1030 }

1031

1032 return RESPST_DONE;

1033 }

1034

1035 static struct resp_res *find_resource(struct rxe_qp *qp, u32 psn)

1036 {

1037 int i;

1038

1039 for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {

1040 struct resp_res *res = &qp->resp.resources[i];

1041

1042 if (res->type == 0)

1043 continue;

1044

1045 if (psn_compare(psn, res->first_psn) >= 0 &&

1046 psn_compare(psn, res->last_psn) <= 0) {

1047 return res;

1048 }

1049 }

1050

1051 return NULL;

1052 }

1053

1054 static enum resp_states duplicate_request(struct rxe_qp *qp,

1055 struct rxe_pkt_info *pkt)

1056 {

1057 enum resp_states rc;

1058 u32 prev_psn = (qp->resp.ack_psn - 1) & BTH_PSN_MASK;

1059

1060 if (pkt->mask & RXE_SEND_MASK ||

1061 pkt->mask & RXE_WRITE_MASK) {

1062 /* SEND. Ack again and cleanup. C9-105. */

1063 if (bth_ack(pkt))

1064 send_ack(qp, pkt, AETH_ACK_UNLIMITED, prev_psn);

1065 rc = RESPST_CLEANUP;

1066 goto out;

1067 } else if (pkt->mask & RXE_READ_MASK) {

1068 struct resp_res *res;

1069

1070 res = find_resource(qp, pkt->psn);

1071 if (!res) {

1072 /* Resource not found. Class D error. Drop the

1073 * request.

1074 */

1075 rc = RESPST_CLEANUP;

1076 goto out;

1077 } else {

1078 /* Ensure this new request is the same as the previous

1079 * one or a subset of it.

1080 */

1081 u64 iova = reth_va(pkt);

1082 u32 resid = reth_len(pkt);

1083

1084 if (iova < res->read.va_org ||

1085 resid > res->read.length ||

1086 (iova + resid) > (res->read.va_org +

1087 res->read.length)) {

1088 rc = RESPST_CLEANUP;

1089 goto out;

1090 }

1091

1092 if (reth_rkey(pkt) != res->read.rkey) {

1093 rc = RESPST_CLEANUP;

1094 goto out;

1095 }

1096

1097 res->cur_psn = pkt->psn;

1098 res->state = (pkt->psn == res->first_psn) ?

1099 rdatm_res_state_new :

1100 rdatm_res_state_replay;

1101 res->replay = 1;

1102

1103 /* Reset the resource, except length. */

1104 res->read.va_org = iova;

1105 res->read.va = iova;

1106 res->read.resid = resid;

1107

1108 /* Replay the RDMA read reply. */

1109 qp->resp.res = res;

1110 rc = RESPST_READ_REPLY;

1111 goto out;

1112 }

1113 } else {

1114 struct resp_res *res;

1115

1116 /* Find the operation in our list of responder resources. */

1117 res = find_resource(qp, pkt->psn);

1118 if (res) {

1119 skb_get(res->atomic.skb);

1120 /* Resend the result. */

1121 rc = rxe_xmit_packet(qp, pkt, res->atomic.skb);

1122 if (rc) {

1123 pr_err("Failed resending result. This flow is not handled - skb ignored\n");

1124 rc = RESPST_CLEANUP;

1125 goto out;

1126 }

1127 }

1128

1129 /* Resource not found. Class D error. Drop the request. */

1130 rc = RESPST_CLEANUP;

1131 goto out;

1132 }

1133 out:

1134 return rc;

1135 }

1136

1137 /* Process a class A or C. Both are treated the same in this implementation. */

1138 static void do_class_ac_error(struct rxe_qp *qp, u8 syndrome,

1139 enum ib_wc_status status)

1140 {

1141 qp->resp.aeth_syndrome = syndrome;

1142 qp->resp.status = status;

1143

1144 /* indicate that we should go through the ERROR state */

1145 qp->resp.goto_error = 1;

1146 }

1147

1148 static enum resp_states do_class_d1e_error(struct rxe_qp *qp)

1149 {

1150 /* UC */

1151 if (qp->srq) {

1152 /* Class E */

1153 qp->resp.drop_msg = 1;

1154 if (qp->resp.wqe) {

1155 qp->resp.status = IB_WC_REM_INV_REQ_ERR;

1156 return RESPST_COMPLETE;

1157 } else {

1158 return RESPST_CLEANUP;

1159 }

1160 } else {

1161 /* Class D1. This packet may be the start of a

1162 * new message and could be valid. The previous

1163 * message is invalid and ignored. reset the

1164 * recv wr to its original state

1165 */

1166 if (qp->resp.wqe) {

1167 qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length;

1168 qp->resp.wqe->dma.cur_sge = 0;

1169 qp->resp.wqe->dma.sge_offset = 0;

1170 qp->resp.opcode = -1;

1171 }

1172

1173 if (qp->resp.mr) {

1174 rxe_drop_ref(qp->resp.mr);

1175 qp->resp.mr = NULL;

1176 }

1177

1178 return RESPST_CLEANUP;

1179 }

1180 }

1181

1182 static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)

1183 {

1184 struct sk_buff *skb;

1185

1186 while ((skb = skb_dequeue(&qp->req_pkts))) {

1187 rxe_drop_ref(qp);

1188 kfree_skb(skb);

1189 }

1190

1191 if (notify)

1192 return;

1193

1194 while (!qp->srq && qp->rq.queue && queue_head(qp->rq.queue))

1195 advance_consumer(qp->rq.queue);

1196 }

1197

1198 int rxe_responder(void *arg)

1199 {

1200 struct rxe_qp *qp = (struct rxe_qp *)arg;

1201 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);

1202 enum resp_states state;

1203 struct rxe_pkt_info *pkt = NULL;

1204 int ret = 0;

1205

1206 rxe_add_ref(qp);

1207

1208 qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;

1209

1210 if (!qp->valid) {

1211 ret = -EINVAL;

1212 goto done;

1213 }

1214

1215 switch (qp->resp.state) {

1216 case QP_STATE_RESET:

1217 state = RESPST_RESET;

1218 break;

1219

1220 default:

1221 state = RESPST_GET_REQ;

1222 break;

1223 }

1224

1225 while (1) {

1226 pr_debug("qp#%d state = %s\n", qp_num(qp),

1227 resp_state_name[state]);

1228 switch (state) {

1229 case RESPST_GET_REQ:

1230 state = get_req(qp, &pkt);

1231 break;

1232 case RESPST_CHK_PSN:

1233 state = check_psn(qp, pkt);

1234 break;

1235 case RESPST_CHK_OP_SEQ:

1236 state = check_op_seq(qp, pkt);

1237 break;

1238 case RESPST_CHK_OP_VALID:

1239 state = check_op_valid(qp, pkt);

1240 break;

1241 case RESPST_CHK_RESOURCE:

1242 state = check_resource(qp, pkt);

1243 break;

1244 case RESPST_CHK_LENGTH:

1245 state = check_length(qp, pkt);

1246 break;

1247 case RESPST_CHK_RKEY:

1248 state = check_rkey(qp, pkt);

1249 break;

1250 case RESPST_EXECUTE:

1251 state = execute(qp, pkt);

1252 break;

1253 case RESPST_COMPLETE:

1254 state = do_complete(qp, pkt);

1255 break;

1256 case RESPST_READ_REPLY:

1257 state = read_reply(qp, pkt);

1258 break;

1259 case RESPST_ACKNOWLEDGE:

1260 state = acknowledge(qp, pkt);

1261 break;

1262 case RESPST_CLEANUP:

1263 state = cleanup(qp, pkt);

1264 break;

1265 case RESPST_DUPLICATE_REQUEST:

1266 state = duplicate_request(qp, pkt);

1267 break;

1268 case RESPST_ERR_PSN_OUT_OF_SEQ:

1269 /* RC only - Class B. Drop packet. */

1270 send_ack(qp, pkt, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn);

1271 state = RESPST_CLEANUP;

1272 break;

1273

1274 case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ:

1275 case RESPST_ERR_MISSING_OPCODE_FIRST:

1276 case RESPST_ERR_MISSING_OPCODE_LAST_C:

1277 case RESPST_ERR_UNSUPPORTED_OPCODE:

1278 case RESPST_ERR_MISALIGNED_ATOMIC:

1279 /* RC Only - Class C. */

1280 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,

1281 IB_WC_REM_INV_REQ_ERR);

1282 state = RESPST_COMPLETE;

1283 break;

1284

1285 case RESPST_ERR_MISSING_OPCODE_LAST_D1E:

1286 state = do_class_d1e_error(qp);

1287 break;

1288 case RESPST_ERR_RNR:

1289 if (qp_type(qp) == IB_QPT_RC) {

1290 rxe_counter_inc(rxe, RXE_CNT_SND_RNR);

1291 /* RC - class B */

1292 send_ack(qp, pkt, AETH_RNR_NAK |

1293 (~AETH_TYPE_MASK &

1294 qp->attr.min_rnr_timer),

1295 pkt->psn);

1296 } else {

1297 /* UD/UC - class D */

1298 qp->resp.drop_msg = 1;

1299 }

1300 state = RESPST_CLEANUP;

1301 break;

1302

1303 case RESPST_ERR_RKEY_VIOLATION:

1304 if (qp_type(qp) == IB_QPT_RC) {

1305 /* Class C */

1306 do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,

1307 IB_WC_REM_ACCESS_ERR);

1308 state = RESPST_COMPLETE;

1309 } else {

1310 qp->resp.drop_msg = 1;

1311 if (qp->srq) {

1312 /* UC/SRQ Class D */

1313 qp->resp.status = IB_WC_REM_ACCESS_ERR;

1314 state = RESPST_COMPLETE;

1315 } else {

1316 /* UC/non-SRQ Class E. */

1317 state = RESPST_CLEANUP;

1318 }

1319 }

1320 break;

1321

1322 case RESPST_ERR_LENGTH:

1323 if (qp_type(qp) == IB_QPT_RC) {

1324 /* Class C */

1325 do_class_ac_error(qp, AETH_NAK_INVALID_REQ,

1326 IB_WC_REM_INV_REQ_ERR);

1327 state = RESPST_COMPLETE;

1328 } else if (qp->srq) {

1329 /* UC/UD - class E */

1330 qp->resp.status = IB_WC_REM_INV_REQ_ERR;

1331 state = RESPST_COMPLETE;

1332 } else {

1333 /* UC/UD - class D */

1334 qp->resp.drop_msg = 1;

1335 state = RESPST_CLEANUP;

1336 }

1337 break;

1338

1339 case RESPST_ERR_MALFORMED_WQE:

1340 /* All, Class A. */

1341 do_class_ac_error(qp, AETH_NAK_REM_OP_ERR,

1342 IB_WC_LOC_QP_OP_ERR);

1343 state = RESPST_COMPLETE;

1344 break;

1345

1346 case RESPST_ERR_CQ_OVERFLOW:

1347 /* All - Class G */

1348 state = RESPST_ERROR;

1349 break;

1350

1351 case RESPST_DONE:

1352 if (qp->resp.goto_error) {

1353 state = RESPST_ERROR;

1354 break;

1355 }

1356

1357 goto done;

1358

1359 case RESPST_EXIT:

1360 if (qp->resp.goto_error) {

1361 state = RESPST_ERROR;

1362 break;

1363 }

1364

1365 goto exit;

1366

1367 case RESPST_RESET:

1368 rxe_drain_req_pkts(qp, false);

1369 qp->resp.wqe = NULL;

1370 goto exit;

1371

1372 case RESPST_ERROR:

1373 qp->resp.goto_error = 0;

1374 pr_warn("qp#%d moved to error state\n", qp_num(qp));

1375 rxe_qp_error(qp);

1376 goto exit;

1377

1378 default:

1379 WARN_ON_ONCE(1);

1380 }

1381 }

1382

1383 exit:

1384 ret = -EAGAIN;

1385 done:

1386 rxe_drop_ref(qp);

1387 return ret;

1388 }