/* $NetBSD: iscsi_send.c,v 1.41 2024/08/24 09:39:44 mlelstv Exp $ */

/* $NetBSD: iscsi_send.c,v 1.41 2024/08/24 09:39:44 mlelstv Exp $ */

/*-
* Copyright (c) 2004,2005,2006,2011 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Wasabi Systems, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "iscsi_globals.h"

#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/socket.h>
#include <sys/socketvar.h>

/*#define LUN_1 1 */

/*****************************************************************************/

/*
* my_soo_write:
* Replacement for soo_write with flag handling.
*
* Parameter:
* conn The connection
* u The uio descriptor
*
* Returns: 0 on success, else EIO.
*/

STATIC int
my_soo_write(connection_t *conn, struct uio *u)
{
struct socket *so;
int ret;
#ifdef ISCSI_DEBUG
size_t resid = u->uio_resid;
#endif

KASSERT(u->uio_resid != 0);

rw_enter(&conn->c_sock_rw, RW_READER);
if (conn->c_sock == NULL) {
ret = EIO;
} else {
so = conn->c_sock->f_socket;
ret = (*so->so_send)(so, NULL, u,
NULL, NULL, 0, conn->c_threadobj);
}
rw_exit(&conn->c_sock_rw);

DEB(99, ("soo_write done: len = %zu\n", u->uio_resid));

if (ret != 0 || u->uio_resid) {
DEBC(conn, 0, ("Write failed sock %p (ret: %d, req: %zu, resid: %zu)\n",
conn->c_sock, ret, resid, u->uio_resid));
handle_connection_error(conn, ISCSI_STATUS_SOCKET_ERROR, NO_LOGOUT);
return EIO;
}
return 0;
}

/*****************************************************************************/

/*
* assign_connection:
* This function returns the connection to use for the next transaction.
*
* Parameter: The session
*
* Returns: The connection
*/

connection_t *
assign_connection(session_t *sess, bool waitok)
{
connection_t *conn, *next;

mutex_enter(&sess->s_lock);
do {
if (sess->s_terminating ||
(conn = sess->s_mru_connection) == NULL) {
mutex_exit(&sess->s_lock);
return NULL;
}
next = conn;
do {
next = TAILQ_NEXT(next, c_connections);
if (next == NULL) {
next = TAILQ_FIRST(&sess->s_conn_list);
}
} while (next != NULL && next != conn &&
next->c_state != ST_FULL_FEATURE);

if (next->c_state != ST_FULL_FEATURE) {
if (waitok) {
cv_wait(&sess->s_sess_cv, &sess->s_lock);
next = TAILQ_FIRST(&sess->s_conn_list);
} else {
mutex_exit(&sess->s_lock);
return NULL;
}
} else {
sess->s_mru_connection = next;
}
} while (next != NULL && next->c_state != ST_FULL_FEATURE);
mutex_exit(&sess->s_lock);

return next;
}

/*
* reassign_tasks:
* Reassign pending commands to one of the still existing connections
* of a session.
*
* Parameter:
* oldconn The terminating connection
*/

STATIC void
reassign_tasks(connection_t *oldconn)
{
session_t *sess = oldconn->c_session;
connection_t *conn;
ccb_t *ccb;
ccb_list_t old_waiting;
pdu_t *pdu = NULL;
pdu_t *opdu;
int no_tm = 1;
int rc = 1;
uint32_t sn;

if ((conn = assign_connection(sess, FALSE)) == NULL) {
DEB(1, ("Reassign_tasks of Session %d, connection %d failed, "
"no active connection\n",
sess->s_id, oldconn->c_id));
/* XXX here we need to abort the waiting CCBs */
return;
}

TAILQ_INIT(&old_waiting);

mutex_enter(&oldconn->c_lock);

if (sess->s_ErrorRecoveryLevel >= 2) {
if (oldconn->c_loggedout == NOT_LOGGED_OUT) {
oldconn->c_loggedout = LOGOUT_SENT;
no_tm = send_logout(conn, oldconn, RECOVER_CONNECTION, TRUE);
oldconn->c_loggedout = (rc) ? LOGOUT_FAILED : LOGOUT_SUCCESS;
if (!oldconn->c_Time2Retain) {
DEBC(conn, 1, ("Time2Retain is zero, setting no_tm\n"));
no_tm = 1;
}
} else if (oldconn->c_loggedout == LOGOUT_SUCCESS) {
no_tm = 0;
}
if (!no_tm && oldconn->c_Time2Wait) {
DEBC(conn, 1, ("Time2Wait=%d, hz=%d, waiting...\n",
oldconn->c_Time2Wait, hz));
kpause("Time2Wait", false, oldconn->c_Time2Wait * hz, &oldconn->c_lock);
}
}

while ((ccb = TAILQ_FIRST(&oldconn->c_ccbs_waiting)) != NULL) {
suspend_ccb(ccb, FALSE);
TAILQ_INSERT_TAIL(&old_waiting, ccb, ccb_chain);
}

mutex_exit(&oldconn->c_lock);

DEBC(conn, 1, ("Reassign_tasks: S%dC%d -> S%dC%d, no_tm=%d, pdus old %d + new %d\n",
sess->s_id, oldconn->c_id, sess->s_id, conn->c_id, no_tm,
oldconn->c_pducount, conn->c_pducount));

/* XXX reassign waiting CCBs to new connection */

while ((ccb = TAILQ_FIRST(&old_waiting)) != NULL) {
/* Copy PDU contents (PDUs are bound to connection) */
if ((pdu = get_pdu(conn, TRUE)) == NULL) {
DEBC(conn, 0, ("get_pdu failed, terminating=%d\n", conn->c_terminating));
/* new connection is terminating */
break;
}

TAILQ_REMOVE(&old_waiting, ccb, ccb_chain);

/* adjust CCB and clone PDU for new connection */

opdu = ccb->ccb_pdu_waiting;
KASSERT((opdu->pdu_flags & PDUF_INQUEUE) == 0);

*pdu = *opdu;

/* restore overwritten back ptr */
pdu->pdu_connection = conn;

/* fixup saved UIO and IOVEC (regular one will be overwritten anyway) */
pdu->pdu_save_uio.uio_iov = pdu->pdu_io_vec;
pdu->pdu_save_iovec [0].iov_base = &pdu->pdu_hdr;

if (conn->c_DataDigest && pdu->pdu_save_uio.uio_iovcnt > 1) {
if (pdu->pdu_save_iovec [2].iov_base == NULL) {
pdu->pdu_save_iovec [2].iov_base = &pdu->pdu_data_digest;
pdu->pdu_save_uio.uio_iovcnt = 3;
} else {
pdu->pdu_save_iovec [3].iov_base = &pdu->pdu_data_digest;
pdu->pdu_save_uio.uio_iovcnt = 4;
}
}
pdu->pdu_save_iovec [0].iov_len =
(conn->c_HeaderDigest) ? BHS_SIZE + 4 : BHS_SIZE;

/* link new PDU into old CCB */
ccb->ccb_pdu_waiting = pdu;
/* link new CCB into new connection */
ccb->ccb_connection = conn;
/* reset timeouts */
ccb->ccb_num_timeouts = 0;

DEBC(conn, 1, ("CCB %p: Copied PDU %p to %p\n",
ccb, opdu, pdu));

/* kill temp pointer that is now referenced by the new PDU */
opdu->pdu_temp_data = NULL;

/* and free the old PDU */
free_pdu(opdu);

mutex_enter(&conn->c_lock);

/* fixup reference counts */
mutex_enter(&oldconn->c_lock);
oldconn->c_usecount--;
conn->c_usecount++;
mutex_exit(&oldconn->c_lock);

/* put ready CCB into waiting list of new connection */
suspend_ccb(ccb, TRUE);

mutex_exit(&conn->c_lock);
}

if (TAILQ_FIRST(&old_waiting) != NULL) {
DEBC(conn, 0, ("Error while copying PDUs in reassign_tasks!\n"));
/*
* give up recovering, the other connection is screwed up
* as well...
*/

while ((ccb = TAILQ_FIRST(&old_waiting)) != NULL) {
TAILQ_REMOVE(&old_waiting, ccb, ccb_chain);

DEBC(oldconn, 1, ("Wake CCB %p for connection %d, terminating %d\n",
ccb, ccb->ccb_connection->c_id, oldconn->c_terminating));
mutex_enter(&oldconn->c_lock);
suspend_ccb(ccb, TRUE);
mutex_exit(&oldconn->c_lock);
wake_ccb(ccb, oldconn->c_terminating);
}

return;
}

TAILQ_FOREACH(ccb, &conn->c_ccbs_waiting, ccb_chain) {
if (!no_tm) {
rc = send_task_management(conn, ccb, NULL, TASK_REASSIGN);
}
/* if we get an error on reassign, restart the original request */
if (no_tm || rc) {
mutex_enter(&sess->s_lock);
if (ccb->ccb_CmdSN < sess->s_ExpCmdSN) {
pdu = ccb->ccb_pdu_waiting;
sn = get_sernum(sess, pdu);

/* update CmdSN */
DEBC(conn, 1, ("Resend Updating CmdSN - old %d, new %d\n",
ccb->ccb_CmdSN, sn));
ccb->ccb_CmdSN = sn;
pdu->pdu_hdr.pduh_p.command.CmdSN = htonl(ccb->ccb_CmdSN);
}
mutex_exit(&sess->s_lock);
resend_pdu(ccb);
} else {
ccb_timeout_start(ccb, COMMAND_TIMEOUT);
}
DEBC(conn, 1, ("Reassign ccb %p, no_tm=%d, rc=%d\n",
ccb, no_tm, rc));
}
}

/*
* iscsi_send_thread:
* This thread services the send queue, writing the PDUs to the socket.
* It also handles the cleanup when the connection is terminated.
*
* Parameter:
* par The connection this thread services
*/

void
iscsi_send_thread(void *par)
{
connection_t *conn = (connection_t *) par;
session_t *sess;
ccb_t *ccb, *nccb;
pdu_t *pdu;
struct file *fp;
pdu_disp_t pdisp;

sess = conn->c_session;
/* so cleanup thread knows there's someone left */
iscsi_num_send_threads++;

do {
mutex_enter(&conn->c_lock);
while (!conn->c_terminating) {
while (!conn->c_terminating &&
(pdu = TAILQ_FIRST(&conn->c_pdus_to_send)) != NULL) {
TAILQ_REMOVE(&conn->c_pdus_to_send, pdu, pdu_send_chain);
pdu->pdu_flags &= ~PDUF_INQUEUE;
mutex_exit(&conn->c_lock);

/* update ExpStatSN here to avoid discontinuities */
/* and delays in updating target */
pdu->pdu_hdr.pduh_p.command.ExpStatSN = htonl(conn->c_StatSN_buf.ExpSN);

if (conn->c_HeaderDigest)
pdu->pdu_hdr.pduh_HeaderDigest = gen_digest(&pdu->pdu_hdr, BHS_SIZE);

DEBC(conn, 99, ("Transmitting PDU CmdSN = %u, ExpStatSN = %u\n",
ntohl(pdu->pdu_hdr.pduh_p.command.CmdSN),
ntohl(pdu->pdu_hdr.pduh_p.command.ExpStatSN)));
my_soo_write(conn, &pdu->pdu_uio);

mutex_enter(&conn->c_lock);
pdisp = pdu->pdu_disp;
if (pdisp > PDUDISP_FREE)
pdu->pdu_flags &= ~PDUF_BUSY;
mutex_exit(&conn->c_lock);
if (pdisp <= PDUDISP_FREE)
free_pdu(pdu);

mutex_enter(&conn->c_lock);
}

if (!conn->c_terminating)
cv_wait(&conn->c_conn_cv, &conn->c_lock);
}
mutex_exit(&conn->c_lock);

/* ------------------------------------------------------------------------
* Here this thread takes over cleanup of the terminating connection.
* ------------------------------------------------------------------------
*/
connection_timeout_stop(conn);
conn->c_idle_timeout_val = CONNECTION_IDLE_TIMEOUT;

fp = conn->c_sock;

/*
* We shutdown the socket here to force the receive
* thread to wake up
*/
DEBC(conn, 1, ("Closing Socket %p\n", conn->c_sock));
solock(fp->f_socket);
soshutdown(fp->f_socket, SHUT_RDWR);
sounlock(fp->f_socket);

/* wake up any non-reassignable waiting CCBs */
TAILQ_FOREACH_SAFE(ccb, &conn->c_ccbs_waiting, ccb_chain, nccb) {
if (!(ccb->ccb_flags & CCBF_REASSIGN) || ccb->ccb_pdu_waiting == NULL) {
DEBC(conn, 1, ("Terminating CCB %p (t=%p)\n",
ccb,&ccb->ccb_timeout));
wake_ccb(ccb, conn->c_terminating);
} else {
ccb_timeout_stop(ccb);
ccb->ccb_num_timeouts = 0;
}
}

/* clean out anything left in send queue */
mutex_enter(&conn->c_lock);
while ((pdu = TAILQ_FIRST(&conn->c_pdus_to_send)) != NULL) {
TAILQ_REMOVE(&conn->c_pdus_to_send, pdu, pdu_send_chain);
pdu->pdu_flags &= ~(PDUF_INQUEUE | PDUF_BUSY);
mutex_exit(&conn->c_lock);
/* if it's not attached to a waiting CCB, free it */
if (pdu->pdu_owner == NULL ||
pdu->pdu_owner->ccb_pdu_waiting != pdu) {
free_pdu(pdu);
}
mutex_enter(&conn->c_lock);
}
mutex_exit(&conn->c_lock);

/* If there's another connection available, transfer pending tasks */
if (sess->s_active_connections &&
TAILQ_FIRST(&conn->c_ccbs_waiting) != NULL) {

reassign_tasks(conn);
} else if (!conn->c_destroy && conn->c_Time2Wait) {
DEBC(conn, 1, ("Time2Wait\n"));
kpause("Time2Wait", false, conn->c_Time2Wait * hz, NULL);
DEBC(conn, 1, ("Time2Wait\n"));
}
/* notify event handlers of connection shutdown */
DEBC(conn, 1, ("%s\n", conn->c_destroy ? "TERMINATED" : "RECOVER"));
add_event(conn->c_destroy ? ISCSI_CONNECTION_TERMINATED
: ISCSI_RECOVER_CONNECTION,
sess->s_id, conn->c_id, conn->c_terminating);

DEBC(conn, 1, ("Waiting for conn_idle\n"));
mutex_enter(&conn->c_lock);
if (!conn->c_destroy)
cv_timedwait(&conn->c_idle_cv, &conn->c_lock, CONNECTION_IDLE_TIMEOUT);
mutex_exit(&conn->c_lock);
DEBC(conn, 1, ("Waited for conn_idle, destroy = %d\n", conn->c_destroy));

} while (!conn->c_destroy);

/* wake up anyone waiting for a PDU */
mutex_enter(&conn->c_lock);
cv_broadcast(&conn->c_conn_cv);
mutex_exit(&conn->c_lock);

/* wake up any waiting CCBs */
while ((ccb = TAILQ_FIRST(&conn->c_ccbs_waiting)) != NULL) {
KASSERT(ccb->ccb_disp >= CCBDISP_NOWAIT);
wake_ccb(ccb, conn->c_terminating);
/* NOTE: wake_ccb will remove the CCB from the queue */
}

add_connection_cleanup(conn);

conn->c_sendproc = NULL;
DEBC(conn, 1, ("Send thread exits\n"));
iscsi_num_send_threads--;
kthread_exit(0);
}

/*
* send_pdu:
* Enqueue a PDU to be sent, and handle its disposition as well as
* the disposition of its associated CCB.
*
* Parameter:
* ccb The associated CCB. May be NULL if cdisp is CCBDISP_NOWAIT
* and pdisp is not PDUDISP_WAIT
* cdisp The CCB's disposition
* pdu The PDU
* pdisp The PDU's disposition
*/

STATIC void
send_pdu(ccb_t *ccb, pdu_t *pdu, ccb_disp_t cdisp, pdu_disp_t pdisp)
{
connection_t *conn = pdu->pdu_connection;
ccb_disp_t prev_cdisp = 0;

if (ccb != NULL) {
prev_cdisp = ccb->ccb_disp;
pdu->pdu_hdr.pduh_InitiatorTaskTag = ccb->ccb_ITT;
pdu->pdu_owner = ccb;
if (cdisp != CCBDISP_NOWAIT)
ccb->ccb_disp = cdisp;
}

pdu->pdu_disp = pdisp;

DEBC(conn, 10, ("Send_pdu: CmdSN=%u ExpStatSN~%u ccb=%p, pdu=%p\n",
ntohl(pdu->pdu_hdr.pduh_p.command.CmdSN),
conn->c_StatSN_buf.ExpSN,
ccb, pdu));

mutex_enter(&conn->c_lock);
if (pdisp == PDUDISP_WAIT) {
KASSERT(ccb != NULL);

ccb->ccb_pdu_waiting = pdu;

/* save UIO and IOVEC for retransmit */
pdu->pdu_save_uio = pdu->pdu_uio;
memcpy(pdu->pdu_save_iovec, pdu->pdu_io_vec, sizeof(pdu->pdu_save_iovec));

pdu->pdu_flags |= PDUF_BUSY;
}
/* Enqueue for sending */
pdu->pdu_flags |= PDUF_INQUEUE;

if (pdu->pdu_flags & PDUF_PRIORITY)
TAILQ_INSERT_HEAD(&conn->c_pdus_to_send, pdu, pdu_send_chain);
else
TAILQ_INSERT_TAIL(&conn->c_pdus_to_send, pdu, pdu_send_chain);

cv_broadcast(&conn->c_conn_cv);

if (cdisp != CCBDISP_NOWAIT) {
KASSERT(ccb != NULL);
KASSERTMSG(ccb->ccb_connection == conn, "conn mismatch %p != %p\n", ccb->ccb_connection, conn);

if (prev_cdisp <= CCBDISP_NOWAIT)
suspend_ccb(ccb, TRUE);

mutex_exit(&conn->c_lock);
ccb_timeout_start(ccb, COMMAND_TIMEOUT);
mutex_enter(&conn->c_lock);

while (ccb->ccb_disp == CCBDISP_WAIT) {
DEBC(conn, 15, ("Send_pdu: ccb=%p cdisp=%d waiting\n",
ccb, ccb->ccb_disp));
cv_wait(&conn->c_ccb_cv, &conn->c_lock);
DEBC(conn, 15, ("Send_pdu: ccb=%p cdisp=%d returned\n",
ccb, ccb->ccb_disp));
}
}

mutex_exit(&conn->c_lock);
}

/*
* resend_pdu:
* Re-Enqueue a PDU that has apparently gotten lost.
*
* Parameter:
* ccb The associated CCB.
*/

void
resend_pdu(ccb_t *ccb)
{
connection_t *conn = ccb->ccb_connection;
pdu_t *pdu = ccb->ccb_pdu_waiting;

mutex_enter(&conn->c_lock);
if (pdu == NULL || (pdu->pdu_flags & PDUF_BUSY)) {
mutex_exit(&conn->c_lock);
return;
}
pdu->pdu_flags |= PDUF_BUSY;
mutex_exit(&conn->c_lock);

/* restore UIO and IOVEC */
pdu->pdu_uio = pdu->pdu_save_uio;
memcpy(pdu->pdu_io_vec, pdu->pdu_save_iovec, sizeof(pdu->pdu_io_vec));

DEBC(conn, 8, ("ReSend_pdu: CmdSN=%u ExpStatSN~%u ccb=%p, pdu=%p\n",
ntohl(pdu->pdu_hdr.pduh_p.command.CmdSN),
conn->c_StatSN_buf.ExpSN,
ccb, pdu));

mutex_enter(&conn->c_lock);
/* Enqueue for sending */
pdu->pdu_flags |= PDUF_INQUEUE;

if (pdu->pdu_flags & PDUF_PRIORITY) {
TAILQ_INSERT_HEAD(&conn->c_pdus_to_send, pdu, pdu_send_chain);
} else {
TAILQ_INSERT_TAIL(&conn->c_pdus_to_send, pdu, pdu_send_chain);
}
cv_broadcast(&conn->c_conn_cv);
mutex_exit(&conn->c_lock);

ccb_timeout_start(ccb, COMMAND_TIMEOUT);
}

/*
* setup_tx_uio:
* Initialize the uio structure for sending, including header,
* data (if present), padding, and Data Digest.
* Header Digest is generated in send thread.
*
* Parameter:
* pdu The PDU
* dsl The Data Segment Length
* data The data pointer
* read TRUE if this is a read operation
*/

STATIC void
setup_tx_uio(pdu_t *pdu, uint32_t dsl, void *data, bool read)
{
static uint8_t pad_bytes[4] = { 0 };
struct uio *uio;
int i, pad, hlen;
connection_t *conn = pdu->pdu_connection;

DEB(99, ("SetupTxUio: dlen = %d, dptr: %p, read: %d\n",
dsl, data, read));

if (!read && dsl) {
hton3(dsl, pdu->pdu_hdr.pduh_DataSegmentLength);
}
hlen = (conn->c_HeaderDigest) ? BHS_SIZE + 4 : BHS_SIZE;

pdu->pdu_io_vec[0].iov_base = &pdu->pdu_hdr;
pdu->pdu_io_vec[0].iov_len = hlen;

uio = &pdu->pdu_uio;

uio->uio_iov = pdu->pdu_io_vec;
uio->uio_iovcnt = 1;
uio->uio_rw = UIO_WRITE;
uio->uio_resid = hlen;
UIO_SETUP_SYSSPACE(uio);

if (!read && dsl) {
uio->uio_iovcnt++;
pdu->pdu_io_vec[1].iov_base = data;
pdu->pdu_io_vec[1].iov_len = dsl;
uio->uio_resid += dsl;

/* Pad to next multiple of 4 */
pad = uio->uio_resid & 0x03;
if (pad) {
i = uio->uio_iovcnt++;
pad = 4 - pad;
pdu->pdu_io_vec[i].iov_base = pad_bytes;
pdu->pdu_io_vec[i].iov_len = pad;
uio->uio_resid += pad;
}

if (conn->c_DataDigest) {
pdu->pdu_data_digest = gen_digest_2(data, dsl, pad_bytes, pad);
i = uio->uio_iovcnt++;
pdu->pdu_io_vec[i].iov_base = &pdu->pdu_data_digest;
pdu->pdu_io_vec[i].iov_len = 4;
uio->uio_resid += 4;
}
}
}

/*
* init_login_pdu:
* Initialize the login PDU.
*
* Parameter:
* conn The connection
* ccb The CCB
* pdu The PDU
*/

STATIC void
init_login_pdu(connection_t *conn, ccb_t *ccb, pdu_t *ppdu, bool next)
{
pdu_header_t *hpdu = &ppdu->pdu_hdr;
login_isid_t *isid = (login_isid_t *) & hpdu->pduh_LUN;
uint8_t c_phase;

hpdu->pduh_Opcode = IOP_Login_Request | OP_IMMEDIATE;

mutex_enter(&conn->c_session->s_lock);
ccb->ccb_CmdSN = get_sernum(conn->c_session, ppdu);
mutex_exit(&conn->c_session->s_lock);

if (next) {
c_phase = (hpdu->pduh_Flags >> CSG_SHIFT) & SG_MASK;
hpdu->pduh_Flags = FLAG_TRANSIT | (c_phase << CSG_SHIFT) |
NEXT_PHASE(c_phase);
}

DEB(99, ("InitLoginPdu: Flags=%x Phase=%x->%x\n",
hpdu->pduh_Flags,
(hpdu->pduh_Flags >> CSG_SHIFT) & SG_MASK,
hpdu->pduh_Flags & SG_MASK));

memcpy(isid, &iscsi_InitiatorISID, 6);
isid->TSIH = conn->c_session->s_TSIH;

hpdu->pduh_p.login_req.CID = htons(conn->c_id);
hpdu->pduh_p.login_req.CmdSN = htonl(ccb->ccb_CmdSN);
}

/*
* negotiate_login:
* Control login negotiation.
*
* Parameter:
* conn The connection
* rx_pdu The received login response PDU
* tx_ccb The originally sent login CCB
*/

void
negotiate_login(connection_t *conn, pdu_t *rx_pdu, ccb_t *tx_ccb)
{
int rc;
bool next = TRUE;
pdu_t *tx_pdu;
uint8_t c_phase;

if (rx_pdu->pdu_hdr.pduh_Flags & FLAG_TRANSIT)
c_phase = rx_pdu->pdu_hdr.pduh_Flags & SG_MASK;
else
c_phase = (rx_pdu->pdu_hdr.pduh_Flags >> CSG_SHIFT) & SG_MASK;

DEB(99, ("NegotiateLogin: Flags=%x Phase=%x\n",
rx_pdu->pdu_hdr.pduh_Flags, c_phase));

if (c_phase == SG_FULL_FEATURE_PHASE) {
session_t *sess = conn->c_session;

if (!sess->s_TSIH)
sess->s_TSIH = ((login_isid_t *) &rx_pdu->pdu_hdr.pduh_LUN)->TSIH;

if (rx_pdu->pdu_temp_data != NULL)
assemble_negotiation_parameters(conn, tx_ccb, rx_pdu, NULL);

/* negotiated values are now valid */
set_negotiated_parameters(tx_ccb);

DEBC(conn, 5, ("Login Successful!\n"));
wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
return;
}

tx_pdu = get_pdu(conn, TRUE);
if (tx_pdu == NULL)
return;

tx_pdu->pdu_hdr.pduh_Flags = c_phase << CSG_SHIFT;

switch (c_phase) {
case SG_SECURITY_NEGOTIATION:
rc = assemble_security_parameters(conn, tx_ccb, rx_pdu, tx_pdu);
if (rc < 0)
next = FALSE;
break;

case SG_LOGIN_OPERATIONAL_NEGOTIATION:

if (conn->c_state == ST_SEC_FIN) {

/*
* Both sides announced to continue with
* operational negotation, but this is the
* last target packet from mutual CHAP
* that needs to be validated.
*/
rc = assemble_security_parameters(conn, tx_ccb, rx_pdu, tx_pdu);
if (rc)
break;

/*
* Response was valid, drop (security) parameters
* so that we start negotiating operational
* parameters.
*/
rx_pdu->pdu_temp_data = NULL;
}

rc = assemble_negotiation_parameters(conn, tx_ccb, rx_pdu, tx_pdu);
break;

default:
DEBOUT(("Invalid phase %x in negotiate_login\n", c_phase));
rc = ISCSI_STATUS_TARGET_ERROR;
break;
}

if (rc > 0) {
wake_ccb(tx_ccb, rc);
free_pdu(tx_pdu);
} else {
init_login_pdu(conn, tx_ccb, tx_pdu, next);
setup_tx_uio(tx_pdu, tx_pdu->pdu_temp_data_len, tx_pdu->pdu_temp_data, FALSE);
send_pdu(tx_ccb, tx_pdu, CCBDISP_NOWAIT, PDUDISP_FREE);
}
}

/*
* init_text_pdu:
* Initialize the text PDU.
*
* Parameter:
* conn The connection
* ccb The transmit CCB
* ppdu The transmit PDU
* rx_pdu The received PDU if this is an unsolicited negotiation
*/

STATIC void
init_text_pdu(connection_t *conn, ccb_t *ccb, pdu_t *ppdu, pdu_t *rx_pdu)
{
pdu_header_t *hpdu = &ppdu->pdu_hdr;

hpdu->pduh_Opcode = IOP_Text_Request | OP_IMMEDIATE;
hpdu->pduh_Flags = FLAG_FINAL;

mutex_enter(&conn->c_session->s_lock);
ccb->ccb_CmdSN = get_sernum(conn->c_session, ppdu);
mutex_exit(&conn->c_session->s_lock);

if (rx_pdu != NULL) {
hpdu->pduh_p.text_req.TargetTransferTag =
rx_pdu->pdu_hdr.pduh_p.text_rsp.TargetTransferTag;
hpdu->pduh_LUN = rx_pdu->pdu_hdr.pduh_LUN;
} else
hpdu->pduh_p.text_req.TargetTransferTag = 0xffffffff;

hpdu->pduh_p.text_req.CmdSN = htonl(ccb->ccb_CmdSN);
}

/*
* acknowledge_text:
* Acknowledge a continued login or text response.
*
* Parameter:
* conn The connection
* rx_pdu The received login/text response PDU
* tx_ccb The originally sent login/text request CCB
*/

void
acknowledge_text(connection_t *conn, pdu_t *rx_pdu, ccb_t *tx_ccb)
{
pdu_t *tx_pdu;

tx_pdu = get_pdu(conn, TRUE);
if (tx_pdu == NULL)
return;

if (rx_pdu != NULL &&
(rx_pdu->pdu_hdr.pduh_Opcode & OPCODE_MASK) == IOP_Login_Request)
init_login_pdu(conn, tx_ccb, tx_pdu, FALSE);
else
init_text_pdu(conn, tx_ccb, tx_pdu, rx_pdu);

setup_tx_uio(tx_pdu, 0, NULL, FALSE);
send_pdu(tx_ccb, tx_pdu, CCBDISP_NOWAIT, PDUDISP_FREE);
}

/*
* start_text_negotiation:
* Handle target request to negotiate (via asynch event)
*
* Parameter:
* conn The connection
*/

void
start_text_negotiation(connection_t *conn)
{
pdu_t *pdu;
ccb_t *ccb;

ccb = get_ccb(conn, TRUE);
if (ccb == NULL)
return;
pdu = get_pdu(conn, TRUE);
if (pdu == NULL) {
free_ccb(ccb);
return;
}

if (init_text_parameters(conn, ccb)) {
free_ccb(ccb);
free_pdu(pdu);
return;
}

init_text_pdu(conn, ccb, pdu, NULL);
setup_tx_uio(pdu, 0, NULL, FALSE);
send_pdu(ccb, pdu, CCBDISP_FREE, PDUDISP_WAIT);
}

/*
* negotiate_text:
* Handle received text negotiation.
*
* Parameter:
* conn The connection
* rx_pdu The received text response PDU
* tx_ccb The original CCB
*/

void
negotiate_text(connection_t *conn, pdu_t *rx_pdu, ccb_t *tx_ccb)
{
int rc;
pdu_t *tx_pdu;

if (tx_ccb->ccb_flags & CCBF_SENDTARGET) {
if (!(rx_pdu->pdu_hdr.pduh_Flags & FLAG_FINAL)) {
handle_connection_error(conn, ISCSI_STATUS_PROTOCOL_ERROR,
LOGOUT_CONNECTION);
return;
}
/* transfer ownership of text to CCB */
tx_ccb->ccb_text_data = rx_pdu->pdu_temp_data;
tx_ccb->ccb_text_len = rx_pdu->pdu_temp_data_len;
rx_pdu->pdu_temp_data = NULL;
wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
} else {
if (!(rx_pdu->pdu_hdr.pduh_Flags & FLAG_FINAL))
tx_pdu = get_pdu(conn, TRUE);
else
tx_pdu = NULL;

rc = assemble_negotiation_parameters(conn, tx_ccb, rx_pdu, tx_pdu);
if (rc) {
if (tx_pdu != NULL)
free_pdu(tx_pdu);

handle_connection_error(conn, rc, LOGOUT_CONNECTION);
} else if (tx_pdu != NULL) {
init_text_pdu(conn, tx_ccb, tx_pdu, rx_pdu);
setup_tx_uio(tx_pdu, tx_pdu->pdu_temp_data_len,
tx_pdu->pdu_temp_data, FALSE);
send_pdu(tx_ccb, tx_pdu, CCBDISP_NOWAIT, PDUDISP_FREE);
} else {
set_negotiated_parameters(tx_ccb);
wake_ccb(tx_ccb, ISCSI_STATUS_SUCCESS);
}
}
}

/*
* send_send_targets:
* Send out a SendTargets text request.
* The result is stored in the fields in the session structure.
*
* Parameter:
* session The session
* key The text key to use
*
* Returns: 0 on success, else an error code.
*/

int
send_send_targets(session_t *sess, uint8_t *key)
{
ccb_t *ccb;
pdu_t *pdu;
int rc = 0;
connection_t *conn;

DEB(9, ("Send_send_targets\n"));

conn = assign_connection(sess, TRUE);
if (conn == NULL || conn->c_terminating || conn->c_state != ST_FULL_FEATURE)
return (conn != NULL && conn->c_terminating) ? conn->c_terminating
: ISCSI_STATUS_CONNECTION_FAILED;

ccb = get_ccb(conn, TRUE);
if (ccb == NULL)
return conn->c_terminating;
pdu = get_pdu(conn, TRUE);
if (pdu == NULL) {
free_ccb(ccb);
return conn->c_terminating;
}

ccb->ccb_flags |= CCBF_SENDTARGET;

if ((rc = assemble_send_targets(pdu, key)) != 0) {
free_ccb(ccb);
free_pdu(pdu);
return rc;
}

init_text_pdu(conn, ccb, pdu, NULL);

setup_tx_uio(pdu, pdu->pdu_temp_data_len, pdu->pdu_temp_data, FALSE);
send_pdu(ccb, pdu, CCBDISP_WAIT, PDUDISP_WAIT);

rc = ccb->ccb_status;
if (!rc) {
/* transfer ownership of data */
sess->s_target_list = ccb->ccb_text_data;
sess->s_target_list_len = ccb->ccb_text_len;
ccb->ccb_text_data = NULL;
}
free_ccb(ccb);
return rc;
}

/*
* send_nop_out:
* Send nop out request.
*
* Parameter:
* conn The connection
* rx_pdu The received Nop-In PDU
*
* Returns: 0 on success, else an error code.
*/

int
send_nop_out(connection_t *conn, pdu_t *rx_pdu)
{
session_t *sess;
ccb_t *ccb;
pdu_t *ppdu;
pdu_header_t *hpdu;
uint32_t sn;

if (rx_pdu != NULL) {
ccb = NULL;
ppdu = get_pdu(conn, TRUE);
if (ppdu == NULL)
return 1;
} else {
ccb = get_ccb(conn, FALSE);
if (ccb == NULL) {
DEBOUT(("Can't get CCB in send_nop_out\n"));
return 1;
}
ppdu = get_pdu(conn, FALSE);
if (ppdu == NULL) {
free_ccb(ccb);
DEBOUT(("Can't get PDU in send_nop_out\n"));
return 1;
}
}

hpdu = &ppdu->pdu_hdr;
hpdu->pduh_Flags = FLAG_FINAL;
hpdu->pduh_Opcode = IOP_NOP_Out | OP_IMMEDIATE;

sess = conn->c_session;

mutex_enter(&sess->s_lock);
sn = get_sernum(sess, ppdu);
mutex_exit(&sess->s_lock);

if (rx_pdu != NULL) {
hpdu->pduh_p.nop_out.TargetTransferTag =
rx_pdu->pdu_hdr.pduh_p.nop_in.TargetTransferTag;
hpdu->pduh_InitiatorTaskTag = rx_pdu->pdu_hdr.pduh_InitiatorTaskTag;
hpdu->pduh_p.nop_out.CmdSN = htonl(sn);
hpdu->pduh_LUN = rx_pdu->pdu_hdr.pduh_LUN;
} else {
hpdu->pduh_p.nop_out.TargetTransferTag = 0xffffffff;
hpdu->pduh_InitiatorTaskTag = 0xffffffff;
ccb->ccb_CmdSN = sn;
hpdu->pduh_p.nop_out.CmdSN = htonl(sn);
}

DEBC(conn, 10, ("Send NOP_Out CmdSN=%d, rx_pdu=%p\n", sn, rx_pdu));

setup_tx_uio(ppdu, 0, NULL, FALSE);
send_pdu(ccb, ppdu, (rx_pdu != NULL) ? CCBDISP_NOWAIT : CCBDISP_FREE,
PDUDISP_FREE);
return 0;
}

/*
* snack_missing:
* Send SNACK request for missing data.
*
* Parameter:
* conn The connection
* ccb The task's CCB (for Data NAK only)
* type The SNACK type
* BegRun The BegRun field
* RunLength The RunLength field
*/

void
snack_missing(connection_t *conn, ccb_t *ccb, uint8_t type,
uint32_t BegRun, uint32_t RunLength)
{
pdu_t *ppdu;
pdu_header_t *hpdu;

ppdu = get_pdu(conn, TRUE);
if (ppdu == NULL)
return;
hpdu = &ppdu->pdu_hdr;
hpdu->pduh_Opcode = IOP_SNACK_Request;
hpdu->pduh_Flags = FLAG_FINAL | type;

hpdu->pduh_InitiatorTaskTag = (type == SNACK_DATA_NAK) ? ccb->ccb_ITT : 0xffffffff;
hpdu->pduh_p.snack.TargetTransferTag = 0xffffffff;
hpdu->pduh_p.snack.BegRun = htonl(BegRun);
hpdu->pduh_p.snack.RunLength = htonl(RunLength);

ppdu->pdu_flags = PDUF_PRIORITY;

setup_tx_uio(ppdu, 0, NULL, FALSE);
send_pdu(NULL, ppdu, CCBDISP_NOWAIT, PDUDISP_FREE);
}

/*
* send_snack:
* Send SNACK request.
*
* Parameter:
* conn The connection
* rx_pdu The received data in PDU
* tx_ccb The original command CCB (required for Data ACK only)
* type The SNACK type
*
* Returns: 0 on success, else an error code.
*/

void
send_snack(connection_t *conn, pdu_t *rx_pdu, ccb_t *tx_ccb, uint8_t type)
{
pdu_t *ppdu;
pdu_header_t *hpdu;

ppdu = get_pdu(conn, TRUE);
if (ppdu == NULL)
return;
hpdu = &ppdu->pdu_hdr;
hpdu->pduh_Opcode = IOP_SNACK_Request;
hpdu->pduh_Flags = FLAG_FINAL | type;

switch (type) {
case SNACK_DATA_NAK:
hpdu->pduh_InitiatorTaskTag = rx_pdu->pdu_hdr.pduh_InitiatorTaskTag;
hpdu->pduh_p.snack.TargetTransferTag = 0xffffffff;
hpdu->pduh_p.snack.BegRun = rx_pdu->pdu_hdr.pduh_p.data_in.DataSN;
hpdu->pduh_p.snack.RunLength = htonl(1);
break;

case SNACK_STATUS_NAK:
hpdu->pduh_InitiatorTaskTag = 0xffffffff;
hpdu->pduh_p.snack.TargetTransferTag = 0xffffffff;
hpdu->pduh_p.snack.BegRun = rx_pdu->pdu_hdr.pduh_p.response.StatSN;
hpdu->pduh_p.snack.RunLength = htonl(1);
break;

case SNACK_DATA_ACK:
hpdu->pduh_InitiatorTaskTag = 0xffffffff;
hpdu->pduh_p.snack.TargetTransferTag =
rx_pdu->pdu_hdr.pduh_p.data_in.TargetTransferTag;
hpdu->pduh_p.snack.BegRun = tx_ccb->ccb_DataSN_buf.ExpSN;
hpdu->pduh_p.snack.RunLength = 0;
break;

default:
DEBOUT(("Invalid type %d in send_snack\n", type));
return;
}

hpdu->pduh_LUN = rx_pdu->pdu_hdr.pduh_LUN;

ppdu->pdu_flags = PDUF_PRIORITY;

setup_tx_uio(ppdu, 0, NULL, FALSE);
send_pdu(NULL, ppdu, CCBDISP_NOWAIT, PDUDISP_FREE);
}

/*
* send_login:
* Send login request.
*
* Parameter:
* conn The connection
* par The login parameters (for negotiation)
*
* Returns: 0 on success, else an error code.
*/

int
send_login(connection_t *conn)
{
ccb_t *ccb;
pdu_t *pdu;
int rc;

DEBC(conn, 9, ("Send_login\n"));
ccb = get_ccb(conn, TRUE);
/* only if terminating (which couldn't possibly happen here, but...) */
if (ccb == NULL)
return conn->c_terminating;
pdu = get_pdu(conn, TRUE);
if (pdu == NULL) {
free_ccb(ccb);
return conn->c_terminating;
}

if ((rc = assemble_login_parameters(conn, ccb, pdu)) <= 0) {
init_login_pdu(conn, ccb, pdu, !rc);
setup_tx_uio(pdu, pdu->pdu_temp_data_len, pdu->pdu_temp_data, FALSE);
send_pdu(ccb, pdu, CCBDISP_WAIT, PDUDISP_FREE);
rc = ccb->ccb_status;
} else {
free_pdu(pdu);
}
free_ccb(ccb);
return rc;
}

/*
* send_logout:
* Send logout request.
* NOTE: This function does not wait for the logout to complete.
*
* Parameter:
* conn The connection
* refconn The referenced connection
* reason The reason code
* wait Wait for completion if TRUE
*
* Returns: 0 on success (logout sent), else an error code.
*/

int
send_logout(connection_t *conn, connection_t *refconn, int reason,
bool wait)
{
ccb_t *ccb;
pdu_t *ppdu;
pdu_header_t *hpdu;

DEBC(conn, 5, ("Send_logout\n"));
ccb = get_ccb(conn, TRUE);
/* can only happen if terminating... */
if (ccb == NULL)
return conn->c_terminating;
ppdu = get_pdu(conn, TRUE);
if (ppdu == NULL) {
free_ccb(ccb);
return conn->c_terminating;
}

hpdu = &ppdu->pdu_hdr;
hpdu->pduh_Opcode = IOP_Logout_Request | OP_IMMEDIATE;

hpdu->pduh_Flags = FLAG_FINAL | reason;
ccb->ccb_CmdSN = conn->c_session->s_CmdSN;
hpdu->pduh_p.logout_req.CmdSN = htonl(ccb->ccb_CmdSN);
if (reason > 0)
hpdu->pduh_p.logout_req.CID = htons(refconn->c_id);

ccb->ccb_par = refconn;
if (refconn != conn) {
ccb->ccb_flags |= CCBF_OTHERCONN;
} else {
conn->c_state = ST_LOGOUT_SENT;
conn->c_loggedout = LOGOUT_SENT;
}

setup_tx_uio(ppdu, 0, NULL, FALSE);
send_pdu(ccb, ppdu, (wait) ? CCBDISP_WAIT : CCBDISP_FREE, PDUDISP_FREE);

if (wait) {
int rc = ccb->ccb_status;
free_ccb (ccb);
return rc;
}
return 0;
}

/*
* send_task_management:
* Send task management request.
*
* Parameter:
* conn The connection
* ref_ccb The referenced command (NULL if none)
* xs The scsipi command structure (NULL if not a scsipi request)
* function The function code
*
* Returns: 0 on success, else an error code.
*/

int
send_task_management(connection_t *conn, ccb_t *ref_ccb, struct scsipi_xfer *xs,
int function)
{
ccb_t *ccb;
pdu_t *ppdu;
pdu_header_t *hpdu;

DEBC(conn, 5, ("Send_task_management, ref_ccb=%p, func = %d\n",
ref_ccb, function));

if (function == TASK_REASSIGN && conn->c_session->s_ErrorRecoveryLevel < 2)
return ISCSI_STATUS_CANT_REASSIGN;

ccb = get_ccb(conn, xs == NULL);
/* can only happen if terminating... */
if (ccb == NULL) {
DEBC(conn, 0, ("send_task_management, ref_ccb=%p, xs=%p, term=%d. No CCB\n",
ref_ccb, xs, conn->c_terminating));
return conn->c_terminating;
}
ppdu = get_pdu(conn, xs == NULL);
if (ppdu == NULL) {
DEBC(conn, 0, ("send_task_management, ref_ccb=%p, xs=%p, term=%d. No PDU\n",
ref_ccb, xs, conn->c_terminating));
free_ccb(ccb);
return conn->c_terminating;
}

ccb->ccb_xs = xs;

hpdu = &ppdu->pdu_hdr;
hpdu->pduh_Opcode = IOP_SCSI_Task_Management | OP_IMMEDIATE;
hpdu->pduh_Flags = FLAG_FINAL | function;

ccb->ccb_CmdSN = conn->c_session->s_CmdSN;
hpdu->pduh_p.task_req.CmdSN = htonl(ccb->ccb_CmdSN);

if (ref_ccb != NULL) {
hpdu->pduh_p.task_req.ReferencedTaskTag = ref_ccb->ccb_ITT;
hpdu->pduh_p.task_req.RefCmdSN = htonl(ref_ccb->ccb_CmdSN);
hpdu->pduh_p.task_req.ExpDataSN = htonl(ref_ccb->ccb_DataSN_buf.ExpSN);
} else
hpdu->pduh_p.task_req.ReferencedTaskTag = 0xffffffff;

ppdu->pdu_flags |= PDUF_PRIORITY;

setup_tx_uio(ppdu, 0, NULL, FALSE);
send_pdu(ccb, ppdu, (xs) ? CCBDISP_SCSIPI : CCBDISP_WAIT, PDUDISP_FREE);

if (xs == NULL) {
int rc = ccb->ccb_status;
free_ccb(ccb);
return rc;
}
return 0;
}

/*
* send_data_out:
* Send data to target in response to an R2T or as unsolicited data.
*
* Parameter:
* conn The connection
* rx_pdu The received R2T PDU (NULL if unsolicited)
* tx_ccb The originally sent command CCB
* waitok Whether it's OK to wait for an available PDU or not
*/

int
send_data_out(connection_t *conn, pdu_t *rx_pdu, ccb_t *tx_ccb,
ccb_disp_t disp, bool waitok)
{
pdu_header_t *hpdu;
uint32_t totlen, len, offs, sn;
pdu_t *tx_pdu;

KASSERT(conn->c_max_transfer != 0);

if (rx_pdu) {
offs = ntohl(rx_pdu->pdu_hdr.pduh_p.r2t.BufferOffset);
totlen = ntohl(rx_pdu->pdu_hdr.pduh_p.r2t.DesiredDataTransferLength);
} else {
offs = conn->c_max_firstimmed;
totlen = min(conn->c_max_firstdata - offs, tx_ccb->ccb_data_len - offs);
}
sn = 0;

while (totlen) {
len = min(totlen, conn->c_max_transfer);

tx_pdu = get_pdu(conn, waitok);
if (tx_pdu == NULL) {
DEBC(conn, 5, ("No PDU in send_data_out\n"));

tx_ccb->ccb_disp = disp;
tx_ccb->ccb_status = ISCSI_STATUS_NO_RESOURCES;
handle_connection_error(conn, ISCSI_STATUS_NO_RESOURCES, NO_LOGOUT);

return ISCSI_STATUS_NO_RESOURCES;
}

totlen -= len;
hpdu = &tx_pdu->pdu_hdr;
hpdu->pduh_Opcode = IOP_SCSI_Data_out;
if (!totlen)
hpdu->pduh_Flags = FLAG_FINAL;

if (rx_pdu != NULL)
hpdu->pduh_p.data_out.TargetTransferTag =
rx_pdu->pdu_hdr.pduh_p.r2t.TargetTransferTag;
else
hpdu->pduh_p.data_out.TargetTransferTag = 0xffffffff;
hpdu->pduh_p.data_out.BufferOffset = htonl(offs);
hpdu->pduh_p.data_out.DataSN = htonl(sn);

DEBC(conn, 10, ("Send DataOut: DataSN %d, len %d offs %x totlen %d\n",
sn, len, offs, totlen));

setup_tx_uio(tx_pdu, len, tx_ccb->ccb_data_ptr + offs, FALSE);
send_pdu(tx_ccb, tx_pdu, (totlen) ? CCBDISP_NOWAIT : disp, PDUDISP_FREE);

sn++;
offs += len;
}
return 0;
}

/*
* send_command:
* Send a SCSI command request.
*
* Parameter:
* CCB The CCB
* disp The CCB disposition
*/

void
send_command(ccb_t *ccb, ccb_disp_t disp, bool waitok, bool immed)
{
uint32_t totlen, len;
connection_t *conn = ccb->ccb_connection;
session_t *sess = ccb->ccb_session;
pdu_t *ppdu;
pdu_header_t *hpdu;

mutex_enter(&sess->s_lock);
while (!sernum_in_window(sess)) {
mutex_exit(&sess->s_lock);
ccb->ccb_disp = disp;
wake_ccb(ccb, ISCSI_STATUS_QUEUE_FULL);
return;
}
mutex_exit(&sess->s_lock);

/* Don't confuse targets during (re-)negotations */
if (conn->c_state != ST_FULL_FEATURE) {
DEBOUT(("Invalid connection for send_command, ccb = %p\n",ccb));
ccb->ccb_disp = disp;
wake_ccb(ccb, ISCSI_STATUS_TARGET_BUSY);
return;
}

ppdu = get_pdu(conn, waitok);
if (ppdu == NULL) {
DEBOUT(("No PDU for send_command, ccb = %p\n",ccb));
ccb->ccb_disp = disp;
wake_ccb(ccb, ISCSI_STATUS_NO_RESOURCES);
return;
}

totlen = len = ccb->ccb_data_len;

hpdu = &ppdu->pdu_hdr;
hpdu->pduh_LUN = htonq(ccb->ccb_lun);
memcpy(hpdu->pduh_p.command.SCSI_CDB, ccb->ccb_cmd, ccb->ccb_cmdlen);
hpdu->pduh_Opcode = IOP_SCSI_Command;
if (immed)
hpdu->pduh_Opcode |= OP_IMMEDIATE;
hpdu->pduh_p.command.ExpectedDataTransferLength = htonl(totlen);

if (totlen) {
if (ccb->ccb_data_in) {
hpdu->pduh_Flags = FLAG_READ;
totlen = 0;
} else {
hpdu->pduh_Flags = FLAG_WRITE;
/* immediate data we can send */
len = min(totlen, conn->c_max_firstimmed);

/* can we send more unsolicited data ? */
totlen = conn->c_max_firstdata ? totlen - len : 0;
}
}
if (!totlen)
hpdu->pduh_Flags |= FLAG_FINAL;
hpdu->pduh_Flags |= ccb->ccb_tag;

if (ccb->ccb_data_in)
init_sernum(&ccb->ccb_DataSN_buf);

ccb->ccb_sense_len_got = 0;
ccb->ccb_xfer_len = 0;
ccb->ccb_residual = 0;
ccb->ccb_flags |= CCBF_REASSIGN;

mutex_enter(&sess->s_lock);
ccb->ccb_CmdSN = get_sernum(sess, ppdu);
mutex_exit(&sess->s_lock);

hpdu->pduh_p.command.CmdSN = htonl(ccb->ccb_CmdSN);

DEBC(conn, 10, ("Send Command: CmdSN %d (%d), data_in %d, len %d, totlen %d\n",
ccb->ccb_CmdSN, sess->s_MaxCmdSN, ccb->ccb_data_in, len, totlen));

setup_tx_uio(ppdu, len, ccb->ccb_data_ptr, ccb->ccb_data_in);
send_pdu(ccb, ppdu, (totlen) ? CCBDISP_DEFER : disp, PDUDISP_WAIT);

if (totlen)
send_data_out(conn, NULL, ccb, disp, waitok);
}

/*
* send_run_xfer:
* Handle a SCSI command transfer request from scsipi.
*
* Parameter:
* session The session
* xs The transfer parameters
*/

void
send_run_xfer(session_t *session, struct scsipi_xfer *xs)
{
ccb_t *ccb;
connection_t *conn;
bool waitok;

waitok = !(xs->xs_control & XS_CTL_NOSLEEP);

DEB(10, ("RunXfer: flags=%x, data=%p, datalen=%d, resid=%d, cmdlen=%d, "
"waitok=%d\n", xs->xs_control, xs->data, xs->datalen,
xs->resid, xs->cmdlen, waitok));

conn = assign_connection(session, waitok);

if (conn == NULL || conn->c_terminating || conn->c_state != ST_FULL_FEATURE) {
if (session->s_terminating)
xs->error = XS_SELTIMEOUT;
else
xs->error = XS_BUSY;
DEBC(conn, 10, ("run_xfer on dead connection\n"));
scsipi_done(xs);
unref_session(session);
return;
}

if (xs->xs_control & XS_CTL_RESET) {
if (send_task_management(conn, NULL, xs, TARGET_WARM_RESET)) {
DEBC(conn, 0, ("send_task_management TARGET_WARM_RESET failed\n"));
xs->error = XS_SELTIMEOUT;
scsipi_done(xs);
unref_session(session);
}
return;
}

ccb = get_ccb(conn, waitok);
if (ccb == NULL) {
xs->error = XS_BUSY;
DEBC(conn, 5, ("No CCB in run_xfer, %d in use.\n", conn->c_usecount));
scsipi_done(xs);
unref_session(session);
return;
}
/* copy parameters into CCB for easier access */
ccb->ccb_xs = xs;

ccb->ccb_data_in = (xs->xs_control & XS_CTL_DATA_IN) != 0;
ccb->ccb_data_len = (uint32_t) xs->datalen;
ccb->ccb_data_ptr = xs->data;

ccb->ccb_sense_len_req = sizeof(xs->sense.scsi_sense);
ccb->ccb_sense_ptr = &xs->sense;

ccb->ccb_lun = ((uint64_t) (uint8_t) xs->xs_periph->periph_lun) << 48;
ccb->ccb_cmd = (uint8_t *) xs->cmd;
ccb->ccb_cmdlen = xs->cmdlen;
DEB(10, ("RunXfer: Periph_lun = %d, cmd[1] = %x, cmdlen = %d\n",
xs->xs_periph->periph_lun, ccb->ccb_cmd[1], xs->cmdlen));

ccb->ccb_ITT |= xs->xs_tag_id << 24;
switch (xs->xs_tag_type) {
case MSG_ORDERED_Q_TAG:
ccb->ccb_tag = ATTR_ORDERED;
break;
case MSG_SIMPLE_Q_TAG:
ccb->ccb_tag = ATTR_SIMPLE;
break;
case MSG_HEAD_OF_Q_TAG:
ccb->ccb_tag = ATTR_HEAD_OF_QUEUE;
break;
default:
ccb->ccb_tag = 0;
break;
}

#ifdef LUN_1
ccb->ccb_lun += 0x1000000000000LL;
ccb->ccb_cmd[1] += 0x10;
#endif
send_command(ccb, CCBDISP_SCSIPI, waitok, FALSE);
}

#ifndef ISCSI_MINIMAL
/*
* send_io_command:
* Handle a SCSI io command request from user space.
*
* Parameter:
* session The session
* lun The LUN to use
* req The SCSI request block
* immed Immediate command if TRUE
* conn_id Assign to this connection ID if nonzero
*/

int
send_io_command(session_t *session, uint64_t lun, scsireq_t *req,
bool immed, uint32_t conn_id)
{
ccb_t *ccb;
connection_t *conn;
int rc;

DEB(9, ("IoCommand: lun=%x, datalen=%d, cmdlen=%d, immed=%d, cid=%d\n",
(int) lun, (int) req->datalen, (int) req->cmdlen, immed, conn_id));

conn = (conn_id) ? find_connection(session, conn_id)
: assign_connection(session, TRUE);

if (conn == NULL || conn->c_terminating || conn->c_state != ST_FULL_FEATURE) {
DEBOUT(("io_command on dead connection (state = %d)\n",
(conn != NULL) ? conn->c_state : -1));
return ISCSI_STATUS_INVALID_CONNECTION_ID;
}

ccb = get_ccb(conn, TRUE);
if (ccb == NULL) {
DEBOUT(("No CCB in io_command\n"));
return ISCSI_STATUS_NO_RESOURCES;
}

ccb->ccb_data_in = (req->flags & SCCMD_READ) != 0;
ccb->ccb_data_len = (uint32_t) req->datalen;
ccb->ccb_data_ptr = req->databuf;

ccb->ccb_sense_len_req = req->senselen;
ccb->ccb_sense_ptr = &req->sense;

ccb->ccb_lun = lun;
ccb->ccb_cmd = (uint8_t *) req->cmd;
ccb->ccb_cmdlen = req->cmdlen;
DEBC(conn, 10, ("IoCommand: cmd[1] = %x, cmdlen = %d\n",
ccb->ccb_cmd[1], ccb->ccb_cmdlen));

send_command(ccb, CCBDISP_WAIT, TRUE, immed);

rc = ccb->ccb_status;

req->senselen_used = ccb->ccb_sense_len_got;
req->datalen_used = req->datalen - ccb->ccb_residual;

free_ccb(ccb);

return rc;
}
#endif

/*****************************************************************************
* Timeout handlers
*****************************************************************************/
/*
* connection_timeout:
* Handle prolonged silence on a connection by checking whether
* it's still alive.
* This has the side effect of discovering missing status or lost commands
* before those time out.
*
* Parameter:
* conn The connection
*/

void
connection_timeout(connection_t *conn)
{
if (++conn->c_num_timeouts > MAX_CONN_TIMEOUTS) {
DEBC(conn, 1, ("connection timeout %d\n", conn->c_num_timeouts));
handle_connection_error(conn, ISCSI_STATUS_TIMEOUT, NO_LOGOUT);
} else {
if (conn->c_state == ST_FULL_FEATURE)
send_nop_out(conn, NULL);

connection_timeout_start(conn, CONNECTION_TIMEOUT);
}
}

/*
* ccb_timeout:
* Handle timeout of a sent command.
*
* Parameter:
* ccb The CCB
*/

void
ccb_timeout(ccb_t *ccb)
{
connection_t *conn = ccb->ccb_connection;

if (conn == NULL) {
/* XXX Should never happen */
printf("ccb_timeout: num=%d total=%d disp=%d invalid ccb=%p\n",
ccb->ccb_num_timeouts+1, ccb->ccb_total_tries,
ccb->ccb_disp, ccb);
return;
}

ccb->ccb_total_tries++;

DEBC(conn, 0, ("ccb_timeout: num=%d total=%d disp=%d\n",
ccb->ccb_num_timeouts+1, ccb->ccb_total_tries, ccb->ccb_disp));

if (++ccb->ccb_num_timeouts > MAX_CCB_TIMEOUTS ||
ccb->ccb_total_tries > MAX_CCB_TRIES ||
ccb->ccb_disp <= CCBDISP_FREE ||
!ccb->ccb_session->s_ErrorRecoveryLevel) {

wake_ccb(ccb, ISCSI_STATUS_TIMEOUT);
handle_connection_error(conn,
ISCSI_STATUS_TIMEOUT, RECOVER_CONNECTION);
} else {
if (ccb->ccb_data_in && ccb->ccb_xfer_len < ccb->ccb_data_len) {
/* request resend of all missing data */
snack_missing(conn, ccb, SNACK_DATA_NAK, 0, 0);
} else {
/* request resend of all missing status */
snack_missing(conn, NULL, SNACK_STATUS_NAK, 0, 0);
}
ccb_timeout_start(ccb, COMMAND_TIMEOUT);
}
}